Spdi normalization with refseq files

.github/workflows/main.yml

@@ -28,20 +28,24 @@ jobs:
           args: --extend-ignore E501,E741
 
       - name: Run Tests
-        run: python -m pytest
+        # run: ./fetch_refseq.sh python -m pytest
+        run: "echo ${heroku_app_name} && false"
+        with:
+          heroku_app_name: ${{secrets.HEROKU_DEV_APP_NAME}}
 
+  # TODO: Add a way to deploy to Prod manually
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
 
-    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && always() && contains(join(needs.*.result, ','), 'success') }}
+    if: ${{ contains(join(needs.*.result, ','), 'success') }}
     needs: [test]
 
     steps:
       - uses: actions/checkout@v2
 
-      - uses: akhileshns/heroku-deploy@v3.12.12
+      - uses: akhileshns/heroku-deploy@v3.12.14
         with:
           heroku_api_key: ${{secrets.HEROKU_API_KEY}}
-          heroku_app_name: ${{secrets.HEROKU_APP_NAME}}
+          heroku_app_name: ${{secrets.HEROKU_DEV_APP_NAME}}
           heroku_email: ${{secrets.HEROKU_EMAIL}}

Procfile

@@ -1 +1 @@
-web: gunicorn run:app
+web: ./fetch_refseq.sh && gunicorn run:app

README.md

@@ -48,3 +48,9 @@ run `python3 -m pytest` from the terminal to execute them all.
 Additionally, since the tests run against the Mongo DB database, if you need to update the test data in this repo, you
 can run `OVERWRITE_TEST_EXPECTED_DATA=true python3 -m pytest` from the terminal and then create a pull request with the
 changes.
+
+## Development environment on Heroku
+
+Pull requests will trigger a deployment to this environment automatically which is accessible at the following URL:
+
+https://fhir-gen-ops-dev-ca42373833b6.herokuapp.com/

app/api_spec.yml

@@ -102,7 +102,7 @@ paths:
             type: boolean
             default: false
           description: Include sequence phase relationships in response if set to true.
-  
+
   /subject-operations/genotype-operations/$find-subject-specific-variants:
     get:
       description: >
@@ -177,7 +177,7 @@ paths:
               - "germline"
               - "somatic"
           description: Enables an App to limit results to those that are 'germline' or 'somatic'. Default is to include variants irrespective of genomic source class.
-  
+
   /subject-operations/genotype-operations/$find-subject-structural-intersecting-variants:
     get:
       description: >
@@ -262,7 +262,7 @@ paths:
             type: boolean
             default: false
           description: Include variants in response if set to true.
-  
+
   /subject-operations/genotype-operations/$find-subject-structural-subsuming-variants:
     get:
       description: >
@@ -346,7 +346,7 @@ paths:
             type: boolean
             default: false
           description: Include variants in response if set to true.
-  
+
   /subject-operations/genotype-operations/$find-subject-haplotypes:
     get:
       description: >
@@ -422,7 +422,7 @@ paths:
               - "germline"
               - "somatic"
           description: Enables an App to limit results to those that are 'germline' or 'somatic'. Default is to include variants irrespective of genomic source class.
-  
+
   /subject-operations/genotype-operations/$find-subject-specific-haplotypes:
     get:
       description: >
@@ -497,7 +497,7 @@ paths:
               - "germline"
               - "somatic"
           description: Enables an App to limit results to those that are 'germline' or 'somatic'. Default is to include variants irrespective of genomic source class.
-  
+
   /subject-operations/phenotype-operations/$find-subject-tx-implications:
     get:
       description: |-
@@ -614,7 +614,7 @@ paths:
               - "germline"
               - "somatic"
           description: Enables an App to limit results to those that are 'germline' or 'somatic'. Default is to include variants irrespective of genomic source class.
-  
+
   /subject-operations/phenotype-operations/$find-subject-dx-implications:
     get:
       description: |-
@@ -713,7 +713,7 @@ paths:
               - "germline"
               - "somatic"
           description: Enables an App to limit results to those that are 'germline' or 'somatic'. Default is to include variants irrespective of genomic source class.
-  
+
   /subject-operations/metadata-operations/$find-study-metadata:
     get:
       description: |-
@@ -1147,6 +1147,7 @@ paths:
             type: string
             pattern: '^\s*[Nn][Pp]_\d{4,10}(\.)?(\d{1,2})?\s*$'
           example: "NP_000005.3"
+
   /utilities/find-the-gene:
     get:
       summary: "Find The Gene"
@@ -1170,6 +1171,42 @@ paths:
             pattern: '^\s*[Nn][Cc]_\d{4,10}(\.)(\d{1,2}):\d{1,10}-\d{1,10}\s*$'
             example: "NC_000001.11:11794399-11794400"
 
+  /utilities/seqfetcher/1/sequence/{ref_seq}:
+    get:
+      summary: "Seqfetcher"
+      operationId: "app.utilities_endpoints.seqfetcher"
+      tags:
+        - "Seqfetcher Utility"
+      responses:
+        '200':
+          description: "Returns RefSeq subsequence"
+          content:
+            text/plain:
+              schema:
+                type: string
+      parameters:
+        - name: ref_seq
+          in: path
+          required: true
+          description: RefSeq
+          schema:
+            type: string
+            example: "NC_000001.10"
+        - name: start
+          in: query
+          required: true
+          description: Subsequence start index
+          schema:
+            type: integer
+            example: 1
+        - name: end
+          in: query
+          required: true
+          description: Subsequence end index
+          schema:
+            type: integer
+            example: 2
+
 tags:
   - name: Subject Genotype Operations
   - name: Subject Phenotype Operations
@@ -1178,7 +1215,7 @@ tags:
   - name: Population Phenotype Operations
   - name: Feature Coordinates Utility
     description: This utility returns genomic feature coordinates and other annotations. All data are from <a href="https://www.ncbi.nlm.nih.gov/genome/guide/human/">NCBI Human Genome Resources</a>. For chromosomes, build 37 and build 38 reference sequences are returned. For genes, genomic coordinates are returned, along with a list of transcripts. MANE transcript is flagged. For transcripts, genomic coordinates are returned, along with the gene name and composite exons, along with exon coordinates. For proteins, the corresponding transcript is returned.
-  
+
   - name: Find The Gene Utility
     description: This utility returns all genes that intersect with a provided genomic region.
 

app/common.py

@@ -2,12 +2,12 @@
 from threading import Lock
 from uuid import uuid4
 import pyliftover
-import requests
 from datetime import datetime
 import pymongo
 from flask import abort
 from itertools import groupby
 import re
+from .input_normalization import normalize
 
 # MongoDB Client URIs
 FHIR_genomics_data_client_uri = "mongodb+srv://download:download@cluster0.8ianr.mongodb.net/FHIRGenomicsData"
@@ -116,10 +116,6 @@ def get_liftover(from_db, to_db):
 
 SUPPORTED_GENOMIC_SOURCE_CLASSES = ['germline', 'somatic']
 
-NCBI_VARIATION_SERVICES_BASE_URL = 'https://api.ncbi.nlm.nih.gov/variation/v0/'
-
-CHROMOSOME_CSV_FILE = 'app/_Dict_Chromosome.csv'
-
 # Utility Functions
 
 
@@ -163,26 +159,6 @@ def merge_ranges(ranges):
     return merged_ranges
 
 
-def get_hgvs_contextuals_url(hgvs):
-    return f"{NCBI_VARIATION_SERVICES_BASE_URL}hgvs/{hgvs}/contextuals"
-
-
-def get_spdi_all_equivalent_contextual_url(contextual_SPDI):
-    return f'{NCBI_VARIATION_SERVICES_BASE_URL}spdi/{contextual_SPDI}/all_equivalent_contextual'
-
-
-def get_spdi_canonical_representative_url(contextual_SPDI):
-    return f'{NCBI_VARIATION_SERVICES_BASE_URL}spdi/{contextual_SPDI}/canonical_representative'
-
-
-def build_spdi(seq_id, position, deleted_sequence, inserted_sequence):
-    return f"{seq_id}:{position}:{deleted_sequence}:{inserted_sequence}"
-
-
-def get_spdi_elements(response_object):
-    return (response_object['seq_id'], response_object['position'], response_object['deleted_sequence'], response_object['inserted_sequence'])
-
-
 def validate_subject(patient_id):
     if not patients_db.find_one({"patientID": patient_id}):
         abort(400, f"Patient ({patient_id}) not found.")
@@ -196,7 +172,7 @@ def get_variant(variant):
     variant = variant.lstrip()
 
     if variant.count(":") == 1:  # HGVS expression
-        SPDIs = hgvs_2_contextual_SPDIs(variant)
+        SPDIs = normalize(variant)
         if not SPDIs:
             abort(400, f'Cannot normalize variant: {variant}')
         elif not SPDIs["GRCh37"] and not SPDIs["GRCh38"]:
@@ -205,7 +181,7 @@ def get_variant(variant):
             normalized_variant = {"variant": variant, "GRCh37": SPDIs["GRCh37"], "GRCh38": SPDIs["GRCh38"]}
 
     elif variant.count(":") == 3:  # SPDI expression
-        SPDIs = SPDI_2_contextual_SPDIs(variant)
+        SPDIs = normalize(variant)
         if not SPDIs:
             abort(400, f'Cannot normalize variant: {variant}')
         elif not SPDIs["GRCh37"] and not SPDIs["GRCh38"]:
@@ -1002,136 +978,6 @@ def get_intersected_regions(bed_id, build, chrom, start, end, intersected_region
             intersected_regions.append(f'{ref_seq}:{max(start, csePair["Start"])}-{min(end, csePair["End"])}')
 
 
-def hgvs_2_contextual_SPDIs(hgvs):
-
-    # convert hgvs to contextualSPDI
-    url = get_hgvs_contextuals_url(hgvs)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_data = response['data']
-    raw_SPDI = raw_data['spdis'][0]
-
-    seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(raw_SPDI)
-
-    contextual_SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-
-    # convert contextualSPDI to build37 and build38 contextual SPDIs
-    url = get_spdi_all_equivalent_contextual_url(contextual_SPDI)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_SPDI_List = response['data']['spdis']
-
-    b37SPDI = None
-    b38SPDI = None
-    for item in raw_SPDI_List:
-        if item['seq_id'].startswith("NC_"):
-            temp = get_build_and_chrom_by_ref_seq(item['seq_id'])
-            if temp:
-                seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(item)
-
-                if temp['build'] == 'GRCh37':
-                    b37SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-                elif temp['build'] == 'GRCh38':
-                    b38SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-            else:
-                return False
-
-    return {"GRCh37": b37SPDI, "GRCh38": b38SPDI}
-
-
-def hgvs_2_canonical_SPDI(hgvs):
-
-    # convert hgvs to contextualSPDI
-    url = get_hgvs_contextuals_url(hgvs)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_data = response['data']
-    raw_SPDI = raw_data['spdis'][0]
-
-    seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(raw_SPDI)
-
-    contextual_SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-
-    # convert contextualSPDI to canonical SPDI
-    url = get_spdi_canonical_representative_url(contextual_SPDI)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_SPDI = response['data']
-
-    seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(raw_SPDI)
-
-    canonical_SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-
-    return {"canonicalSPDI": canonical_SPDI}
-
-
-def SPDI_2_contextual_SPDIs(spdi):
-    url = get_spdi_all_equivalent_contextual_url(spdi)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_SPDI_List = response['data']['spdis']
-
-    b37SPDI = None
-    b38SPDI = None
-    for item in raw_SPDI_List:
-        if item['seq_id'].startswith("NC_"):
-            temp = get_build_and_chrom_by_ref_seq(item['seq_id'])
-            if temp:
-                seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(item)
-
-                if temp['build'] == 'GRCh37':
-                    b37SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-                elif temp['build'] == 'GRCh38':
-                    b38SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-            else:
-                return False
-
-    return {"GRCh37": b37SPDI, "GRCh38": b38SPDI}
-
-
-def SPDI_2_canonical_SPDI(spdi):
-    url = get_spdi_canonical_representative_url(spdi)
-    headers = {'Accept': 'application/json'}
-
-    r = requests.get(url, headers=headers)
-    if r.status_code != 200:
-        return False
-
-    response = r.json()
-    raw_SPDI = response['data']
-
-    seq_id, position, deleted_sequence, inserted_sequence = get_spdi_elements(raw_SPDI)
-
-    canonical_SPDI = build_spdi(seq_id, position, deleted_sequence, inserted_sequence)
-
-    return {"canonicalSPDI": canonical_SPDI}
-
-
 def query_clinvar_by_variants(normalized_variant_list, code_list, query, population=False):
     variant_list = []
     for item in normalized_variant_list: