diff --git a/src/orchestration/dags/config/etl.conf b/src/orchestration/dags/config/etl.conf index f131ea69..509b9f6f 100644 --- a/src/orchestration/dags/config/etl.conf +++ b/src/orchestration/dags/config/etl.conf @@ -22,35 +22,6 @@ spark_settings: { # ============================= STEP CONFIGURATION ============================= steps: { - reactome: { - input: { - pathways: { - format: "csv" - path: ${common.path}"/input/reactome/ReactomePathways.txt" - options: [ - { k: "sep", v: "\\t" } - { k: "header", v: false } - { k: "inferSchema", v: true } - ] - } - relations: { - format: "csv" - path: ${common.path}"/input/reactome/ReactomePathwaysRelation.txt" - options: [ - { k: "sep", v: "\\t" } - { k: "header", v: false } - { k: "inferSchema", v: true } - ] - } - } - output: { - reactome: { - format: ${common.output_format} - path: ${common.output_path}"/intermediate/reactome" - } - } - } - expression: { input: { rna: { diff --git a/src/orchestration/dags/config/pts.yaml b/src/orchestration/dags/config/pts.yaml index 800eb619..0e2ad44f 100644 --- a/src/orchestration/dags/config/pts.yaml +++ b/src/orchestration/dags/config/pts.yaml @@ -38,6 +38,16 @@ steps: destination: output/go/go.parquet ################################################################################################## + #: REACTOME STEP :################################################################################ + reactome: + - name: pyspark reactome + pyspark: reactome + source: + pathways: input/reactome/ReactomePathways.txt + relations: input/reactome/ReactomePathwaysRelation.txt + destination: output/reactome + ################################################################################################## + #: EXPRESSION STEP :############################################################################## expression: - name: unzip normal tissue diff --git a/src/orchestration/dags/config/unified_pipeline.yaml b/src/orchestration/dags/config/unified_pipeline.yaml index f46dcc83..38c1a54b 100644 --- a/src/orchestration/dags/config/unified_pipeline.yaml +++ b/src/orchestration/dags/config/unified_pipeline.yaml @@ -95,6 +95,9 @@ steps: pis_target: pis_target_prioritisation: # PTS STEPS + pts_reactome: + depends_on: + - pis_reactome pts_go: depends_on: - pis_go @@ -406,10 +409,6 @@ steps: num_partitions: 4 depends_on: - pts_expression - etl_reactome: - num_partitions: 1 - depends_on: - - pis_reactome etl_otar: ppp_only: true depends_on: @@ -420,7 +419,7 @@ steps: depends_on: - pts_disease - pts_target - - etl_reactome + - pts_reactome - pts_target_safety - pts_target_gene_essentiality etl_search_facet: