From bf5541ce7db67b0ce6999f2b3eeb8e1280267085 Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:26:50 +1000 Subject: [PATCH 1/7] Update import examples to align with TIG (use placeholder --- .../schema/2_populateTables.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/AMTv4-sample-scripts/schema/2_populateTables.sql b/AMTv4-sample-scripts/schema/2_populateTables.sql index 30ee892..c9d8927 100644 --- a/AMTv4-sample-scripts/schema/2_populateTables.sql +++ b/AMTv4-sample-scripts/schema/2_populateTables.sql @@ -12,7 +12,7 @@ USE `sctau`; -- RF2_CONCEPTS_SNAPSHOT TRUNCATE TABLE concepts_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Terminology/xsct2_Concept_Snapshot_AU1000036_20231130.txt' INTO TABLE concepts_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Terminology/xsct2_Concept_Snapshot_AU1000036_20231130.txt' INTO TABLE concepts_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, definitionstatusid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); @@ -20,7 +20,7 @@ set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- RF2_DESCRIPTIONS_SNAPSHOT TRUNCATE TABLE descriptions_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Terminology/xsct2_Description_Snapshot-en-AU_AU1000036_20231130.txt' INTO TABLE descriptions_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Terminology/xsct2_Description_Snapshot-en-AU_AU1000036_20231130.txt' INTO TABLE descriptions_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, conceptid, languagecode, typeid, term, casesignificanceid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); @@ -28,28 +28,28 @@ set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- RF2_RELATIONSHIPS_SNAPSHOT TRUNCATE TABLE relationships_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Terminology/xsct2_Relationship_Snapshot_AU1000036_20231130.txt' INTO TABLE relationships_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Terminology/xsct2_Relationship_Snapshot_AU1000036_20231130.txt' INTO TABLE relationships_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, sourceid, destinationid, relationshipgroup, typeid, characteristictypeid, modifierid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- RF2_RELATIONSHIPS_CONCRETE_VALUES_SNAPSHOT TRUNCATE TABLE relationships_concrete_values_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Terminology/xsct2_RelationshipConcreteValues_Snapshot_AU1000036_20231130.txt' INTO TABLE relationships_concrete_values_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Terminology/xsct2_RelationshipConcreteValues_Snapshot_AU1000036_20231130.txt' INTO TABLE relationships_concrete_values_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, sourceid, value, relationshipgroup, typeid, characteristictypeid, modifierid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- RF2_LANGUAGE_REFSET_SNAPSHOT TRUNCATE TABLE language_refset_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Language/xder2_cRefset_LanguageSnapshot-en-AU_AU1000036_20231130.txt' INTO TABLE language_refset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Language/xder2_cRefset_LanguageSnapshot-en-AU_AU1000036_20231130.txt' INTO TABLE language_refset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid, acceptabilityid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- RF2_REFSET_SNAPSHOT -- AMTv4, only a single import is required. All simple refsets exist in the same file. -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Content/xder2_Refset_SimpleSnapshot_AU1000036_20231130.txt' INTO TABLE refset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Content/xder2_Refset_SimpleSnapshot_AU1000036_20231130.txt' INTO TABLE refset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); @@ -57,11 +57,11 @@ set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- Import historical association reference sets TRUNCATE TABLE crefset_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Content/xder2_cRefset_AssociationSnapshot_AU1000036_20231130.txt' INTO TABLE crefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Content/xder2_cRefset_AssociationSnapshot_AU1000036_20231130.txt' INTO TABLE crefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid, targetComponentid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Content/xder2_cRefset_AttributeValueSnapshot_AU1000036_20231130.txt' INTO TABLE crefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Content/xder2_cRefset_AttributeValueSnapshot_AU1000036_20231130.txt' INTO TABLE crefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid, targetComponentid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); @@ -70,7 +70,7 @@ set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- Import extended association schema refset. Currently only one exists - Route and form extended association TRUNCATE TABLE ccrefset_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Content/xder2_ccRefset_ExtendedAssociationSnapshot_AU1000036_20231130.txt' INTO TABLE ccrefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Content/xder2_ccRefset_ExtendedAssociationSnapshot_AU1000036_20231130.txt' INTO TABLE ccrefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid,value1,value2) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); @@ -79,6 +79,6 @@ set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); -- Import the 11000168105 ARTG Id reference set TRUNCATE TABLE irefset_snapshot; -LOAD DATA LOCAL INFILE 'C://Releases/SnomedCT_AMT_Experiment_20231130/Snapshot/Refset/Map/xder2_iRefset_SimpleMapSnapshot_AU1000036_20231130.txt' INTO TABLE irefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES +LOAD DATA LOCAL INFILE '/Snapshot/Refset/Map/xder2_iRefset_SimpleMapSnapshot_AU1000036_20231130.txt' INTO TABLE irefset_snapshot CHARACTER SET 'utf8' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id, @effectivetime, active, moduleid, refsetid, referencedcomponentid, schemeValue) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); From 17c5950adadc864827d55aae99c81e86016ce497 Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:42:06 +1000 Subject: [PATCH 2/7] Update sample query to only return ADRS Prefered Synoyms. --- AMTv4-sample-scripts/sql/sample_queries.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/AMTv4-sample-scripts/sql/sample_queries.sql b/AMTv4-sample-scripts/sql/sample_queries.sql index 84ccce4..dd6f6e7 100644 --- a/AMTv4-sample-scripts/sql/sample_queries.sql +++ b/AMTv4-sample-scripts/sql/sample_queries.sql @@ -22,14 +22,18 @@ FROM refset_snapshot AS rs, descriptions_snapshot AS d, language_refset_snapshot AS adrs + WHERE c.id=rs.referencedComponentId AND c.id=d.conceptid AND d.id=adrs.referencedComponentId -AND adrs.acceptabilityid=900000000000548007 -- ID of Preferred Term AND rs.refsetid= 11000036103 -- ID of Adverse reaction type refset +AND d.typeid = 900000000000013009 -- Synonym +AND adrs.refsetid = 32570271000036106 -- ADRS +AND adrs.acceptabilityid=900000000000548007 -- ID of Preferred Term AND c.active=1 AND d.active=1 AND rs.active=1 +AND ADRS.active = 1 ORDER BY preferred_term; -- 4. List of Australian reference sets & member count From 88e623d7cecd65993f22bb71273c312dd6a8f113 Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:46:22 +1000 Subject: [PATCH 3/7] Update sample query to use Synonyms only as part of ADRS preferred Term --- AMTv4-sample-scripts/sql/sample_queries.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AMTv4-sample-scripts/sql/sample_queries.sql b/AMTv4-sample-scripts/sql/sample_queries.sql index dd6f6e7..8ec2d09 100644 --- a/AMTv4-sample-scripts/sql/sample_queries.sql +++ b/AMTv4-sample-scripts/sql/sample_queries.sql @@ -46,7 +46,8 @@ FROM (SELECT term,id,conceptid FROM descriptions_snapshot AS ds - WHERE active=1) AS desc_active, + WHERE active=1 AND typeid = 900000000000013009 -- Synonym + ) AS desc_active, (SELECT referencedComponentId From 76223d1c79dca4008bc2407f9a52b2a1c8fb105a Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:57:01 +1000 Subject: [PATCH 4/7] Add ADRS and Synonym filter --- AMTv4-sample-scripts/sql/sample_queries.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AMTv4-sample-scripts/sql/sample_queries.sql b/AMTv4-sample-scripts/sql/sample_queries.sql index 8ec2d09..4d5c527 100644 --- a/AMTv4-sample-scripts/sql/sample_queries.sql +++ b/AMTv4-sample-scripts/sql/sample_queries.sql @@ -80,9 +80,9 @@ JOIN (SELECT sourceId ) AS ffd ON c.id=ffd.sourceid JOIN descriptions_snapshot AS d - ON c.id=d.conceptid + ON c.id=d.conceptid AND d.typeid = 900000000000013009 -- Synonym JOIN language_refset_snapshot AS adrs - ON d.id=adrs.referencedComponentId + ON d.id=adrs.referencedComponentId AND adrs.refsetId = 32570271000036106 -- Australian dialect refset WHERE adrs.acceptabilityid = 900000000000548007 -- ID of Preferred Term AND c.active=1 AND d.active=1 From 2be54cae5bb5ebbc442947fb5bb8cc899a2de571 Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:03:58 +1000 Subject: [PATCH 5/7] Add ADRS + Synonym filter --- AMTv4-sample-scripts/sql/sample_queries.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AMTv4-sample-scripts/sql/sample_queries.sql b/AMTv4-sample-scripts/sql/sample_queries.sql index 4d5c527..d9b3f26 100644 --- a/AMTv4-sample-scripts/sql/sample_queries.sql +++ b/AMTv4-sample-scripts/sql/sample_queries.sql @@ -111,6 +111,8 @@ language_refset_snapshot AS adrs WHERE c.id=ffd.sourceid AND c.id=d.conceptid AND d.id=adrs.referencedComponentId +AND d.typeid = 900000000000013009 -- Synonym +AND adrs.refsetId = 32570271000036106 -- Australian dialect refset AND adrs.acceptabilityid=900000000000548007 -- ID of Preferred Term AND c.active=1 AND d.active=1 From d28329ca03abc74a2c3a46ec90d9ec44244afa75 Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:06:30 +1000 Subject: [PATCH 6/7] Add ADRS Synonym filter --- AMTv4-sample-scripts/sql/sample_queries.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AMTv4-sample-scripts/sql/sample_queries.sql b/AMTv4-sample-scripts/sql/sample_queries.sql index d9b3f26..881d8d9 100644 --- a/AMTv4-sample-scripts/sql/sample_queries.sql +++ b/AMTv4-sample-scripts/sql/sample_queries.sql @@ -137,6 +137,8 @@ WHERE c.id=pd.sourceid AND c.id=d.conceptid AND d.id=adrs.referencedComponentId +AND d.typeid = 900000000000013009 -- Synonym +AND adrs.refsetId = 32570271000036106 -- Australian dialect refset AND adrs.acceptabilityid=900000000000548007 -- ID of Preferred Term AND c.active=1 AND d.active=1 From b27bc988c66f23289f2543a1159d3c8363b7a5fe Mon Sep 17 00:00:00 2001 From: MattCordell <5741701+MattCordell@users.noreply.github.com> Date: Wed, 17 Jul 2024 11:11:59 +1000 Subject: [PATCH 7/7] Refactor conversion of OWL representaiton of concrete domains (strings prefixed with) to numeric --- AMTv4-sample-scripts/schema/2_populateTables.sql | 4 ++++ AMTv4-sample-scripts/schema/6_createAMTv4Objects.sql | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AMTv4-sample-scripts/schema/2_populateTables.sql b/AMTv4-sample-scripts/schema/2_populateTables.sql index c9d8927..f53849b 100644 --- a/AMTv4-sample-scripts/schema/2_populateTables.sql +++ b/AMTv4-sample-scripts/schema/2_populateTables.sql @@ -39,6 +39,10 @@ LOAD DATA LOCAL INFILE '/Snapshot/Terminology/xsct2_RelationshipC (id, @effectivetime, active, moduleid, sourceid, value, relationshipgroup, typeid, characteristictypeid, modifierid) set effectivetime = str_to_date(@effectivetime, '%Y%m%d'); +-- Remove the # character from the value field (Description logic artefact) +UPDATE relationships_concrete_values_snapshot +SET value = REPLACE(value, '#', ''); + -- RF2_LANGUAGE_REFSET_SNAPSHOT TRUNCATE TABLE language_refset_snapshot; diff --git a/AMTv4-sample-scripts/schema/6_createAMTv4Objects.sql b/AMTv4-sample-scripts/schema/6_createAMTv4Objects.sql index a566253..15d0ed1 100644 --- a/AMTv4-sample-scripts/schema/6_createAMTv4Objects.sql +++ b/AMTv4-sample-scripts/schema/6_createAMTv4Objects.sql @@ -147,10 +147,6 @@ FROM v4_MPPhasMPUU MPPhasMPUU AND ConcentrationUnit.typeid = 999000031000168102 -- Has concentration strength unit (attribute) AND ConcentrationUnit.active = 1; --- Clean up the numeric columns by removing the # character -UPDATE v4_ingredient_strength SET ConcentrationValue = REPLACE(ConcentrationValue, '#', ''); -UPDATE v4_ingredient_strength SET TotalQuantity = REPLACE(TotalQuantity, '#', ''); - -- Create Indexes for v4_ingredient_strength table CREATE INDEX v4_ingredient_strength ON v4_ingredient_strength(mppid);