Regenerate parsers with schema graph property (#145)

Co-authored-by: Michael R. Crusoe <[email protected]>

Author: GlassOfWhiskey

README.rst

@@ -93,13 +93,13 @@ Regenerate parsers
 To regenerate install the ``schema_salad`` package and run:
 
 ``cwl_utils/parser/cwl_v1_0.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/CommonWorkflowLanguage.yml``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_0"``
 
 ``cwl_utils/parser/cwl_v1_1.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/main/CommonWorkflowLanguage.yml``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/main/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_1"``
 
 ``cwl_utils/parser/cwl_v1_2.py`` was created via
-``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml``
+``schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/1.2.1_proposed/CommonWorkflowLanguage.yml --codegen-parser-info "org.w3id.cwl.v1_2"``
 
 Release
 ~~~~~~~

cwl_utils/parser/cwl_v1_0.py

@@ -3,11 +3,13 @@
 # The code itself is released under the Apache 2.0 license and the help text is
 # subject to the license of the original schema.
 import copy
+import logging
 import os
 import pathlib
 import re
 import tempfile
 import uuid as _uuid__  # pylint: disable=unused-import # noqa: F401
+import xml.sax  # nosec
 from abc import ABC, abstractmethod
 from io import StringIO
 from typing import (
@@ -21,27 +23,32 @@
     Tuple,
     Type,
     Union,
+    cast,
 )
 from urllib.parse import quote, urlparse, urlsplit, urlunsplit
 from urllib.request import pathname2url
 
+from rdflib import Graph
+from rdflib.plugins.parsers.notation3 import BadSyntax
 from ruamel.yaml.comments import CommentedMap
 
 from schema_salad.exceptions import SchemaSaladException, ValidationException
-from schema_salad.fetcher import DefaultFetcher, Fetcher
+from schema_salad.fetcher import DefaultFetcher, Fetcher, MemoryCachingFetcher
 from schema_salad.sourceline import SourceLine, add_lc_filename
 from schema_salad.utils import yaml_no_ts  # requires schema-salad v8.2+
 
 _vocab: Dict[str, str] = {}
 _rvocab: Dict[str, str] = {}
 
+_logger = logging.getLogger("salad")
+
 
 class LoadingOptions:
     def __init__(
         self,
         fetcher: Optional[Fetcher] = None,
         namespaces: Optional[Dict[str, str]] = None,
-        schemas: Optional[Dict[str, str]] = None,
+        schemas: Optional[List[str]] = None,
         fileuri: Optional[str] = None,
         copyfrom: Optional["LoadingOptions"] = None,
         original_doc: Optional[Any] = None,
@@ -77,6 +84,10 @@ def __init__(
         else:
             self.fetcher = fetcher
 
+        self.cache = (
+            self.fetcher.cache if isinstance(self.fetcher, MemoryCachingFetcher) else {}
+        )
+
         self.vocab = _vocab
         self.rvocab = _rvocab
 
@@ -87,6 +98,42 @@ def __init__(
                 self.vocab[k] = v
                 self.rvocab[v] = k
 
+    @property
+    def graph(self) -> Graph:
+        """Generate a merged rdflib.Graph from all entries in self.schemas."""
+        graph = Graph()
+        if not self.schemas:
+            return graph
+        key = str(hash(tuple(self.schemas)))
+        if key in self.cache:
+            return cast(Graph, self.cache[key])
+        for schema in self.schemas:
+            fetchurl = (
+                self.fetcher.urljoin(self.fileuri, schema)
+                if self.fileuri is not None
+                else pathlib.Path(schema).resolve().as_uri()
+            )
+            try:
+                if fetchurl not in self.cache or self.cache[fetchurl] is True:
+                    _logger.debug("Getting external schema %s", fetchurl)
+                    content = self.fetcher.fetch_text(fetchurl)
+                    self.cache[fetchurl] = newGraph = Graph()
+                    for fmt in ["xml", "turtle"]:
+                        try:
+                            newGraph.parse(
+                                data=content, format=fmt, publicID=str(fetchurl)
+                            )
+                            break
+                        except (xml.sax.SAXParseException, TypeError, BadSyntax):
+                            pass
+                graph += self.cache[fetchurl]
+            except Exception as e:
+                _logger.warning(
+                    "Could not load extension schema %s: %s", fetchurl, str(e)
+                )
+        self.cache[key] = graph
+        return graph
+
 
 class Savable(ABC):
     """Mark classes than have a save() and fromDoc() function."""
@@ -101,14 +148,12 @@ def fromDoc(
         docRoot: Optional[str] = None,
     ) -> "Savable":
         """Construct this object from the result of yaml.load()."""
-        pass
 
     @abstractmethod
     def save(
         self, top: bool = False, base_url: str = "", relative_uris: bool = True
     ) -> Dict[str, Any]:
         """Convert this object to a JSON/YAML friendly dictionary."""
-        pass
 
 
 def load_field(val, fieldtype, baseuri, loadingOptions):
@@ -140,7 +185,6 @@ def save(
     base_url: str = "",
     relative_uris: bool = True,
 ) -> save_type:
-
     if isinstance(val, Savable):
         return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
     if isinstance(val, MutableSequence):

cwl_utils/parser/cwl_v1_1.py

@@ -3,11 +3,13 @@
 # The code itself is released under the Apache 2.0 license and the help text is
 # subject to the license of the original schema.
 import copy
+import logging
 import os
 import pathlib
 import re
 import tempfile
 import uuid as _uuid__  # pylint: disable=unused-import # noqa: F401
+import xml.sax  # nosec
 from abc import ABC, abstractmethod
 from io import StringIO
 from typing import (
@@ -21,27 +23,32 @@
     Tuple,
     Type,
     Union,
+    cast,
 )
 from urllib.parse import quote, urlparse, urlsplit, urlunsplit
 from urllib.request import pathname2url
 
+from rdflib import Graph
+from rdflib.plugins.parsers.notation3 import BadSyntax
 from ruamel.yaml.comments import CommentedMap
 
 from schema_salad.exceptions import SchemaSaladException, ValidationException
-from schema_salad.fetcher import DefaultFetcher, Fetcher
+from schema_salad.fetcher import DefaultFetcher, Fetcher, MemoryCachingFetcher
 from schema_salad.sourceline import SourceLine, add_lc_filename
 from schema_salad.utils import yaml_no_ts  # requires schema-salad v8.2+
 
 _vocab: Dict[str, str] = {}
 _rvocab: Dict[str, str] = {}
 
+_logger = logging.getLogger("salad")
+
 
 class LoadingOptions:
     def __init__(
         self,
         fetcher: Optional[Fetcher] = None,
         namespaces: Optional[Dict[str, str]] = None,
-        schemas: Optional[Dict[str, str]] = None,
+        schemas: Optional[List[str]] = None,
         fileuri: Optional[str] = None,
         copyfrom: Optional["LoadingOptions"] = None,
         original_doc: Optional[Any] = None,
@@ -77,6 +84,10 @@ def __init__(
         else:
             self.fetcher = fetcher
 
+        self.cache = (
+            self.fetcher.cache if isinstance(self.fetcher, MemoryCachingFetcher) else {}
+        )
+
         self.vocab = _vocab
         self.rvocab = _rvocab
 
@@ -87,6 +98,42 @@ def __init__(
                 self.vocab[k] = v
                 self.rvocab[v] = k
 
+    @property
+    def graph(self) -> Graph:
+        """Generate a merged rdflib.Graph from all entries in self.schemas."""
+        graph = Graph()
+        if not self.schemas:
+            return graph
+        key = str(hash(tuple(self.schemas)))
+        if key in self.cache:
+            return cast(Graph, self.cache[key])
+        for schema in self.schemas:
+            fetchurl = (
+                self.fetcher.urljoin(self.fileuri, schema)
+                if self.fileuri is not None
+                else pathlib.Path(schema).resolve().as_uri()
+            )
+            try:
+                if fetchurl not in self.cache or self.cache[fetchurl] is True:
+                    _logger.debug("Getting external schema %s", fetchurl)
+                    content = self.fetcher.fetch_text(fetchurl)
+                    self.cache[fetchurl] = newGraph = Graph()
+                    for fmt in ["xml", "turtle"]:
+                        try:
+                            newGraph.parse(
+                                data=content, format=fmt, publicID=str(fetchurl)
+                            )
+                            break
+                        except (xml.sax.SAXParseException, TypeError, BadSyntax):
+                            pass
+                graph += self.cache[fetchurl]
+            except Exception as e:
+                _logger.warning(
+                    "Could not load extension schema %s: %s", fetchurl, str(e)
+                )
+        self.cache[key] = graph
+        return graph
+
 
 class Savable(ABC):
     """Mark classes than have a save() and fromDoc() function."""
@@ -101,14 +148,12 @@ def fromDoc(
         docRoot: Optional[str] = None,
     ) -> "Savable":
         """Construct this object from the result of yaml.load()."""
-        pass
 
     @abstractmethod
     def save(
         self, top: bool = False, base_url: str = "", relative_uris: bool = True
     ) -> Dict[str, Any]:
         """Convert this object to a JSON/YAML friendly dictionary."""
-        pass
 
 
 def load_field(val, fieldtype, baseuri, loadingOptions):
@@ -140,7 +185,6 @@ def save(
     base_url: str = "",
     relative_uris: bool = True,
 ) -> save_type:
-
     if isinstance(val, Savable):
         return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
     if isinstance(val, MutableSequence):