diff --git a/bluepyemodel/export_emodel/export_emodel.py b/bluepyemodel/export_emodel/export_emodel.py index 9d8084d..10f6252 100644 --- a/bluepyemodel/export_emodel/export_emodel.py +++ b/bluepyemodel/export_emodel/export_emodel.py @@ -20,11 +20,9 @@ import os import pathlib import shutil -import time import h5py -from bluepyemodel.access_point.forge_access_point import DEFAULT_NEXUS_ENDPOINT from bluepyemodel.evaluation.evaluation import compute_responses from bluepyemodel.evaluation.evaluation import get_evaluator_from_access_point from bluepyemodel.export_emodel.utils import get_hoc_file_path @@ -246,141 +244,3 @@ def export_emodels_hoc( if not cell_model.morphology.morph_modifiers: # Turn [] into None cell_model.morphology.morph_modifiers = None _export_emodel_hoc(cell_model, mo, output_dir=None, new_emodel_name=new_emodel_name) - - -def export_emodels_nexus( - local_access_point, - nexus_organisation, - nexus_project, - nexus_endpoint=DEFAULT_NEXUS_ENDPOINT, - forge_path=None, - forge_ontology_path=None, - access_token=None, - only_validated=False, - only_best=True, - seeds=None, - description=None, - sleep_time=10, - sonata=True, -): - """Transfer e-models from the LocalAccessPoint to a Nexus project - - Args: - local_access_point (LocalAccessPoint): The local access point containing the e-models. - nexus_organisation (str): The Nexus organisation to which the e-models will be transferred. - nexus_project (str): The Nexus project to which the e-models will be transferred. - nexus_endpoint (str, optional): The Nexus endpoint. - Defaults to DEFAULT_NEXUS_ENDPOINT. - forge_path (str, optional): The path to the forge. - forge_ontology_path (str, optional): The path to the forge ontology. - access_token (str, optional): The access token for Nexus. - only_validated (bool, optional): If True, only validated e-models will be transferred. - only_best (bool, optional): If True, only the best e-models will be transferred. - seeds (list, optional): The chosen seeds to export. - description (str, optional): Optional description to add to the resources in Nexus. - sleep_time (int, optional): time to wait between two Nexus requests - (in case of slow indexing). - sonata (bool, optional): Determines the format for registering e-models. - If True (default), uses Sonata hoc format. Otherwise, uses NEURON hoc format. - - Returns: - None - """ - - from bluepyemodel.access_point.nexus import NexusAccessPoint - - emodels = local_access_point.get_emodels() - emodels = select_emodels( - local_access_point.emodel_metadata.emodel, - emodels, - only_validated=only_validated, - only_best=only_best, - seeds=seeds, - ) - if not emodels: - return - - metadata = vars(local_access_point.emodel_metadata) - iteration = metadata.pop("iteration") - metadata.pop("allen_notation") - nexus_access_point = NexusAccessPoint( - **metadata, - iteration_tag=iteration, - project=nexus_project, - organisation=nexus_organisation, - endpoint=nexus_endpoint, - access_token=access_token, - forge_path=forge_path, - forge_ontology_path=forge_ontology_path, - sleep_time=sleep_time, - ) - - pipeline_settings = local_access_point.pipeline_settings - fitness_configuration = local_access_point.get_fitness_calculator_configuration() - model_configuration = local_access_point.get_model_configuration() - targets_configuration = local_access_point.get_targets_configuration() - - # Register the resources - logger.info("Exporting the emodel %s to Nexus...", local_access_point.emodel_metadata.emodel) - logger.info("Registering EModelPipelineSettings...") - nexus_access_point.store_pipeline_settings(pipeline_settings) - - logger.info("Registering ExtractionTargetsConfiguration...") - # Set local filepath to None to avoid discrepancies between local and Nexus paths - for file in targets_configuration.files: - file.filepath = None - nexus_access_point.store_targets_configuration(targets_configuration) - - logger.info("Registering EModelConfiguration...") - # Remove unused local data from the model configuration before uploading to Nexus - model_configuration.morphology.path = None - nexus_access_point.store_model_configuration(model_configuration) - - logger.info("Registering EModelWorkflow...") - filters = {"type": "EModelWorkflow", "eModel": metadata["emodel"], "iteration": iteration} - filters_legacy = { - "type": "EModelWorkflow", - "emodel": metadata["emodel"], - "iteration": iteration, - } - nexus_access_point.access_point.deprecate(filters, filters_legacy) - time.sleep(sleep_time) - emw = nexus_access_point.create_emodel_workflow(state="done") - nexus_access_point.store_or_update_emodel_workflow(emw) - - logger.info("Registering FitnessCalculatorConfiguration...") - time.sleep(sleep_time) - nexus_access_point.store_fitness_calculator_configuration(fitness_configuration) - - for mo in emodels: - time.sleep(sleep_time) - mo.emodel_metadata.allen_notation = nexus_access_point.emodel_metadata.allen_notation - mo.copy_pdf_dependencies_to_new_path(seed=mo.seed) - logger.info("Registering EModel %s...", mo.emodel_metadata.emodel) - nexus_access_point.store_emodel(mo, description=description) - - time.sleep(sleep_time) - if sonata: - logger.info( - "Registering EModelScript (in sonata hoc format with threshold_current and " - "holding_current in node.h5 file) for circuit building using neurodamus..." - ) - nexus_access_point.store_emodels_sonata( - only_best=only_best, - only_validated=only_validated, - seeds=seeds, - description=description, - ) - else: - logger.info("Registering EModelScript (in hoc format to run e-model using NEURON)...") - nexus_access_point.store_emodels_hoc( - only_best=only_best, - only_validated=only_validated, - seeds=seeds, - description=description, - ) - - logger.info( - "Exporting the emodel %s to Nexus done.", - local_access_point.emodel_metadata.emodel, - ) diff --git a/examples/others/local2nexus/README.md b/examples/others/local2nexus/README.md deleted file mode 100644 index 8dcad99..0000000 --- a/examples/others/local2nexus/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Export local e-model to Nexus - -To export an e-model built locally to Nexus, start by copying the ``export_local_to_nexus.py`` script and the YAML files (forge.yml and nsg.yml) into your local e-model folder. This folder should already contain the following files created during the e-model construction process: - -- ``final.json``: which contains the analysis of each optimised model -- ``run/``: folder named after the githash of the commit generated by the run you wish to export. - - ``config/``: folder containing the configuration files - - ``morphologies/``: folder containing the morphology file -- ``figures/``: folder containing the figures generated by the analysis step of the local e-model pipeline -- ``export_emodels_hoc/``: folder containing the hoc file generated by the export step of the local e-model pipeline. - -Please ensure that you run both the analysis and export hoc step of the local e-model pipeline, see [here](https://github.com/BlueBrain/BluePyEModel/tree/main/examples/L5PC#running-the-different-steps). - -Once all the required files are in place, adjust the variables in the script to match the e-model data you wish to export, as well as the Nexus-related settings. Additionally, verify that the electrophysiological data, mechanisms, and morphology are registered in the Nexus project to which you want to export the e-model. -The script is designed exclusively to create the e-model related resources within the specified Nexus project and link the respective data registered on Nexus. - -Once everything is set, run the script to export the e-model to Nexus: -``` -python export_local_to_nexus.py -``` - -If the script runs successfully, the following e-model resources will be created in the specified Nexus project: -- ``EModelPipelineSettings`` (EMPS): the pipeline settings of the e-model. -- ``ExtractionTargetsConfiguration`` (ETC): the extraction target configuration of the e-model from the ``run/{githash}/config/extract_config`` folder, as well as the links to the ephys data. -- ``EModelConfiguration`` (EMC): the configuration of the e-model, which links to the morphology and mechanisms and stores a reformatted version of the parameters file of the e-model from ``run/{githash}/config/params`` folder. -- ``FitnessCalculatorConfiguration`` (FCC): the fitness calculator configuration of the e-model, which stores the features and protocols of the e-model from ``run/{githash}/config/features/`` folder. -- ``EmodelScript`` (ES): the hoc file of the e-model. -- ``EModel`` (EM): all the information related to an optimised e-model. It contains the final parameters of the e-model from final.json, and pdfs of the e-model distribution plots, features scores and e-model response traces. It also links to EModelWorflow. -- ``EModelWorkflow`` (EMW): the resource to which all the above resources are linked also contains the workflow state. - -The graph structure of the e-model resources is shown below: - -``` - EModelWorkflow - | - ├──> EModelPipelineSettings - | - ├──> ExtractionTargetsConfiguration - | | - | ├──> Trace1 - | ├──> ... - | └──> TraceN - | - ├──> EModelConfiguration - | | - | ├──> Mechanism1 - | ├──> ... - | └──> MechanismN - | └──> Morphology - | - ├──> FitnessCalculatorConfiguration - | - ├──> EModel - | - └──> EModelScript -``` - -## Troubleshooting: Delays in Resource Registration with Nexus -During the resource registration process, errors may occur if the resource is not yet fully registered and therefore not ready to be retrieved from Nexus. To address this, the ``sleep_time`` variable in the script can be adjusted ensuring the resource fully registers with Nexus and becomes retrievable. \ No newline at end of file diff --git a/examples/others/local2nexus/export_local_to_nexus.py b/examples/others/local2nexus/export_local_to_nexus.py deleted file mode 100644 index ab51bdd..0000000 --- a/examples/others/local2nexus/export_local_to_nexus.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Upload local models to nexus.""" - -""" -Copyright 2024, EPFL/Blue Brain Project - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import getpass -import os -from bluepyemodel.export_emodel.export_emodel import export_emodels_nexus -from bluepyemodel.access_point.local import LocalAccessPoint -import logging - -logger = logging.getLogger(__name__) - -# Please change the following settings according to your needs and data: -emodel = "YOUR_EMODEL_NAME_HERE" # name of the local emodel you want to upload as it appears in the config/recipes.json file -githash = ( - "YOUR_GITHASH_HERE" # githash of the local emodel (provided during the optimization step). -) -only_validated = False # only upload validated emodels - -only_best = False # only upload best emodel -seeds = [1] # list of seeds that you want to upload, Please leave it empty if only_best=True. - -# should match the data of your LocalAccessPoint emodel, if it was not set, use None -etype = None -mtype = None -ttype = None -species = "rat" # e.g. "mouse" -brain_region = "SSCX" # e.g. "SSCX" - -description = "" - -# Nexus settings -nexus_project = "" # a valid Nexus project name to which the emodel should be uploaded. -nexus_organisation = "" # replace with the organisation name -# Nexus advanced settings (only change if you know what you are doing) -nexus_endpoint = "" # replace with the Nexus endpoint url -forge_path = "./forge.yml" -forge_ontology_path = "./nsg.yml" -sleep_time = 10 # increase the delay in case indexing is slow - - -def main(): - - # Set the logging level - logging.basicConfig(level=logging.INFO) - - print("Please input your access token you got from nexus:") - access_token = getpass.getpass() - - local_access_point = LocalAccessPoint( - emodel=emodel, - etype=etype, - mtype=mtype, - ttype=ttype, - species=species, - brain_region=brain_region, - final_path=os.path.join(".", "final.json"), - iteration_tag=githash, - recipes_path=os.path.join(".", "config", "recipes.json"), - ) - - export_emodels_nexus( - local_access_point, - nexus_organisation=nexus_organisation, - nexus_project=nexus_project, - nexus_endpoint=nexus_endpoint, - forge_path=forge_path, - access_token=access_token, - only_validated=only_validated, - only_best=only_best, - seeds=seeds, - description=description, - forge_ontology_path=forge_ontology_path, - sleep_time=sleep_time, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/others/local2nexus/forge.yml b/examples/others/local2nexus/forge.yml deleted file mode 100644 index 2ba9597..0000000 --- a/examples/others/local2nexus/forge.yml +++ /dev/null @@ -1,73 +0,0 @@ -Model: - name: RdfModel - origin: store - source: BlueBrainNexus - context: - iri: "https://bbp.neuroshapes.org" - bucket: "neurosciencegraph/datamodels" - -Store: - name: BlueBrainNexus - endpoint: https://bbp.epfl.ch/nexus/v1 - model: - name: RdfModel - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - elastic: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" - mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" - default_str_keyword_field: "keyword" - vocabulary: - metadata: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" - local_iri: "https://bluebrainnexus.io/contexts/metadata.json" - namespace: "https://bluebrain.github.io/nexus/vocabulary/" - deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" - project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 5 - versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" - file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson - -Resolvers: - ontology: - - resolver: OntologyResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: terms - bucket: neurosciencegraph/datamodels - - identifier: CellType - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainCellType - - identifier: BrainRegion - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainRegion - - identifier: Species - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: Species - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson - agent: - - resolver: AgentResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: agents - bucket: bbp/agents - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson - -Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} - identifier_bbn_self: https://bbp.epfl.ch/resources/{}/{}/{}/{} # https://bbp.epfl.ch/nexus/v1/resources/{organization}/{project}/{schema}/{id} diff --git a/examples/others/local2nexus/nsg.yml b/examples/others/local2nexus/nsg.yml deleted file mode 100644 index 356ac1e..0000000 --- a/examples/others/local2nexus/nsg.yml +++ /dev/null @@ -1,73 +0,0 @@ -Model: - name: RdfModel - origin: store - source: BlueBrainNexus - context: - iri: "https://neuroshapes.org" - bucket: "neurosciencegraph/datamodels" - -Store: - name: BlueBrainNexus - endpoint: https://bbp.epfl.ch/nexus/v1 - model: - name: RdfModel - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - elastic: - endpoint: "https://bbp.epfl.ch/neurosciencegraph/data/views/aggreg-es/dataset" - mapping: "https://bbp.epfl.ch/neurosciencegraph/data/views/es/dataset" - default_str_keyword_field: "keyword" - vocabulary: - metadata: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" - local_iri: "https://bluebrainnexus.io/contexts/metadata.json" - namespace: "https://bluebrain.github.io/nexus/vocabulary/" - deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" - project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 5 - versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" - file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson - -Resolvers: - ontology: - - resolver: OntologyResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: terms - bucket: neurosciencegraph/datamodels - - identifier: CellType - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainCellType - - identifier: BrainRegion - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: BrainRegion - - identifier: Species - bucket: neurosciencegraph/datamodels - filters: - - path: subClassOf*.id - value: Species - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/term-to-resource-mapping.hjson - agent: - - resolver: AgentResolver - origin: store - source: BlueBrainNexus - targets: - - identifier: agents - bucket: bbp/agents - searchendpoints: - sparql: - endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" - result_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-resolver/agent-to-resource-mapping.hjson - -Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} - identifier_bbn_self: https://bbp.epfl.ch/resources/{}/{}/{}/{} # https://bbp.epfl.ch/nexus/v1/resources/{organization}/{project}/{schema}/{id} diff --git a/pyproject.toml b/pyproject.toml index 226048b..c29f1cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ luigi = [ ] docs = [ "graphviz", - "sphinx", + "sphinx<9", "sphinx-bluebrain-theme", "luigi>=3.0", "luigi-tools>=0.0.12",