fix(sdk): expose datasets client as part of the sdk_client PLA-366 (#358)

Author: bbatha

.gitignore

@@ -106,3 +106,7 @@ ENV/
 .idea
 paperspace-python.zip
 *.env
+
+bin/
+lib64
+share/

gradient/api_sdk/clients/__init__.py

@@ -13,6 +13,7 @@
 from .project_client import ProjectsClient
 from .secret_client import SecretsClient
 from .storage_provider_client import StorageProvidersClient
-from .sdk_client import SdkClient
 from .tensorboards_client import TensorboardClient
 from .workflow_client import WorkflowsClient
+
+from .sdk_client import SdkClient

gradient/api_sdk/clients/sdk_client.py

@@ -1,7 +1,9 @@
 from . import DeploymentsClient, ExperimentsClient, HyperparameterJobsClient, ModelsClient, ProjectsClient, \
-    MachinesClient, NotebooksClient, SecretsClient
+    MachinesClient, NotebooksClient, SecretsClient, DatasetsClient, MachineTypesClient, DatasetVersionsClient, \
+    DatasetTagsClient, ClustersClient, StorageProvidersClient
 from .job_client import JobsClient
 from .workflow_client import WorkflowsClient
+from .tensorboards_client import TensorboardClient
 from .. import logger as sdk_logger
 
 
@@ -11,13 +13,23 @@ def __init__(self, api_key, logger=sdk_logger.MuteLogger()):
         :param str api_key: API key
         :param sdk_logger.Logger logger:
         """
-        self.experiments = ExperimentsClient(api_key=api_key, logger=logger)
+        self.clusters = ClustersClient(api_key=api_key, logger=logger)
+        self.datasets = DatasetsClient(api_key=api_key, logger=logger)
+        self.dataset_tags = DatasetTagsClient(api_key=api_key, logger=logger)
+        self.dataset_versions = DatasetVersionsClient(
+            api_key=api_key, logger=logger)
         self.deployments = DeploymentsClient(api_key=api_key, logger=logger)
-        self.hyperparameters = HyperparameterJobsClient(api_key=api_key, logger=logger)
-        self.models = ModelsClient(api_key=api_key, logger=logger)
+        self.experiments = ExperimentsClient(api_key=api_key, logger=logger)
+        self.hyperparameters = HyperparameterJobsClient(
+            api_key=api_key, logger=logger)
         self.jobs = JobsClient(api_key=api_key, logger=logger)
-        self.projects = ProjectsClient(api_key=api_key, logger=logger)
+        self.machine_types = MachineTypesClient(api_key=api_key, logger=logger)
         self.machines = MachinesClient(api_key=api_key, logger=logger)
+        self.models = ModelsClient(api_key=api_key, logger=logger)
         self.notebooks = NotebooksClient(api_key=api_key, logger=logger)
+        self.projects = ProjectsClient(api_key=api_key, logger=logger)
         self.secrets = SecretsClient(api_key=api_key, logger=logger)
+        self.storage_providers = StorageProvidersClient(
+            api_key=api_key, logger=logger)
+        self.tensorboards = TensorboardClient(api_key=api_key, logger=logger)
         self.workflows = WorkflowsClient(api_key=api_key, logger=logger)

gradient/api_sdk/clients/tensorboards_client.py

@@ -170,7 +170,8 @@ def add_experiments(self, id, added_experiments):
         :raises: ResourceFetchingError: When there is problem with response from API
         """
         repository = self.build_repository(repositories.UpdateTensorboard)
-        tensorboard = repository.update(id=id, added_experiments=added_experiments)
+        tensorboard = repository.update(
+            id=id, added_experiments=added_experiments)
         return tensorboard
 
     def remove_experiments(self, id, removed_experiments):
@@ -197,7 +198,8 @@ def remove_experiments(self, id, removed_experiments):
         :raises: ResourceFetchingError: When there is problem with response from API
         """
         repository = self.build_repository(repositories.UpdateTensorboard)
-        tensorboard = repository.update(id=id, removed_experiments=removed_experiments)
+        tensorboard = repository.update(
+            id=id, removed_experiments=removed_experiments)
         return tensorboard
 
     def delete(self, id):

gradient/commands/experiments.py

@@ -54,7 +54,8 @@ def maybe_add_to_tensorboard(self, tensorboard_id, experiment_id):
 
         tensorboards = self._get_tensorboards()
         if len(tensorboards) == 1:
-            self._add_experiment_to_tensorboard(tensorboards[0].id, experiment_id)
+            self._add_experiment_to_tensorboard(
+                tensorboards[0].id, experiment_id)
         else:
             self._create_tensorboard_with_experiment(experiment_id)
 
@@ -64,15 +65,17 @@ def _add_experiment_to_tensorboard(self, tensorboard_id, experiment_id):
         :param str tensorboard_id:
         :param str experiment_id:
         """
-        command = tensorboards_commands.AddExperimentToTensorboard(api_key=self.api_key)
+        command = tensorboards_commands.AddExperimentToTensorboard(
+            api_key=self.api_key)
         command.execute(tensorboard_id, [experiment_id])
 
     def _get_tensorboards(self):
         """Get tensorboards
 
         :rtype: list[api_sdk.Tensorboard]
         """
-        tensorboard_client = TensorboardClient(api_key=self.api_key, logger=self.logger)
+        tensorboard_client = TensorboardClient(
+            api_key=self.api_key, logger=self.logger)
         tensorboards = tensorboard_client.list()
         return tensorboards
 
@@ -81,7 +84,8 @@ def _create_tensorboard_with_experiment(self, experiment_id):
 
         :param str experiment_id:
         """
-        command = tensorboards_commands.CreateTensorboardCommand(api_key=self.api_key)
+        command = tensorboards_commands.CreateTensorboardCommand(
+            api_key=self.api_key)
         command.execute(experiments=[experiment_id])
 
 
@@ -101,14 +105,18 @@ def execute(self, json_, add_to_tensorboard=False):
         with halo.Halo(text=self.SPINNER_MESSAGE, spinner="dots"):
             experiment_id = self._create(json_)
 
-        self.logger.log(self.CREATE_SUCCESS_MESSAGE_TEMPLATE.format(experiment_id))
-        self.logger.log(self.get_instance_url(experiment_id, json_["project_id"]))
+        self.logger.log(
+            self.CREATE_SUCCESS_MESSAGE_TEMPLATE.format(experiment_id))
+        self.logger.log(self.get_instance_url(
+            experiment_id, json_["project_id"]))
 
-        self._maybe_add_to_tensorboard(add_to_tensorboard, experiment_id, self.api_key)
+        self._maybe_add_to_tensorboard(
+            add_to_tensorboard, experiment_id, self.api_key)
         return experiment_id
 
     def get_instance_url(self, instance_id, project_id):
-        url = concatenate_urls(config.WEB_URL, "{}/projects/{}/experiments/{}".format(self.get_namespace(), project_id, instance_id))
+        url = concatenate_urls(config.WEB_URL, "{}/projects/{}/experiments/{}".format(
+            self.get_namespace(), project_id, instance_id))
         return url
 
     def _handle_workspace(self, instance_dict):
@@ -129,7 +137,8 @@ def _maybe_add_to_tensorboard(self, tensorboard_id, experiment_id, api_key):
         """
         if tensorboard_id is not False:
             tensorboard_handler = TensorboardHandler(api_key)
-            tensorboard_handler.maybe_add_to_tensorboard(tensorboard_id, experiment_id)
+            tensorboard_handler.maybe_add_to_tensorboard(
+                tensorboard_id, experiment_id)
 
     @staticmethod
     def _handle_dataset_data(json_):
@@ -151,12 +160,13 @@ def _handle_dataset_data(json_):
             return
         else:
             datasets_len = max(len(datasets[0]), len(datasets[1]))
-            other_dataset_param_max_len = max(len(elem) for elem in datasets[2:])
+            other_dataset_param_max_len = max(
+                len(elem) for elem in datasets[2:])
             if datasets_len < other_dataset_param_max_len:
                 # there no point in defining n+1 dataset parameters of one type for n datasets
                 raise click.BadParameter(
                     "Too many dataset parameter sets ({}) for {} dataset URIs. Forgot to add one more dataset URI?"
-                        .format(other_dataset_param_max_len, datasets_len))
+                    .format(other_dataset_param_max_len, datasets_len))
 
         datasets = [none_strings_to_none_objects(d) for d in datasets]
 
@@ -194,7 +204,8 @@ def _create(self, json_):
 
 class CreateMpiMultiNodeExperimentCommand(BaseCreateExperimentCommandMixin, BaseExperimentCommand):
     def _create(self, json_):
-        json_.pop("experiment_type_id", None)  # for MPI there is no experiment_type_id parameter in client method
+        # for MPI there is no experiment_type_id parameter in client method
+        json_.pop("experiment_type_id", None)
         handle = self.client.create_mpi_multi_node(**json_)
         return handle
 
@@ -213,7 +224,8 @@ class CreateAndStartMpiMultiNodeExperimentCommand(BaseCreateExperimentCommandMix
     CREATE_SUCCESS_MESSAGE_TEMPLATE = "New experiment created and started with ID: {}"
 
     def _create(self, json_):
-        json_.pop("experiment_type_id", None)  # for MPI there is no experiment_type_id parameter in client method
+        # for MPI there is no experiment_type_id parameter in client method
+        json_.pop("experiment_type_id", None)
         handle = self.client.run_mpi_multi_node(**json_)
         return handle
 
@@ -283,7 +295,8 @@ def _get_table_data(self, experiment):
         if experiment.experiment_type_id == constants.ExperimentType.MPI_MULTI_NODE:
             return self._get_multi_node_mpi_data(experiment)
 
-        raise ValueError("Wrong experiment type: {}".format(experiment.experiment_type_id))
+        raise ValueError("Wrong experiment type: {}".format(
+            experiment.experiment_type_id))
 
     @staticmethod
     def _get_single_node_data(experiment):
@@ -323,13 +336,15 @@ def _get_multi_node_grpc_data(experiment):
             ("Artifact directory", experiment.artifact_directory),
             ("Cluster ID", experiment.cluster_id),
             ("Experiment Env", experiment.experiment_env),
-            ("Experiment Type", constants.ExperimentType.get_type_str(experiment.experiment_type_id)),
+            ("Experiment Type", constants.ExperimentType.get_type_str(
+                experiment.experiment_type_id)),
             ("Model Type", experiment.model_type),
             ("Model Path", experiment.model_path),
             ("Parameter Server Command", experiment.parameter_server_command),
             ("Parameter Server Container", experiment.parameter_server_container),
             ("Parameter Server Count", experiment.parameter_server_count),
-            ("Parameter Server Machine Type", experiment.parameter_server_machine_type),
+            ("Parameter Server Machine Type",
+             experiment.parameter_server_machine_type),
             ("Ports", experiment.ports),
             ("Project ID", experiment.project_id),
             ("Worker Command", experiment.worker_command),
@@ -356,7 +371,8 @@ def _get_multi_node_mpi_data(experiment):
             ("Artifact directory", experiment.artifact_directory),
             ("Cluster ID", experiment.cluster_id),
             ("Experiment Env", experiment.experiment_env),
-            ("Experiment Type", constants.ExperimentType.get_type_str(experiment.experiment_type_id)),
+            ("Experiment Type", constants.ExperimentType.get_type_str(
+                experiment.experiment_type_id)),
             ("Model Type", experiment.model_type),
             ("Model Path", experiment.model_path),
             ("Master Command", experiment.master_command),
@@ -428,6 +444,7 @@ def execute(self, experiment_id, start, end, interval, built_in_metrics, *args,
         formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True)
         self.logger.log(formatted_metrics)
 
+
 class ListExperimentMetricsCommand(BaseExperimentCommand):
     def execute(self, experiment_id, start, end, interval, *args, **kwargs):
         metrics = self.client.list_metrics(
@@ -439,5 +456,6 @@ def execute(self, experiment_id, start, end, interval, *args, **kwargs):
         formatted_metrics = json.dumps(metrics, indent=2, sort_keys=True)
         self.logger.log(formatted_metrics)
 
+
 class StreamExperimentMetricsCommand(StreamMetricsCommand, BaseExperimentCommand):
     pass

pyvenv.cfg

@@ -0,0 +1,3 @@
+home = /home/linuxbrew/.linuxbrew/bin
+include-system-site-packages = false
+version = 3.9.0