Merge Tutorial Branch Into Main

.devcontainer/webui/devcontainer.json

@@ -0,0 +1,10 @@
+{
+  "image": "docker.io/mlcommons/medperf-codespaces:latest",
+    "features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"moby": false
+		}
+	},
+	"postStartCommand": "bash /workspaces/medperf/tutorials_scripts/codespaces/webui/env_start.sh",
+	"postAttachCommand": "bash /workspaces/medperf/tutorials_scripts/codespaces/webui/run_server_detached.sh"
+}

.github/workflows/auth-ci.yml

@@ -25,13 +25,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.9'
-
-    - name: Setup Chrome
-      run: |
-        sudo apt-get install -y wget
-        wget -O chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_126.0.6478.126-1_amd64.deb
-        sudo apt update && sudo apt install ./chrome.deb -y --allow-downgrades
-        rm chrome.deb
 
     - name: Install dependencies
       working-directory: .

.github/workflows/unittests.yml

@@ -36,7 +36,7 @@ jobs:
         # Ignore E231, as it is raising warnings with auto-generated code.
         flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*",cli/medperf/templates/
     - name: Test with pytest
-      working-directory: ./cli
+      working-directory: ./cli/medperf/tests
       run: |
         pytest
     - name: Set server environment vars

.github/workflows/webui-ci.yml

@@ -0,0 +1,53 @@
+name: WebUI Integration workflow
+
+on: pull_request
+
+jobs:
+  setup:
+    name: webui-integration-test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install dependencies
+      working-directory: .
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e cli/
+        pip install -r cli/test-requirements.txt
+        pip install -r server/requirements.txt
+        pip install -r server/test-requirements.txt
+
+    - name: Set server environment vars
+      working-directory: ./server
+      run: cp .env.local.local-auth .env
+
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
+
+    - name: Run django server in background with generated certs
+      working-directory: ./server
+      run: sh setup-dev-server.sh & sleep 6
+
+    - name: Reset and seed database with tutorial data
+      working-directory: ./server
+      run: sh reset_db.sh && python seed.py --demo tutorial
+
+    - name: Set up tutorial files
+      working-directory: .
+      run: bash tutorials_scripts/setup_webui_tutorial.sh
+
+    - name: Run fastapi server in background
+      working-directory: .
+      run: medperf_webui & sleep 6
+
+    - name: Run webUI tutorial E2E tests with pytest
+      working-directory: ./cli/medperf/web_ui/tests/e2e/
+      run: pytest

cli/cli_tests.sh

@@ -173,15 +173,6 @@ print_eval medperf benchmark submit --name bmk --description bmk --demo-url $DEM
 checkFailed "Benchmark submission failed"
 BMK_UID=$(medperf benchmark ls | grep bmk | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2)
 echo "BMK_UID=$BMK_UID"
-
-# Approve benchmark
-echo $ADMIN
-echo $MOCK_TOKENS_FILE
-ADMIN_TOKEN=$(jq -r --arg ADMIN $ADMIN '.[$ADMIN]' $MOCK_TOKENS_FILE)
-echo $ADMIN_TOKEN
-checkFailed "Retrieving admin token failed"
-print_eval "curl -sk -X PUT $SERVER_URL$VERSION_PREFIX/benchmarks/$BMK_UID/ -d '{\"approval_status\": \"APPROVED\"}' -H 'Content-Type: application/json' -H 'Authorization: Bearer $ADMIN_TOKEN' --fail-with-body"
-checkFailed "Benchmark approval failed"
 ##########################################################
 
 echo "\n"

cli/medperf/config.py

@@ -232,6 +232,13 @@
 ui = "CLI"
 webui = "WEBUI"
 
+# WebUI-related config
+webui_max_log_messages = 200  # Max nb of messages that will appear in LogPanel in WebUI
+webui_max_chunk_age = 2.0  # Max 2 seconds as age of a chunk
+webui_max_chunk_length = 20  # Max 20 events in a chunk
+webui_max_chunk_size = 64 * 1024  # Max 64 Bytes as chunk size
+
+
 default_profile_name = "default"
 testauth_profile_name = "testauth"
 test_profile_name = "local"

cli/medperf/entities/benchmark.py

@@ -22,7 +22,7 @@ class Benchmark(Entity, ApprovableSchema, DeployableSchema):
     what models to run and how to evaluate them.
     """
 
-    description: Optional[str] = Field(None, max_length=20)
+    description: Optional[str] = Field(None, max_length=256)
     docs_url: Optional[HttpUrl]
     demo_dataset_tarball_url: str
     demo_dataset_tarball_hash: Optional[str]

cli/medperf/entities/dataset.py

@@ -22,8 +22,8 @@ class Dataset(Entity, DeployableSchema):
     data preparation output.
     """
 
-    description: Optional[str] = Field(None, max_length=20)
-    location: Optional[str] = Field(None, max_length=20)
+    description: Optional[str] = Field(None, max_length=256)
+    location: Optional[str] = Field(None, max_length=128)
     input_data_hash: str
     generated_uid: str
     data_preparation_mlcube: Union[int, str]

cli/medperf/entities/schemas.py

@@ -11,7 +11,7 @@
 class MedperfSchema(BaseModel):
     for_test: bool = False
     id: Optional[int]
-    name: str = Field(..., max_length=64)
+    name: str = Field(..., max_length=128)
     owner: Optional[int]
     is_valid: bool = True
     created_at: Optional[datetime]
@@ -76,12 +76,6 @@ def empty_str_to_none(cls, v):
             return None
         return v
 
-    @validator("name", pre=True, always=True)
-    def name_max_length(cls, v, *, values, **kwargs):
-        if not values["for_test"] and len(v) > 20:
-            raise ValueError("The name must have no more than 20 characters")
-        return v
-
     class Config:
         allow_population_by_field_name = True
         extra = "allow"

cli/medperf/entities/training_exp.py

@@ -21,7 +21,7 @@ class TrainingExp(Entity, MedperfSchema, ApprovableSchema, DeployableSchema):
     what models to run and how to evaluate them.
     """
 
-    description: Optional[str] = Field(None, max_length=20)
+    description: Optional[str] = Field(None, max_length=256)
     docs_url: Optional[HttpUrl]
     demo_dataset_tarball_url: str
     demo_dataset_tarball_hash: Optional[str]

cli/medperf/ui/web_ui.py

@@ -5,17 +5,18 @@
 import typer
 
 from medperf.ui.cli import CLI
+from medperf.web_ui.schemas import Event, EventsManager
 
 
 class WebUI(CLI):
     def __init__(self):
         super().__init__()
-        self.events: Queue[dict] = Queue()
         self.responses: Queue[dict] = Queue()
         self.is_interactive = False
         self.spinner = yaspin(color="green")
         self.task_id = None
         self.request = None
+        self.events_manager = EventsManager()
 
     def print(self, msg: str = ""):
         """Display a message on the command line
@@ -50,16 +51,14 @@ def _print(self, msg: str = "", type: str = "print"):
         else:
             typer.echo(msg)
 
-        if type == "print" and self.is_interactive:
-            return
-
         self.set_event(
-            {
-                "type": type,
-                "message": msg,
-                "interactive": self.is_interactive,
-                "end": False,
-            }
+            Event(
+                task_id=self.task_id,
+                type=type,
+                message=msg,
+                interactive=self.is_interactive,
+                end=False,
+            )
         )
 
     def start_interactive(self):
@@ -108,12 +107,13 @@ def text(self, msg: str = ""):
         #     self.print(msg)
 
         self.set_event(
-            {
-                "type": "text",
-                "message": msg,
-                "interactive": self.is_interactive,
-                "end": False,
-            }
+            Event(
+                task_id=self.task_id,
+                type="text",
+                message=msg,
+                interactive=self.is_interactive,
+                end=False,
+            )
         )
         self.spinner.text = msg  # TODO
 
@@ -128,12 +128,13 @@ def prompt(self, msg: str) -> str:
         """
         msg = msg.replace(" [Y/n]", "")
         self.set_event(
-            {
-                "type": "prompt",
-                "message": msg,
-                "interactive": self.is_interactive,
-                "end": False,
-            }
+            Event(
+                task_id=self.task_id,
+                type="prompt",
+                message=msg,
+                interactive=self.is_interactive,
+                end=False,
+            )
         )
         add_notification(
             self.request,
@@ -146,7 +147,7 @@ def prompt(self, msg: str) -> str:
         return "n"
 
     def hidden_prompt(self, msg: str) -> str:
-        """Displays a prompt to the user and waits for an aswer. User input is not displayed
+        """Displays a prompt to the user and waits for an answer. User input is not displayed
 
         Args:
             msg (str): message to use for the prompt
@@ -178,12 +179,11 @@ def print_url(self, msg: str):
     def print_code(self, msg: str):
         self._print(msg, "code")
 
-    def set_event(self, event):
-        event["task_id"] = self.task_id
-        self.events.put(event)
+    def set_event(self, event: Event):
+        self.events_manager.process_event(event)
 
-    def get_event(self):
-        return self.events.get()
+    def get_event(self, timeout=None):
+        return self.events_manager.dequeue_event(timeout=timeout)
 
     def set_response(self, event):
         self.responses.put(event)
@@ -192,22 +192,34 @@ def get_response(self):
         return self.responses.get()
 
     def end_task(self, response=None):
-        self.set_event(
-            {
-                "type": "highlight",
-                "message": "",
-                "interactive": self.is_interactive,
-                "end": True,
-                "response": response,
-            }
+        self.events_manager.stop_buffering()
+
+        self.events_manager.enqueue_event(
+            Event(
+                task_id=self.task_id,
+                type="highlight",
+                message="",
+                interactive=self.is_interactive,
+                end=True,
+                response=response,
+            )
         )
         self.unset_task_id()
+        self.unset_request()
+
+    def start_task(self, task_id: str, request):
+        self.set_task_id(task_id)
+        self.set_request(request)
+        self.events_manager.start_buffering()
 
     def set_task_id(self, task_id):
         self.task_id = task_id
 
     def set_request(self, request):
         self.request = request
 
+    def unset_request(self):
+        self.request = None
+
     def unset_task_id(self):
         self.task_id = None

cli/medperf/utils.py

@@ -545,6 +545,18 @@ def sanitize_path(path: str) -> str:
     return resolved_path
 
 
+def print_webui_props(host, port, security_token):
+    print("=" * 40)
+    print()
+    print("Open your browser to:")
+    print(f"\033[1mhttp://{host}:{port}/security_check?token={security_token}\033[0m")
+    print()
+    print("Or use security token to view the web-UI:")
+    print("\033[1m" + security_token + "\033[0m")
+    print()
+    print("=" * 40)
+
+
 def get_webui_properties():
     if not os.path.exists(config.webui_host_props):
         raise CleanExit("Web UI properties file could not be found.")
@@ -553,10 +565,7 @@ def get_webui_properties():
         props = yaml.safe_load(f)
 
     # print security token to CLI (avoid logging to file)
-    print("=" * 40)
-    print()
-    print("Use security token to view the web-UI:")
-    print(props["security_token"])
-    print()
-    print("=" * 40)
-    print(f"URL: http://{props['host']}:{props['port']}")
+    host = props["host"]
+    port = props["port"]
+    security_token = props["security_token"]
+    print_webui_props(host, port, security_token)