refactor: extract worker template and get rid of icij-worker dependency

.github/actions/pylint/action.yml

@@ -14,6 +14,6 @@ runs:
     - name: Run linting test on $ {{ inputs.path }}
       shell: bash
       run: |
-        pylint_rc=$(pwd)/qa/python/pylintrc
+        pylint_rc=$(pwd)/qa/pylintrc
         cd ${{ inputs.path }}
         pylint --fail-under 9.49 --rcfile $pylint_rc --load-plugins=pylint.extensions.bad_builtin -r y -j 0 -f colorized **/*.py

.github/workflows/linting.yml

@@ -4,19 +4,51 @@ on:
   push:
     branches: [ 'main' ]
   pull_request:
+    # TODO: start workflow selectively, rather than running full linting as soon as a file change in one of the monorepos
     paths:
-      - 'datashare_python/**.py'
+      - 'datashare-python/**.py'
+      - 'worker-template/**.py'
+      - 'asr-worker/**.py'
       - '.github/workflows/linting.yml'
 
 # TODO: leverage some caching here
 jobs:
-  linting-worker:
+  datashare-python:
+    runs-on: ubuntu-latest
+    env:
+      RUFF_OUTPUT_FORMAT: github
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check datashare-python
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml datashare-python
+
+  worker-template:
+    runs-on: ubuntu-latest
+    env:
+      RUFF_OUTPUT_FORMAT: github
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check worker-template
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml worker-template
+
+  asr-worker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/pylint
+        name: "Lint ASR worker"
         with:
-          path: "."
+          path: asr-worker
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.ruff.toml

@@ -0,0 +1 @@
+qa/ruff.toml

.testcontainers/temporal/dynamic.yml

@@ -0,0 +1,9 @@
+limit.maxIDLength:
+  - value: 255
+    constraints: { }
+system.namespaceCacheRefreshInterval:
+  - value: 0.1s
+    constraints: { }
+system.forceSearchAttributesCacheRefreshOnRead:
+  - value: true
+    constraints: { }

.testcontainers/temporal/post-init.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+
+NAMESPACE=${DEFAULT_NAMESPACE:-datashare-default}
+TEMPORAL_ADDRESS=${TEMPORAL_ADDRESS:-temporal:7233}
+
+create_temporal_namespace() {
+  echo "Creating namespace '$NAMESPACE'..."
+  temporal operator namespace describe -n "$NAMESPACE" --address "$TEMPORAL_ADDRESS" || temporal operator namespace create -n "$NAMESPACE" --address "$TEMPORAL_ADDRESS"
+  echo "Namespace '$NAMESPACE' created"
+}
+
+create_custom_search_attributes_namespace() {
+  temporal operator search-attribute create --address "$TEMPORAL_ADDRESS" -n "$NAMESPACE" --name="UserId" --type="Keyword"
+  temporal operator search-attribute create --address "$TEMPORAL_ADDRESS" -n "$NAMESPACE" --name="Progress" --type="Double"
+  temporal operator search-attribute create --address "$TEMPORAL_ADDRESS" -n "$NAMESPACE" --name="MaxProgress" --type="Double"
+}
+
+create_temporal_namespace
+create_custom_search_attributes_namespace

.testcontainers/temporal/pre-init.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -eu
+
+# Validate required environment variables
+: "${POSTGRES_SEEDS:?ERROR: POSTGRES_SEEDS environment variable is required}"
+: "${POSTGRES_PWD:?ERROR: POSTGRES_PWD environment variable is required}"
+: "${POSTGRES_USER:?ERROR: POSTGRES_USER environment variable is required}"
+: "${POSTGRES_PORT:?ERROR: POSTGRES_PORT environment variable is required}"
+: "${POSTGRES_DB:?ERROR: POSTGRES_DB environment variable is required}"
+: "${POSTGRES_VISIBILITY_DB:?ERROR: POSTGRES_VISIBILITY_DB environment variable is required}"
+
+setup_postgres() {
+  echo 'Creating temporal base DB'
+
+  # Create and setup temporal database
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_DB}" create
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_DB}" setup-schema -v 0.0
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_DB}" update-schema -d /etc/temporal/schema/postgresql/v12/temporal/versioned
+
+  echo 'Creating temporal visibility DB'
+  # Create and setup temporal database
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_VISIBILITY_DB}" create
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_VISIBILITY_DB}" setup-schema -v 0.0
+  temporal-sql-tool --plugin postgres12 --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -pw "${POSTGRES_PWD}" -p "${POSTGRES_PORT}" --db "${POSTGRES_VISIBILITY_DB}" update-schema -d /etc/temporal/schema/postgresql/v12/visibility/versioned
+
+  echo 'PostgreSQL schema setup complete'
+}
+
+
+setup_postgres

.testcontainers/temporal/temporal.yml

@@ -0,0 +1,42 @@
+log:
+  stdout: true
+  level: { { default .Env.LOG_LEVEL "info" } }
+
+dynamicConfigClient:
+  filepath: '.config/temporalio/dynamic.yml'
+  pollInterval: '10s'
+
+system:
+  forceSearchAttributesCacheRefreshOnRead: true
+
+worker:
+  batcherRPS: 100
+  batcherConcurrency: 10
+
+persistence:
+  defaultStore: postgres-default
+  visibilityStore: postgres-visibility
+  numHistoryShards: 4
+  datastores:
+    postgres-default:
+      sql:
+        pluginName: 'postgres12'
+        databaseName: 'temporal'
+        connectAddr: 'postgres:5432'
+        connectProtocol: 'tcp'
+        user: 'dstest'
+        password: 'test'
+        tls:
+          enabled: false
+          enableHostVerification: false
+    postgres-visibility:
+      sql:
+        pluginName: 'postgres12'
+        databaseName: 'temporal_visibility'
+        connectAddr: 'postgres:5432'
+        connectProtocol: 'tcp'
+        user: 'dstest'
+        password: 'test'
+        tls:
+          enabled: false
+          enableHostVerification: false

datashare-python/datashare_python/__main__.py

@@ -1,7 +1,7 @@
-from datashare_python.cli import cli_app
+from .cli import cli_app
 
 
-def main():
+def main() -> None:
     cli_app()
 
 

datashare-python/datashare_python/cli/__init__.py

@@ -1,19 +1,19 @@
 import importlib.metadata
-from typing import Annotated, Optional
+from typing import Annotated
 
 import typer
 
 import datashare_python
-from datashare_python.cli.tasks import task_app
 from datashare_python.cli.local import local_app
+from datashare_python.cli.tasks import task_app
 from datashare_python.cli.utils import AsyncTyper
 
 cli_app = AsyncTyper(context_settings={"help_option_names": ["-h", "--help"]})
 cli_app.add_typer(task_app)
 cli_app.add_typer(local_app)
 
 
-def version_callback(value: bool):
+def version_callback(value: bool) -> None:  # noqa: FBT001
     if value:
         package_version = importlib.metadata.version(datashare_python.__name__)
         print(package_version)
@@ -22,11 +22,10 @@ def version_callback(value: bool):
 
 @cli_app.callback()
 def main(
-    version: Annotated[  # pylint: disable=unused-argument
-        Optional[bool],
-        typer.Option(  # pylint: disable=unused-argument
-            "--version", callback=version_callback, is_eager=True
-        ),
+    version: Annotated[
+        bool | None,
+        typer.Option("--version", callback=version_callback, is_eager=True),
     ] = None,
-):
-    """Datashare Python CLI"""
+) -> None:
+    """Datashare Python CLI."""
+    pass

datashare-python/datashare_python/cli/local.py

@@ -2,13 +2,11 @@
 from typing import Annotated
 
 import typer
-from datashare_python.cli.utils import AsyncTyper
-from datashare_python.constants import DEFAULT_TEMPORAL_ADDRESS
 
+from datashare_python.cli.utils import AsyncTyper
+from datashare_python.constants import DEFAULT_NAMESPACE, DEFAULT_TEMPORAL_ADDRESS
 from datashare_python.local_client import LocalClient
 
-from datashare_python.constants import DEFAULT_NAMESPACE
-
 _START_WORKERS_HELP = "starts local temporal workers"
 _REGISTER_NAMESPACE_HELP = "register namespace"
 _WORKERS_HELP = "list of worker paths"
@@ -27,7 +25,7 @@ async def start_workers(
     temporal_address: Annotated[
         str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
     ] = DEFAULT_TEMPORAL_ADDRESS,
-):
+) -> None:
     match worker_paths:
         case str():
             worker_paths = json.loads(worker_paths)
@@ -49,13 +47,12 @@ async def register_namespace(
     temporal_address: Annotated[
         str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
     ] = DEFAULT_TEMPORAL_ADDRESS,
-):
+) -> None:
     """Create namespace
 
     :param namespace: namespace
     :param temporal_address: target host
     """
-
     client = LocalClient()
 
     await client.register_namespace(temporal_address, namespace)