Allow to deploy WT flavor using local data catalog

.env

@@ -1,7 +1,9 @@
+export DATACAT=1
 export docker_group=$(getent group docker | cut -d: -f3)
-export domain=local.wholetale.org
+export domain=local.xarthisius.xyz
+#export domain=wt.xarthisius.xyz
 export registry_user=fido
 export registry_pass=secretpass
-export r2d_version=wholetale/repo2docker_wholetale:latest
+export r2d_version=xarthisius/repo2docker_wholetale:20240926
 export girderfs_image=wholetale/girderfs:latest
 export matlab_file_installation_key=secretkey

.gitignore

@@ -1,6 +1,8 @@
+volumes/
 src/*
 registry/docker
 traefik/access.log
 traefik/acme
+traefik/certs/*.pem
 homes/*
 ps/*

Makefile

@@ -1,8 +1,8 @@
-.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources \
+.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources_wt \
 	rebuild_dashboard watch_dashboard \
-	restart_worker restart_girder globus_handler_src status update_src
+	restart_worker restart_girder globus_handler_src status update_src certs
 
-SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp
+SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp volumes/minio
 TAG = latest
 MEM_LIMIT = 2048
 NODE = node --max_old_space_size=${MEM_LIMIT}
@@ -11,14 +11,37 @@ YARN = /usr/local/bin/yarn
 
 images:
 	docker pull traefik:alpine
-	docker pull mongo:3.2
+	docker pull mongo:4.4
 	docker pull redis:latest
 	docker pull registry:2.6
 	docker pull node:carbon-slim
-	docker pull wholetale/girder:$(TAG)
-	docker pull wholetale/gwvolman:$(TAG)
-	docker pull wholetale/repo2docker_wholetale:$(TAG)
-	docker pull wholetale/ngx-dashboard:$(TAG)
+	docker pull xarthisius/girder:$(TAG)
+	docker pull xarthisius/gwvolman:$(TAG)
+	docker pull xarthisius/repo2docker_wholetale:$(TAG)
+	docker pull xarthisius/ngx-dashboard:$(TAG)
+
+.env:
+	curl -s -o .env https://wt.xarthisius.xyz/wt_local_env
+
+traefik/certs:
+	mkdir -p traefik/certs
+
+traefik/certs/fullchain.pem: traefik/certs
+	curl -s -o traefik/certs/fullchain.pem https://wt.xarthisius.xyz/wt_local_cert
+
+traefik/certs/privkey.pem: traefik/certs
+	curl -s -o traefik/certs/privkey.pem https://wt.xarthisius.xyz/wt_local_key
+
+certs: .env traefik/certs/fullchain.pem traefik/certs/privkey.pem
+
+src/sem_viewer:
+	git clone https://github.com/htmdec/sem_viewer src/sem_viewer
+
+src/table_view:
+	git clone https://github.com/htmdec/table_view src/table_view
+
+src/synced_folders:
+	git clone https://github.com/whole-tale/synced_folders src/synced_folders
 
 src/girderfs:
 	git clone https://github.com/whole-tale/girderfs src/girderfs
@@ -47,14 +70,14 @@ src/globus_handler:
 src/ngx-dashboard:
 	git clone https://github.com/whole-tale/ngx-dashboard src/ngx-dashboard
 
-sources: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning
+sources_wt: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning src/sem_viewer src/table_view src/synced_folders certs
 
 dirs: $(SUBDIRS)
 
 $(SUBDIRS):
 	@sudo mkdir -p $@
 
-services: dirs sources
+services: dirs sources_wt
 
 dev: services
 	. ./.env && docker stack config --compose-file docker-stack.yml | docker stack deploy --compose-file - wt
@@ -65,24 +88,10 @@ dev: services
 	    cid=$$(docker ps --filter=name=wt_girder -q) ; \
 	done; \
 	true
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install plugin plugins/wt_data_manager plugins/wholetale plugins/wt_home_dir plugins/globus_handler plugins/virtual_resources plugins/wt_versioning
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install web --dev --plugins=oauth,gravatar,jobs,worker,wt_data_manager,wholetale,wt_home_dir,globus_handler
-	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -r /gwvolman/requirements.txt -e /gwvolman
-	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -e /girderfs
-	./setup_girder.py
+	. ./.env && ./setup_girder.py
 
 restart_girder:
-	which jq || (echo "Please install jq to execute the 'restart_girder' make target" && exit 1)
-	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -r /gwvolman/requirements.txt -e /gwvolman
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) \
-                curl -XPUT -s 'http://localhost:8080/api/v1/system/restart' \
-                        --header 'Content-Type: application/json' \
-                        --header 'Accept: application/json' \
-                        --header 'Content-Length: 0' \
-                        --header "Girder-Token: $$(docker exec -ti $$(docker ps --filter=name=wt_girder -q) \
-                                curl 'http://localhost:8080/api/v1/user/authentication' \
-                                --basic --user admin:arglebargle123 \
-                                        | jq -r .authToken.token)"
+	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) touch /venv/lib/python3.12/site-packages/requests/__init__.py
 
 rebuild_dashboard:
 	docker run \
@@ -95,6 +104,8 @@ rebuild_dashboard:
 		-w /srv/app node:fermium \
 			-c 'yarn install --network-timeout=360000 && \
 			./node_modules/@angular/cli/bin/ng build --deleteOutputPath=false --progress'
+	sudo touch src/ngx-dashboard/dist/browser/assets/env.js
+	sudo chown 101:101 src/ngx-dashboard/dist/browser/assets/env.js
 
 watch_dashboard:
 	docker run \
@@ -143,6 +154,7 @@ clean:
 	  sudo rm -rf volumes/$$dir ; \
 	done; true
 	-docker volume rm wt_mongo-cfg wt_mongo-data
+	rm -rf traefik/certs || true
 
 status:
 	@-./scripts/git_status.sh

README.md

@@ -91,6 +91,14 @@ Run:
 make dev
 ```
 
+or
+
+```
+DATACAT=1 make dev
+```
+
+for WholeTale flavor operating on local data.
+
 To confirm things are working, all `REPLICAS` should show `1/1`
 ```
 $ docker service ls

dagster.yaml

@@ -0,0 +1,66 @@
+telemetry:
+  enabled: false
+
+scheduler:
+  module: dagster.core.scheduler
+  class: DagsterDaemonScheduler
+
+run_coordinator:
+  module: dagster.core.run_coordinator
+  class: QueuedRunCoordinator
+
+run_launcher:
+  module: dagster_docker
+  class: DockerRunLauncher
+  config:
+    env_vars:
+      - DAGSTER_POSTGRES_USER
+      - DAGSTER_POSTGRES_PASSWORD
+      - DAGSTER_POSTGRES_DB
+    network: wt_dagster
+    container_kwargs:
+      volumes: # Make docker client accessible to any launched containers as well
+        - /var/run/docker.sock:/var/run/docker.sock
+        - /tmp/io_manager_storage:/tmp/io_manager_storage
+
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    postgres_db:
+      hostname: postgresql
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+schedule_storage:
+  module: dagster_postgres.schedule_storage
+  class: PostgresScheduleStorage
+  config:
+    postgres_db:
+      hostname: postgresql
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    postgres_db:
+      hostname: postgresql
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432