diff --git a/README.md b/README.md index 13f37b3..e4cb07b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,95 @@ -# Functional Curation experiment runner for the Web Lab +# FC experiment runner for the Web Lab -Initially this is just the web service + Celery task queue from Web Lab version 1. -It will be refactored and extended to support other backends as we progress. +This repository implements a backend job runner for the WebLab’s [Functional Curation (FC)](https://github.com/ModellingWebLab/weblab-fc) experiments. It functions as an asynchronous execution service that bridges the WebLab frontend with long-running FC model/protocol runs, then reports status and artifacts back to the frontend. + +At a high level, it: + +1. Accepts experiment-related requests from a web-facing CGI endpoint. +1. Queues work in [Celery](https://docs.celeryq.dev/). +1. Downloads model/protocol archives (and optional fitting data/specs). +1. Checks protocol-model compatibility. +1. Runs the simulation/fitting executable. +1. Zips results and posts them back to a callback URL. + +## Installation + +### Install dependencies + +Install [Functional Curation (FC)](https://github.com/ModellingWebLab/weblab-fc). + +Install additional dependencies +```bash +python3 -m pip install -r requirements/base.txt +``` + +Update configuration settings in `fcws/config.json` + +### Install service + +Install a Celery service. For example, save this to `/etc/systemd/system/celery.service`: + +```txt +[Unit] +Description=Celery Service +After=network.target + +[Service] +Type=forking +User=celery +Group=celery +EnvironmentFile=-/etc/default/celery + +WorkingDirectory=/path/to/fc-runner +RuntimeDirectory=celery + +ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} -A ${CELERY_APP} \ + --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} \ + --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' + +ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' + +ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} -A ${CELERY_APP} \ + --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} \ + --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' + +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +and save this to `/etc/default/celery` + +```txt +# Configuration for Functional Curation celery daemon. + +# Number of worker nodes +CELERYD_NODES=2 + +# Path to celery executable +CELERY_BIN=/path/to/bin/celery" + +# Celery application +CELERY_APP="fcws.tasks:app" + +# Extra command-line arguments to the workers +CELERYD_OPTS="--concurrency=2 -l info -Ofair --statedb=/var/lib/celery/%n.state -Q:1 admin -Q default,admin" + +CELERYD_LOG_FILE="/var/log/celery/%n%I.log" +CELERYD_PID_FILE="/run/%n.pid" +CELERYD_LOG_LEVEL="INFO" + +# Notes: +# - The default queue is set to 'default' in celeryconfig.py, and admin users +# use 'admin' (see fcws/__init__.py). +# - %n will be replaced with the first part of the nodename. +# - %I will be replaced with the current child process index +# and is important when using the prefork pool to avoid race conditions. +``` + +```sh +sudo systemctl daemon-reload +sudo systemctl start celery.service +``` diff --git a/fcws/__init__.py b/fcws/__init__.py index f4ee5dd..c19a755 100644 --- a/fcws/__init__.py +++ b/fcws/__init__.py @@ -1,4 +1,6 @@ -# Back-end web service routines for the functional curation website +""" +Functional Curation web service (FCWS) back-end routines for the WebLab. +""" import json import os diff --git a/requirements/Makefile b/requirements/Makefile index 7b5f6a1..175bb8c 100644 --- a/requirements/Makefile +++ b/requirements/Makefile @@ -6,7 +6,7 @@ outputs := $(objects:.in=.txt) all: $(outputs) %.txt: %.in - pip-compile -v -U --output-file $@ $< + pip-compile -v --strip-extras -U --output-file $@ $< test.txt: base.txt diff --git a/requirements/base.in b/requirements/base.in index 74d1293..ab5bcb2 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,4 @@ # We use auto-retry with exponential backoff -celery>=4.2 +celery>=5.3 requests diff --git a/requirements/base.txt b/requirements/base.txt index fafe8a3..e9b56e5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,38 +1,59 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --output-file=base.txt base.in +# pip-compile --output-file=base.txt --strip-extras base.in # -amqp==2.5.1 +amqp==5.3.1 # via kombu -billiard==3.6.1.0 +billiard==4.2.4 # via celery -celery==4.3.0 +celery==5.6.2 # via -r base.in -certifi==2019.9.11 +certifi==2026.2.25 # via requests -chardet==3.0.4 +charset-normalizer==3.4.6 # via requests -idna==2.8 +click==8.3.1 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1.2 + # via celery +click-repl==0.3.0 + # via celery +exceptiongroup==1.3.1 + # via celery +idna==3.11 # via requests -importlib-metadata==0.23 - # via kombu -kombu==4.6.5 +kombu==5.6.2 # via celery -more-itertools==5.0.0 - # via zipp -pytz==2019.3 +packaging==26.0 + # via kombu +prompt-toolkit==3.0.52 + # via click-repl +python-dateutil==2.9.0.post0 # via celery -requests==2.22.0 +requests==2.32.5 # via -r base.in -six==1.12.0 - # via more-itertools -urllib3==1.25.8 +six==1.17.0 + # via python-dateutil +typing-extensions==4.15.0 + # via exceptiongroup +tzdata==2025.3 + # via kombu +tzlocal==5.3.1 + # via celery +urllib3==2.6.3 # via requests -vine==1.3.0 +vine==5.1.0 # via # amqp # celery -zipp==0.6.0 - # via importlib-metadata + # kombu +wcwidth==0.6.0 + # via prompt-toolkit