From 03bb021b108b609f8cfe87a9835ddd9bfdbbec68 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 10:03:50 +0200 Subject: [PATCH 01/11] add QMRA models for the admin page --- .github/workflows/ci.yaml | 5 + .gitignore | 1 + infra/helm/qmra/dev.values.yaml | 4 + infra/helm/qmra/prod.values.yaml | 4 + infra/helm/qmra/templates/deployment.yaml | 14 + infra/helm/qmra/templates/volumes.yaml | 30 ++ .../collect_static_default_entities.py | 3 +- qmra/management/commands/export_default.py | 48 +++ qmra/management/commands/seed_default_db.py | 26 ++ qmra/risk_assessment/admin.py | 59 ++- qmra/risk_assessment/dbrouter.py | 20 ++ qmra/risk_assessment/forms.py | 13 +- ...erence_qmrasource_qmraexposure_and_more.py | 84 +++++ qmra/risk_assessment/models.py | 268 +------------- qmra/risk_assessment/qmra_models.py | 339 ++++++++++++++++++ qmra/risk_assessment/risk.py | 5 +- .../risk_assessment/tests/test_assess_risk.py | 16 +- qmra/risk_assessment/tests/test_export.py | 5 +- .../tests/test_risk_assessment_form.py | 6 +- .../tests/test_static_entities.py | 36 +- qmra/settings.py | 5 + qmra/static/data/default-inflows.json | 2 +- qmra/templates/layout.html | 5 + qmra/urls.py | 2 +- qmra/user/models.py | 3 - 25 files changed, 691 insertions(+), 312 deletions(-) create mode 100644 qmra/management/commands/export_default.py create mode 100644 qmra/management/commands/seed_default_db.py create mode 100644 qmra/risk_assessment/dbrouter.py create mode 100644 qmra/risk_assessment/migrations/0006_qmrapathogen_qmrareference_qmrasource_qmraexposure_and_more.py create mode 100644 qmra/risk_assessment/qmra_models.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc598d1..4c291b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,9 @@ on: push: branches: - main + pull_request: + branches: + - main jobs: test: @@ -25,6 +28,7 @@ jobs: environment: dev secrets: inherit build-prod: + if: github.ref_name == 'main' needs: test name: build-prod uses: ./.github/workflows/build.yaml @@ -32,6 +36,7 @@ jobs: environment: prod secrets: inherit deploy-prod: + if: github.ref_name == 'main' needs: - build-prod - deploy-dev diff --git a/.gitignore b/.gitignore index 6d6a646..1478d93 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ venv/ .idea .static qmra.db +default_qmra_data.db qmra-prod.db dump* prod-migrations/ diff --git a/infra/helm/qmra/dev.values.yaml b/infra/helm/qmra/dev.values.yaml index a0ee399..0a97e88 100644 --- a/infra/helm/qmra/dev.values.yaml +++ b/infra/helm/qmra/dev.values.yaml @@ -17,6 +17,10 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db +qmra_default: + mount_path: /var/lib/qmra/default_qmra_data.db + hostpath: /var/lib/qmra/default_qmra_data.db + static: mount_path: /var/cache/qmra/static hostpath: /var/cache/qmra/static diff --git a/infra/helm/qmra/prod.values.yaml b/infra/helm/qmra/prod.values.yaml index 5fbfdc4..ed3d45f 100644 --- a/infra/helm/qmra/prod.values.yaml +++ b/infra/helm/qmra/prod.values.yaml @@ -17,6 +17,10 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db +qmra_default: + mount_path: /var/lib/qmra/default_qmra_data.db + hostpath: /var/lib/qmra/default_qmra_data.db + static: mount_path: /var/cache/qmra/static hostpath: /var/cache/qmra/static diff --git a/infra/helm/qmra/templates/deployment.yaml b/infra/helm/qmra/templates/deployment.yaml index 1e3ef03..6f90230 100644 --- a/infra/helm/qmra/templates/deployment.yaml +++ b/infra/helm/qmra/templates/deployment.yaml @@ -37,6 +37,15 @@ spec: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} command: [ python, manage.py, migrate ] + - name: migrate_qmra_default + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + envFrom: + - configMapRef: + name: { { .Values.configmap_name } } + volumeMounts: + - name: qmra_default + mountPath: { { .Values.qmra_default.mount_path } } + command: [ python, manage.py, migrate, --database, qmra ] containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -68,12 +77,17 @@ spec: volumeMounts: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} + - name: qmra_default + mountPath: {{ .Values.qmra_default.mount_path }} - name: static mountPath: {{ .Values.static.mount_path }} volumes: - name: sqlite persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-sqlite-file-pvc + - name: qmra_default + persistentVolumeClaim: + claimName: { { include "app.fullname" . } }-qmra_default-file-pvc - name: static persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-static-files-pvc diff --git a/infra/helm/qmra/templates/volumes.yaml b/infra/helm/qmra/templates/volumes.yaml index 0094e2c..cf7578e 100644 --- a/infra/helm/qmra/templates/volumes.yaml +++ b/infra/helm/qmra/templates/volumes.yaml @@ -31,6 +31,22 @@ spec: type: FileOrCreate --- apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ include "app.fullname" . }}-qmra_default-file-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: microk8s-hostpath + hostPath: + path: {{ .Values.qmra_default.hostpath }} + type: FileOrCreate +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ include "app.fullname" . }}-static-files-pvc @@ -53,6 +69,20 @@ spec: - ReadWriteMany volumeMode: Filesystem volumeName: {{ include "app.fullname" . }}-sqlite-file-pv + resources: + requests: + storage: 1Gi + storageClassName: microk8s-hostpath +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "app.fullname" . }}-qmra_default-file-pvc +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + volumeName: {{ include "app.fullname" . }}-qmra_default-file-pv resources: requests: storage: 1Gi diff --git a/qmra/management/commands/collect_static_default_entities.py b/qmra/management/commands/collect_static_default_entities.py index 0ade9ae..74d1af4 100644 --- a/qmra/management/commands/collect_static_default_entities.py +++ b/qmra/management/commands/collect_static_default_entities.py @@ -30,7 +30,8 @@ def get_default_inflows(): inflows = pd.merge(inflows, sources, left_on="source_id", right_on="id", how="left").rename(columns={"name": "source_name"}) inflows = pd.merge(inflows, pathogens, left_on="pathogen_id", right_on="id", how="left").rename(columns={"name": "pathogen_name"}) inflows = inflows[inflows.pathogen_id.isin((3, 32, 34))] - return inflows.loc[:, ["source_name", "pathogen_name", "min", "max", "ReferenceID"]] + inflows["id"] = list(range(len(inflows.index))) + return inflows.loc[:, ["id", "source_name", "pathogen_name", "min", "max", "ReferenceID"]] def get_default_treatments(): diff --git a/qmra/management/commands/export_default.py b/qmra/management/commands/export_default.py new file mode 100644 index 0000000..94889ab --- /dev/null +++ b/qmra/management/commands/export_default.py @@ -0,0 +1,48 @@ +import json +from django.core.management.base import BaseCommand +from qmra.risk_assessment.qmra_models import QMRAReference, QMRAReferences, QMRASource, \ + QMRASources, QMRAPathogen, QMRAPathogens, QMRAInflow, QMRAInflows, QMRATreatment, \ + QMRATreatments, QMRAExposure, QMRAExposures + + +def save_as_json(data, destination: str): + with open(destination, "w") as f: + json.dump(data, f) + + +class Command(BaseCommand): + help = "export the default data of qmra to json files for serving them as statics" + + def add_arguments(self, parser): + parser.add_argument('--format', type=str, help="'json' (default) or 'csv' ", default="json") + + def handle(self, *args, **options): + save_as_json( + {src.name: src.to_dict() for src in QMRASource.objects.all()}, + QMRASources.source + ) + save_as_json( + {pathogen.name: model_to_dict(pathogen) for pathogen in QMRAPathogen.objects.all()}, + QMRAPathogens.source + ) + save_as_json( + {src.name: [inflow.to_dict() for inflow in QMRAInflow.objects.filter(source__name=src.name).all()] + for src in QMRASource.objects.all()}, + QMRAInflows.source + ) + save_as_json( + {t.name: t.to_dict() for t in QMRATreatment.objects.all()}, + QMRATreatments.source + ) + save_as_json( + {e.name: e.to_dict() for e in QMRAExposure.objects.all()}, + QMRAExposures.source + ) + save_as_json( + {str(ref.pk): ref.to_dict() for ref in QMRAReference.objects.all()}, + QMRAReferences.source + ) + + +if __name__ == '__main__': + Command().handle() diff --git a/qmra/management/commands/seed_default_db.py b/qmra/management/commands/seed_default_db.py new file mode 100644 index 0000000..2e41741 --- /dev/null +++ b/qmra/management/commands/seed_default_db.py @@ -0,0 +1,26 @@ +from django.core.management.base import BaseCommand +from qmra.risk_assessment.qmra_models import QMRAReferences, QMRASources, QMRAPathogens, QMRAInflows, \ + QMRATreatments, QMRAExposures + + +class Command(BaseCommand): + help = "Create the default static data of qmra" + + def handle(self, *args, **options): + for _, ref in QMRAReferences.data.items(): + ref.save() + for _, pat in QMRAPathogens.data.items(): + pat.save() + for _, source in QMRASources.data.items(): + source.save() + for _, inflows in QMRAInflows.data.items(): + for inflow in inflows: + inflow.save() + for _, treatment in QMRATreatments.data.items(): + treatment.save() + for _, exposure in QMRAExposures.data.items(): + exposure.save() + + +if __name__ == '__main__': + Command().handle() diff --git a/qmra/risk_assessment/admin.py b/qmra/risk_assessment/admin.py index 56402c8..806acc7 100644 --- a/qmra/risk_assessment/admin.py +++ b/qmra/risk_assessment/admin.py @@ -1,8 +1,57 @@ from django.contrib import admin +from qmra.risk_assessment.qmra_models import QMRASource, QMRAPathogen, QMRAInflow, \ + QMRATreatment, QMRAExposure -# Register your models here. -# @admin.register(Health) -# class HealthAdmin(admin.ModelAdmin): -# pass +""" +- Separate QMRAModels from UserModels (and db) +- models.py must be able to load without an existing db +- db can be bootstrapped with csvs through seed_qmra_db +- served qmra_db can be downloaded as csv from the admin page +- run export_default and collectstatic every time an entity is saved + from django.core.management import call_command + call_command('my_command', 'foo', bar='baz') +""" -# admin.site.register(RiskAssessment) + +class DefaultInflowModelInline(admin.TabularInline): + model = QMRAInflow + fields = ["pathogen", "min", "max", "reference"] + + +@admin.register(QMRASource) +class DefaultSourceModelAdmin(admin.ModelAdmin): + list_display = ["name", "description"] + inlines = [DefaultInflowModelInline] + + +@admin.register(QMRAExposure) +class DefaultExposureModelAdmin(admin.ModelAdmin): + list_display = ["name", "events_per_year", "volume_per_event"] + # inlines = [ReferenceInline] + + +@admin.register(QMRAPathogen) +class DefaultPathogenModelAdmin(admin.ModelAdmin): + list_display = ["name", "group"] + + +@admin.register(QMRATreatment) +class DefaultTreatmentModelAdmin(admin.ModelAdmin): + list_display = [ + "name", "group", + "bacteria_min", + "bacteria_max", + "viruses_min", + "viruses_max", + "protozoa_min", + "protozoa_max", + ] + fields = [ + ("name", "group"), + ("bacteria_min", "bacteria_max"), + "bacteria_reference", + ("viruses_min", "viruses_max"), + "viruses_reference", + ("protozoa_min", "protozoa_max"), + "protozoa_reference" + ] diff --git a/qmra/risk_assessment/dbrouter.py b/qmra/risk_assessment/dbrouter.py new file mode 100644 index 0000000..ad17999 --- /dev/null +++ b/qmra/risk_assessment/dbrouter.py @@ -0,0 +1,20 @@ +class DBRouter(object): + """Default* entities are managed by admins, everything else is for the users""" + + def db_for_read(self, model, **hints): + if "QMRA" in model.__name__: + return 'qmra' + return "default" + + def db_for_write(self, model, **hints): + if "QMRA" in model.__name__: + return 'qmra' + return "default" + + def allow_migrate(self, db, app_label, model_name=None, **hints): + """ + Make sure the qmra data (and only this data!) is in its db + """ + if model_name is not None and "qmra" in model_name: + return db == "qmra" + return db == "default" diff --git a/qmra/risk_assessment/forms.py b/qmra/risk_assessment/forms.py index f8c1739..d11ec95 100644 --- a/qmra/risk_assessment/forms.py +++ b/qmra/risk_assessment/forms.py @@ -5,8 +5,9 @@ from crispy_forms.helper import FormHelper from crispy_forms.layout import Layout, Field, Row, Column, HTML -from qmra.risk_assessment.models import Inflow, DefaultTreatments, Treatment, \ - RiskAssessment, DefaultExposures, DefaultSources +from qmra.risk_assessment.models import Inflow, Treatment, \ + RiskAssessment +from qmra.risk_assessment.qmra_models import QMRASources, QMRATreatments, QMRAExposures from qmra.user.models import User @@ -31,8 +32,8 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["source_name"].label = "Select a source water type to add pathogen concentrations" - self.fields["exposure_name"].choices = DefaultExposures.choices() - self.fields["source_name"].choices = DefaultSources.choices() + self.fields["exposure_name"].choices = QMRAExposures.choices() + self.fields["source_name"].choices = QMRASources.choices() self.fields['events_per_year'].widget.attrs['min'] = 0 self.fields['volume_per_event'].widget.attrs['min'] = 0 self.fields['volume_per_event'].label = "Volume per event in liters" @@ -207,7 +208,7 @@ def clean(self): class AddTreatmentForm(forms.Form): - select_treatment = forms.ChoiceField(choices=DefaultTreatments.choices(), widget=forms.Select()) + select_treatment = forms.ChoiceField(choices=QMRATreatments.choices(), widget=forms.Select()) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -234,7 +235,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.form_tag = False - self.form.base_fields["name"].choices = DefaultTreatments.choices() + self.form.base_fields["name"].choices = QMRATreatments.choices() if not kwargs.get("queryset", False): self.queryset = Treatment.objects.none() diff --git a/qmra/risk_assessment/migrations/0006_qmrapathogen_qmrareference_qmrasource_qmraexposure_and_more.py b/qmra/risk_assessment/migrations/0006_qmrapathogen_qmrareference_qmrasource_qmraexposure_and_more.py new file mode 100644 index 0000000..53e0403 --- /dev/null +++ b/qmra/risk_assessment/migrations/0006_qmrapathogen_qmrareference_qmrasource_qmraexposure_and_more.py @@ -0,0 +1,84 @@ +# Generated by Django 5.0.6 on 2025-09-17 06:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('risk_assessment', '0005_treatment_train_index'), + ] + + operations = [ + migrations.CreateModel( + name='QMRAPathogen', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('group', models.CharField(choices=[('Bacteria', 'Bacteria'), ('Viruses', 'Viruses'), ('Protozoa', 'Protozoa')], max_length=256)), + ('name', models.CharField(max_length=256)), + ('best_fit_model', models.CharField(choices=[('exponential', 'exponential'), ('beta-Poisson', 'beta-Poisson')], max_length=256)), + ('k', models.FloatField(blank=True, null=True)), + ('alpha', models.FloatField(blank=True, null=True)), + ('n50', models.FloatField(blank=True, null=True)), + ('infection_to_illness', models.FloatField(blank=True, default=True, null=True)), + ('dalys_per_case', models.FloatField(blank=True, default=True, null=True)), + ], + ), + migrations.CreateModel( + name='QMRAReference', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256)), + ('link', models.URLField(max_length=512)), + ], + ), + migrations.CreateModel( + name='QMRASource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256)), + ('description', models.CharField(max_length=512)), + ], + ), + migrations.CreateModel( + name='QMRAExposure', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256)), + ('description', models.CharField(max_length=256)), + ('events_per_year', models.IntegerField()), + ('volume_per_event', models.FloatField()), + ('reference', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='risk_assessment.qmrareference')), + ], + ), + migrations.CreateModel( + name='QMRAInflow', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('min', models.FloatField()), + ('max', models.FloatField()), + ('pathogen', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='risk_assessment.qmrapathogen')), + ('reference', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='risk_assessment.qmrareference')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='risk_assessment.qmrasource')), + ], + ), + migrations.CreateModel( + name='QMRATreatment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256)), + ('group', models.CharField(max_length=256)), + ('description', models.CharField(max_length=512)), + ('bacteria_min', models.FloatField(blank=True, null=True)), + ('bacteria_max', models.FloatField(blank=True, null=True)), + ('viruses_min', models.FloatField(blank=True, null=True)), + ('viruses_max', models.FloatField(blank=True, null=True)), + ('protozoa_min', models.FloatField(blank=True, null=True)), + ('protozoa_max', models.FloatField(blank=True, null=True)), + ('bacteria_reference', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='bacteria_lrv', to='risk_assessment.qmrareference')), + ('protozoa_reference', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='protozoa_lrv', to='risk_assessment.qmrareference')), + ('viruses_reference', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='viruses_lrv', to='risk_assessment.qmrareference')), + ], + ), + ] diff --git a/qmra/risk_assessment/models.py b/qmra/risk_assessment/models.py index e0ec463..1522351 100644 --- a/qmra/risk_assessment/models.py +++ b/qmra/risk_assessment/models.py @@ -1,137 +1,19 @@ -import enum -import json import uuid -import numpy as np from django.db import models from django.db.models import QuerySet +from qmra.risk_assessment.qmra_models import QMRAPathogens, QMRATreatment from qmra.user.models import User -from itertools import groupby -from typing import Optional, Any -import abc -import dataclasses as dtc -from django.utils.functional import classproperty -class ExponentialDistribution: - def __init__(self, k): - self.k = k - - def pdf(self, x): - return 1 - np.exp(-self.k * x) - - -class BetaPoissonDistribution: - def __init__(self, alpha, n50): - self.alpha = alpha - self.n50 = n50 - - def pdf(self, x): - return 1 - (1 + x * (2 ** (1 / self.alpha) - 1) / self.n50) ** -self.alpha - - -class StaticEntity(metaclass=abc.ABCMeta): - _raw_data: Optional[dict[str, dict[str, Any]]] = None - - @property - @abc.abstractmethod - def source(self) -> str: - pass - - @property - @abc.abstractmethod - def model(self) -> dtc.dataclass: - pass - - @property - @abc.abstractmethod - def primary_key(self) -> str: - pass - - @classproperty - def raw_data(cls) -> dict[str, dict[str, Any]]: - if cls._raw_data is None: - with open(cls.source, "r") as f: - cls._raw_data = json.load(f) - return cls._raw_data - - @classproperty - def data(cls) -> dict[str, model]: - return {k: cls.model.from_dict(r) for k, r in cls.raw_data.items()} - - @classmethod - @abc.abstractmethod - def choices(cls): - pass - - @classmethod - def get(cls, pk: str): - return cls.data[pk] - - -class PathogenGroup(models.TextChoices): - Bacteria = "Bacteria" - Viruses = "Viruses" - Protozoa = "Protozoa" - - -class ModelDistributionType(enum.Enum): - exponential = "exponential" - beta_poisson = "beta-Poisson" - - -@dtc.dataclass -class DefaultPathogenModel: - group: PathogenGroup - name: str - # fields from "doseResponse.csv" - best_fit_model: ModelDistributionType - k: Optional[float] - alpha: Optional[float] - n50: Optional[float] - # fields from "health.csv" - infection_to_illness: Optional[float] = None - dalys_per_case: Optional[float] = None - - @classmethod - def from_dict(cls, data) -> "DefaultPathogenModel": - return DefaultPathogenModel( - group=PathogenGroup(data["group"]), - name=data["name"], - best_fit_model=ModelDistributionType(data["best_fit_model"]), - k=data["k"], - alpha=data["alpha"], - n50=data["n50"], - infection_to_illness=data["infection_to_illness"], - dalys_per_case=data["dalys_per_case"], - ) - - def get_distribution(self): - if self.best_fit_model == ModelDistributionType.exponential: - return ExponentialDistribution(self.k) - elif self.best_fit_model == ModelDistributionType.beta_poisson: - return BetaPoissonDistribution(self.alpha, self.n50) - - -class DefaultPathogens(StaticEntity): - source = "qmra/static/data/default-pathogens.json" - model = DefaultPathogenModel - primary_key = "name" - - @classmethod - def choices(cls): - grouped = {grp.value: list(v) for grp, v in groupby(cls.data.values(), key=lambda x: x.group)} - return [ - ("", "---------"), - *[(grp, [(x.name, x.name) for x in v]) for grp, v in grouped.items()], - ] +# @dtc.dataclass class Inflow(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) risk_assessment = models.ForeignKey("RiskAssessment", related_name="inflows", on_delete=models.CASCADE) - pathogen = models.CharField(choices=DefaultPathogens.choices(), + pathogen = models.CharField(choices=QMRAPathogens.choices(), blank=False, null=False, max_length=256) # reference = models.ForeignKey( # Reference, blank=True, null=True, default=None, @@ -142,110 +24,6 @@ class Inflow(models.Model): # notes = models.CharField(max_length=200, default="unknown") -@dtc.dataclass -class DefaultInflowModel: - pathogen_name: str - source_name: str - min: float - max: float - - @classmethod - def from_dict(cls, data: dict): - return DefaultInflowModel( - pathogen_name=data["pathogen_name"], - source_name=data["source_name"], - min=data["min"], - max=data["max"] - ) - - -class DefaultInflows(StaticEntity): - source = "qmra/static/data/default-inflows.json" - model = DefaultInflowModel - primary_key = None - - @classproperty - def data(cls): - return {k: [DefaultInflowModel.from_dict(d) for d in data] - for k, data in cls.raw_data.items()} - - @classmethod - def choices(cls): - return [] - - -@dtc.dataclass -class DefaultSourceModel: - id: int - name: str - description: str - inflows: list[DefaultInflowModel] - - @classmethod - def from_dict(cls, data: dict): - return DefaultSourceModel( - id=data["id"], - name=data["name"], - description=data["description"], - inflows=DefaultInflows.get(data["name"]) - ) - - -class DefaultSources(StaticEntity): - source = "qmra/static/data/default-sources.json" - model = DefaultSourceModel - primary_key = "name" - - @classmethod - def choices(cls): - grouped = {grp: list(v) for grp, v in - groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} - return [ - ("", "---------"), - *[(k, [(x.name, x.name) for x in v]) for k, v in grouped.items()], - ("other", "other") - ] - - -@dtc.dataclass -class DefaultTreatmentModel: - name: str - group: str # TextChoices? - description: str - bacteria_min: Optional[float] - bacteria_max: Optional[float] - viruses_min: Optional[float] - viruses_max: Optional[float] - protozoa_min: Optional[float] - protozoa_max: Optional[float] - - @classmethod - def from_dict(cls, data): - return DefaultTreatmentModel( - name=data['name'], - group=data['group'], - description=data['description'], - bacteria_min=data['bacteria_min'], - bacteria_max=data['bacteria_max'], - viruses_min=data['viruses_min'], - viruses_max=data['viruses_max'], - protozoa_min=data['protozoa_min'], - protozoa_max=data['protozoa_max'], - ) - - -class DefaultTreatments(StaticEntity): - source = "qmra/static/data/default-treatments.json" - model = DefaultTreatmentModel - primary_key = "name" - - @classmethod - def choices(cls): - return [ - *[(x.name, x.name) for x in sorted(cls.data.values(), key=lambda x: x.name)], - ] - - class Treatment(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) risk_assessment = models.ForeignKey("RiskAssessment", related_name="treatments", on_delete=models.CASCADE) @@ -259,7 +37,7 @@ class Treatment(models.Model): protozoa_max = models.FloatField(blank=True, null=True) @classmethod - def from_default(cls, default: DefaultTreatmentModel, risk_assessment): + def from_default(cls, default: QMRATreatment, risk_assessment): return Treatment.objects.create( risk_assessment=risk_assessment, name=default.name, @@ -272,47 +50,13 @@ def from_default(cls, default: DefaultTreatmentModel, risk_assessment): ) -@dtc.dataclass -class DefaultExposureModel: - name: str - description: str - events_per_year: int - volume_per_event: float - - @classmethod - def from_dict(cls, data): - return DefaultExposureModel( - name=data["name"], - description=data["description"], - events_per_year=data["events_per_year"], - volume_per_event=data["volume_per_event"], - ) - - -class DefaultExposures(StaticEntity): - source = "qmra/static/data/default-exposures.json" - model = DefaultExposureModel - primary_key = "name" - - @classmethod - def choices(cls): - grouped = {grp: list(v) for grp, v in - groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} - return [ - ("", "---------"), - *[(k, [(x.name, x.name) for x in sorted(v, key=lambda x: x.name)]) for k, v in grouped.items()], - ("other", "other") - ] - - class RiskAssessment(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="risk_assessments") created_at = models.DateTimeField(auto_now_add=True, null=True) name = models.CharField(max_length=64, default="", blank=True) description = models.TextField(max_length=500, default="", blank=True) - source_name = models.CharField(blank=True, max_length=256 - ) + source_name = models.CharField(blank=True, max_length=256) inflows: QuerySet[Inflow] treatments: QuerySet[Treatment] exposure_name = models.CharField(blank=True, max_length=256) @@ -348,7 +92,7 @@ def __str__(self): class RiskAssessmentResult(models.Model): risk_assessment = models.ForeignKey(RiskAssessment, on_delete=models.CASCADE, related_name="results") - pathogen = models.CharField(choices=DefaultPathogens.choices(), max_length=256) + pathogen = models.CharField(choices=QMRAPathogens.choices(), max_length=256) infection_risk = models.CharField(choices=[("min", "min"), ("max", "max"), ("none", "none")], max_length=4) dalys_risk = models.CharField(choices=[("min", "min"), ("max", "max"), ("none", "none")], max_length=4) infection_minimum_lrv_min = models.FloatField() diff --git a/qmra/risk_assessment/qmra_models.py b/qmra/risk_assessment/qmra_models.py new file mode 100644 index 0000000..3160b6a --- /dev/null +++ b/qmra/risk_assessment/qmra_models.py @@ -0,0 +1,339 @@ +import abc +import dataclasses as dtc +import enum +import json +from itertools import groupby +from typing import Optional, Any + +import numpy as np +from django.db import models +from django.forms import model_to_dict +from django.utils.functional import classproperty + + +class ExponentialDistribution: + def __init__(self, k): + self.k = k + + def pdf(self, x): + return 1 - np.exp(-self.k * x) + + +class BetaPoissonDistribution: + def __init__(self, alpha, n50): + self.alpha = alpha + self.n50 = n50 + + def pdf(self, x): + return 1 - (1 + x * (2 ** (1 / self.alpha) - 1) / self.n50) ** -self.alpha + + +class StaticEntity(metaclass=abc.ABCMeta): + _raw_data: Optional[dict[str, dict[str, Any]]] = None + + @property + @abc.abstractmethod + def source(self) -> str: + pass + + @property + @abc.abstractmethod + def model(self) -> dtc.dataclass: + pass + + @property + @abc.abstractmethod + def primary_key(self) -> str: + pass + + @classproperty + def raw_data(cls) -> dict[str, dict[str, Any]]: + if cls._raw_data is None: + with open(cls.source, "r") as f: + cls._raw_data = json.load(f) + return cls._raw_data + + @classproperty + def data(cls) -> dict[str, model]: + return {k: cls.model.from_dict(r) for k, r in cls.raw_data.items()} + + @classmethod + @abc.abstractmethod + def choices(cls): + pass + + @classmethod + def get(cls, pk: str): + return cls.data[pk] + + +class QMRAReference(models.Model): + name = models.CharField(blank=False, null=False, max_length=256) + link = models.URLField(blank=False, null=False, max_length=512) + + @classmethod + def from_dict(cls, data: dict): + return QMRAReference( + pk=data["ReferenceID"], + name=data["ReferenceName"], + link=data["ReferenceLink"] + ) + + def to_dict(self): + return dict( + ReferenceID=self.pk, + ReferenceName=self.name, + ReferenceLink=self.link + ) + + def __str__(self): + return self.name + + +class QMRAReferences(StaticEntity): + source = "qmra/static/data/default-references.json" + model = QMRAReference + primary_key = "name" + + +class PathogenGroup(models.TextChoices): + Bacteria = "Bacteria" + Viruses = "Viruses" + Protozoa = "Protozoa" + + +class ModelDistributionType(enum.Enum): + exponential = "exponential" + beta_poisson = "beta-Poisson" + + +class QMRASource(models.Model): + name = models.CharField(blank=False, null=False, max_length=256) + description = models.CharField(blank=False, null=False, max_length=512) + + @classmethod + def from_dict(cls, data) -> "QMRASource": + return QMRASource( + pk=data["id"], name=data["name"], description=data['description'] + ) + + def to_dict(self) -> dict: + data = model_to_dict(self) + data["id"] = self.pk + return data + + +class QMRASources(StaticEntity): + source = "qmra/static/data/default-sources.json" + model = QMRASource + primary_key = "name" + + @classmethod + def choices(cls): + grouped = {grp: list(v) for grp, v in + groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} + return [ + ("", "---------"), + *[(k, [(x.name, x.name) for x in v]) for k, v in grouped.items()], + ("other", "other") + ] + + +class QMRAPathogen(models.Model): + group = models.CharField(choices=PathogenGroup.choices, + blank=False, null=False, max_length=256) + name = models.CharField(blank=False, null=False, max_length=256) + # fields from "doseResponse.csv" + best_fit_model = models.CharField(choices=[(m.value, m.value) for m in ModelDistributionType], + blank=False, null=False, max_length=256) + k = models.FloatField(blank=True, null=True) + alpha = models.FloatField(blank=True, null=True) + n50 = models.FloatField(blank=True, null=True) + # fields from "health.csv" + infection_to_illness = models.FloatField(blank=True, null=True, default=True) + dalys_per_case = models.FloatField(blank=True, null=True, default=True) + + @classmethod + def from_dict(cls, data) -> "QMRAPathogen": + return QMRAPathogen( + pk=data["id"], + group=data["group"], + name=data["name"], + best_fit_model=data["best_fit_model"], + k=data["k"], + alpha=data["alpha"], + n50=data["n50"], + infection_to_illness=data["infection_to_illness"], + dalys_per_case=data["dalys_per_case"], + ) + + def to_dict(self) -> dict: + return model_to_dict(self) + + def get_distribution(self): + if self.best_fit_model == ModelDistributionType.exponential.value: + return ExponentialDistribution(self.k) + elif self.best_fit_model == ModelDistributionType.beta_poisson.value: + return BetaPoissonDistribution(self.alpha, self.n50) + else: + raise TypeError(f"Unknown ModelDistributionType: {self.best_fit_model}") + + def __str__(self): + return self.name + + +class QMRAPathogens(StaticEntity): + source = "qmra/static/data/default-pathogens.json" + model = QMRAPathogen + primary_key = "name" + + @classmethod + def choices(cls): + grouped = {grp: list(v) for grp, v in groupby(cls.data.values(), key=lambda x: x.group)} + return [ + ("", "---------"), + *[(grp, [(x.name, x.name) for x in v]) for grp, v in grouped.items()], + ] + + +class QMRAInflow(models.Model): + # pathogen_name: str + # source_name: str + source = models.ForeignKey(QMRASource, on_delete=models.CASCADE) + pathogen = models.ForeignKey(QMRAPathogen, on_delete=models.CASCADE) + min: float = models.FloatField() + max: float = models.FloatField() + reference = models.ForeignKey(QMRAReference, blank=True, null=True, on_delete=models.CASCADE) + + @classmethod + def from_dict(cls, data: dict): + return QMRAInflow( + # pathogen_name=data["pathogen_name"], + # source_name=data["source_name"], + pk=data["id"], + source=QMRASource.objects.get(name=data["source_name"]), + pathogen=QMRAPathogen.objects.get(name=data["pathogen_name"]), + min=data["min"], + max=data["max"] + ) + + def to_dict(self): + data = model_to_dict(self, exclude={"source", "pathogen"}) + data["source_name"] = self.source.name + data["pathogen_name"] = self.pathogen.name + data["id"] = self.pk + return data + + +class QMRAInflows(StaticEntity): + source = "qmra/static/data/default-inflows.json" + model = QMRAInflow + primary_key = None + + @classproperty + def data(cls): + return {k: [QMRAInflow.from_dict(d) for d in data] + for k, data in cls.raw_data.items()} + + @classmethod + def choices(cls): + return [] + + +class QMRATreatment(models.Model): + name: str = models.CharField(max_length=256) + group: str = models.CharField(max_length=256) + description: str = models.CharField(max_length=512) + bacteria_min: Optional[float] = models.FloatField(blank=True, null=True) + bacteria_max: Optional[float] = models.FloatField(blank=True, null=True) + bacteria_reference = models.ForeignKey(QMRAReference, blank=True, null=True, on_delete=models.CASCADE, + related_name="bacteria_lrv") + viruses_min: Optional[float] = models.FloatField(blank=True, null=True) + viruses_max: Optional[float] = models.FloatField(blank=True, null=True) + viruses_reference = models.ForeignKey(QMRAReference, blank=True, null=True, on_delete=models.CASCADE, + related_name="viruses_lrv") + protozoa_min: Optional[float] = models.FloatField(blank=True, null=True) + protozoa_max: Optional[float] = models.FloatField(blank=True, null=True) + protozoa_reference = models.ForeignKey(QMRAReference, blank=True, null=True, on_delete=models.CASCADE, + related_name="protozoa_lrv") + + @classmethod + def from_dict(cls, data): + return QMRATreatment( + pk=data["id"], + name=data['name'], + group=data['group'], + description=data['description'], + bacteria_min=data['bacteria_min'], + bacteria_max=data['bacteria_max'], + bacteria_reference_id=int(data["bacteria_reference"]) \ + if data["bacteria_reference"] is not None else None, + viruses_min=data['viruses_min'], + viruses_max=data['viruses_max'], + viruses_reference_id=int(data["viruses_reference"]) \ + if data["viruses_reference"] is not None else None, + protozoa_min=data['protozoa_min'], + protozoa_max=data['protozoa_max'], + protozoa_reference_id=int(data["protozoa_reference"]) \ + if data["protozoa_reference"] is not None else None, + ) + + def to_dict(self): + data = model_to_dict(self) + data["bacteria_reference"] = str(self.bacteria_reference.pk) if self.bacteria_reference is not None else None + data["viruses_reference"] = str(self.viruses_reference.pk) if self.viruses_reference is not None else None + data["protozoa_reference"] = str(self.protozoa_reference.pk) if self.protozoa_reference is not None else None + return data + + +class QMRATreatments(StaticEntity): + source = "qmra/static/data/default-treatments.json" + model = QMRATreatment + primary_key = "name" + + @classmethod + def choices(cls): + return [ + *[(x.name, x.name) for x in sorted(cls.data.values(), key=lambda x: x.name)], + ] + + +class QMRAExposure(models.Model): + name: str = models.CharField(max_length=256) + description: str = models.CharField(max_length=256) + events_per_year: int = models.IntegerField() + volume_per_event: float = models.FloatField() + reference = models.ForeignKey(QMRAReference, blank=True, null=True, on_delete=models.CASCADE) + + @classmethod + def from_dict(cls, data): + return QMRAExposure( + pk=data["id"], + name=data["name"], + description=data["description"], + events_per_year=data["events_per_year"], + volume_per_event=data["volume_per_event"], + reference_id=int(data["ReferenceID"]) if data["ReferenceID"] is not None else None + ) + + def to_dict(self): + data = model_to_dict(self) + data["id"] = self.pk + data["ReferenceId"] = str(self.reference.pk) + return data + + +class QMRAExposures(StaticEntity): + source = "qmra/static/data/default-exposures.json" + model = QMRAExposure + primary_key = "name" + + @classmethod + def choices(cls): + grouped = {grp: list(v) for grp, v in + groupby(sorted(cls.data.values(), key=lambda x: x.name), key=lambda x: x.name.split(",")[0])} + return [ + ("", "---------"), + *[(k, [(x.name, x.name) for x in sorted(v, key=lambda x: x.name)]) for k, v in grouped.items()], + ("other", "other") + ] diff --git a/qmra/risk_assessment/risk.py b/qmra/risk_assessment/risk.py index a365b0b..eaf888b 100644 --- a/qmra/risk_assessment/risk.py +++ b/qmra/risk_assessment/risk.py @@ -2,7 +2,8 @@ import numpy as np -from qmra.risk_assessment.models import RiskAssessment, RiskAssessmentResult, Treatment, PathogenGroup, DefaultPathogens +from qmra.risk_assessment.models import RiskAssessment, RiskAssessmentResult, Treatment +from qmra.risk_assessment.qmra_models import PathogenGroup, QMRAPathogens def get_annual_risk( @@ -54,7 +55,7 @@ def assess_risk(risk_assessment: RiskAssessment, inflows, treatments, save=True) for inflow in inflows: # unpack params - pathogen = DefaultPathogens.get(inflow.pathogen) + pathogen = QMRAPathogens.get(inflow.pathogen) group = pathogen.group dist = pathogen.get_distribution() diff --git a/qmra/risk_assessment/tests/test_assess_risk.py b/qmra/risk_assessment/tests/test_assess_risk.py index 2de8b47..a9e2dea 100644 --- a/qmra/risk_assessment/tests/test_assess_risk.py +++ b/qmra/risk_assessment/tests/test_assess_risk.py @@ -4,8 +4,8 @@ from django.test import TestCase from assertpy import assert_that -from qmra.risk_assessment.models import RiskAssessment, Inflow, Treatment, DefaultTreatments, RiskAssessmentResult, \ - DefaultPathogens, DefaultInflows +from qmra.risk_assessment.models import RiskAssessment, Inflow, Treatment, RiskAssessmentResult +from qmra.risk_assessment.qmra_models import QMRAPathogens, QMRAInflows, QMRATreatments from qmra.risk_assessment.risk import assess_risk from qmra.user.models import User @@ -39,7 +39,7 @@ def test_with_standard_pathogens_and_all_treatments(self): ] given_treatments = [ Treatment.from_default(t, given_ra) - for _, t in DefaultTreatments.data.items() + for _, t in QMRATreatments.data.items() ] given_ra.inflows.set(given_inflows, bulk=False) given_ra.treatments.set(given_treatments, bulk=False) @@ -66,11 +66,11 @@ def test_with_all_pathogens(self): risk_assessment=given_ra, pathogen=p, min=0.1, max=0.2 - ) for p, _ in DefaultPathogens.data.items() + ) for p, _ in QMRAPathogens.data.items() ] given_treatments = [ - Treatment.from_default(DefaultTreatments.get("Conventional clarification"), given_ra), - Treatment.from_default(DefaultTreatments.get("Slow sand filtration"), given_ra), + Treatment.from_default(QMRATreatments.get("Conventional clarification"), given_ra), + Treatment.from_default(QMRATreatments.get("Slow sand filtration"), given_ra), ] given_ra.inflows.set(given_inflows, bulk=False) given_ra.treatments.set(given_treatments, bulk=False) @@ -108,10 +108,10 @@ def test_regression_test(self): risk_assessment=given_ra, pathogen=inflow.pathogen_name, min=inflow.min, max=inflow.max - ) for inflow in DefaultInflows.get("groundwater") + ) for inflow in QMRAInflows.get("groundwater") ] given_treatments = [ - Treatment.from_default(DefaultTreatments.get("Primary treatment"), given_ra) + Treatment.from_default(QMRATreatments.get("Primary treatment"), given_ra) ] given_ra.inflows.set(given_inflows, bulk=False) given_ra.treatments.set(given_treatments, bulk=False) diff --git a/qmra/risk_assessment/tests/test_export.py b/qmra/risk_assessment/tests/test_export.py index 0a9eebd..773a0cc 100644 --- a/qmra/risk_assessment/tests/test_export.py +++ b/qmra/risk_assessment/tests/test_export.py @@ -5,7 +5,8 @@ import pandas as pd from qmra.risk_assessment import exports -from qmra.risk_assessment.models import RiskAssessment, Inflow, Treatment, DefaultTreatments +from qmra.risk_assessment.models import RiskAssessment, Inflow, Treatment +from qmra.risk_assessment.qmra_models import QMRATreatments from qmra.risk_assessment.risk import assess_risk from qmra.user.models import User @@ -40,7 +41,7 @@ def test_that(self): ] given_treatments = [ Treatment.from_default(t, given_ra) - for t in list(DefaultTreatments.data.values())[:3] + for t in list(QMRATreatments.data.values())[:3] ] given_ra.inflows.set(given_inflows, bulk=False) given_ra.treatments.set(given_treatments, bulk=False) diff --git a/qmra/risk_assessment/tests/test_risk_assessment_form.py b/qmra/risk_assessment/tests/test_risk_assessment_form.py index 2c72153..e28527d 100644 --- a/qmra/risk_assessment/tests/test_risk_assessment_form.py +++ b/qmra/risk_assessment/tests/test_risk_assessment_form.py @@ -2,7 +2,7 @@ from assertpy import assert_that from qmra.risk_assessment.forms import RiskAssessmentForm, InflowForm, InflowFormSet, TreatmentForm, TreatmentFormSet -from qmra.risk_assessment.models import DefaultTreatments +from qmra.risk_assessment.qmra_models import QMRATreatments class TestRiskAssessmentForm(TestCase): @@ -91,7 +91,7 @@ def test_that_negative_are_allowed(self): ) given_form = TreatmentForm(data=data) # ugly hack to work around dynamic choices... - given_form.fields["name"].choices = DefaultTreatments.choices() + given_form.fields["name"].choices = QMRATreatments.choices() given_form.full_clean() print(given_form.errors) assert_that(len(given_form.errors)).is_equal_to(0) @@ -112,7 +112,7 @@ def test_that_min_needs_to_be_less_than_max(self): data = {**default_data, mn: 2, mx: 1} given_form = TreatmentForm(data=data) # ugly hack to work around dynamic choices... - given_form.fields["name"].choices = DefaultTreatments.choices() + given_form.fields["name"].choices = QMRATreatments.choices() given_form.full_clean() assert_that(len(given_form.errors)).is_equal_to(1) assert_that(given_form.errors).contains_key(mn) diff --git a/qmra/risk_assessment/tests/test_static_entities.py b/qmra/risk_assessment/tests/test_static_entities.py index 9946385..7db1cc2 100644 --- a/qmra/risk_assessment/tests/test_static_entities.py +++ b/qmra/risk_assessment/tests/test_static_entities.py @@ -1,41 +1,41 @@ from unittest import TestCase from assertpy import assert_that -from qmra.risk_assessment.models import DefaultPathogens, DefaultPathogenModel, DefaultSources, DefaultSourceModel, \ - DefaultTreatmentModel, DefaultTreatments, PathogenGroup, DefaultExposures, DefaultExposureModel +from qmra.risk_assessment.qmra_models import PathogenGroup, QMRASource, QMRASources, QMRAPathogen, \ + QMRAPathogens, QMRATreatment, QMRATreatments, QMRAExposure, QMRAExposures class TestDefaultPathogens(TestCase): expected_length = 3 def test_properties(self): - under_test = DefaultPathogens + under_test = QMRAPathogens assert_that(under_test.raw_data).is_instance_of(dict) assert_that(len(under_test.raw_data)).is_equal_to(self.expected_length) assert_that(under_test.data).is_instance_of(dict) - assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(DefaultPathogenModel) + assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(QMRAPathogen) assert_that(len(under_test.data)).is_equal_to(self.expected_length) def test_get(self): - under_test = DefaultPathogens + under_test = QMRAPathogens rotavirus = under_test.get("Rotavirus") - assert_that(rotavirus).is_instance_of(DefaultPathogenModel) + assert_that(rotavirus).is_instance_of(QMRAPathogen) assert_that(rotavirus.name).is_equal_to("Rotavirus") assert_that(rotavirus.group).is_equal_to(PathogenGroup.Viruses) jejuni = under_test.get("Campylobacter jejuni") - assert_that(jejuni).is_instance_of(DefaultPathogenModel) + assert_that(jejuni).is_instance_of(QMRAPathogen) assert_that(jejuni.name).is_equal_to("Campylobacter jejuni") assert_that(jejuni.group).is_equal_to(PathogenGroup.Bacteria) parvum = under_test.get("Cryptosporidium parvum") - assert_that(parvum).is_instance_of(DefaultPathogenModel) + assert_that(parvum).is_instance_of(QMRAPathogen) assert_that(parvum.name).is_equal_to("Cryptosporidium parvum") assert_that(parvum.group).is_equal_to(PathogenGroup.Protozoa) def test_choices(self): - under_test = DefaultPathogens + under_test = QMRAPathogens choices = under_test.choices() # print(choices) @@ -50,17 +50,17 @@ class TestDefaultSources(TestCase): expected_length = 8 def test_properties(self): - under_test = DefaultSources + under_test = QMRASources assert_that(under_test.raw_data).is_instance_of(dict) assert_that(len(under_test.raw_data)).is_equal_to(self.expected_length) assert_that(under_test.data).is_instance_of(dict) - assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(DefaultSourceModel) + assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(QMRASource) assert_that(len(under_test.data)).is_equal_to(self.expected_length) def test_choices(self): - under_test = DefaultSources + under_test = QMRASources choices = under_test.choices() # print(choices) @@ -75,17 +75,17 @@ class TestDefaultTreatments(TestCase): expected_length = 22 def test_properties(self): - under_test = DefaultTreatments + under_test = QMRATreatments assert_that(under_test.raw_data).is_instance_of(dict) assert_that(len(under_test.raw_data)).is_equal_to(self.expected_length) assert_that(under_test.data).is_instance_of(dict) - assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(DefaultTreatmentModel) + assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(QMRATreatment) assert_that(len(under_test.data)).is_equal_to(self.expected_length) def test_choices(self): - under_test = DefaultTreatments + under_test = QMRATreatments choices = under_test.choices() # print(choices) @@ -100,17 +100,17 @@ class TestDefaultExposures(TestCase): expected_length = 8 def test_properties(self): - under_test = DefaultExposures + under_test = QMRAExposures assert_that(under_test.raw_data).is_instance_of(dict) assert_that(len(under_test.raw_data)).is_equal_to(self.expected_length) assert_that(under_test.data).is_instance_of(dict) - assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(DefaultExposureModel) + assert_that(under_test.data[list(under_test.data.keys())[0]]).is_instance_of(QMRAExposure) assert_that(len(under_test.data)).is_equal_to(self.expected_length) def test_choices(self): - under_test = DefaultExposures + under_test = QMRAExposures choices = under_test.choices() # print(choices) diff --git a/qmra/settings.py b/qmra/settings.py index c3b5327..372b2fe 100644 --- a/qmra/settings.py +++ b/qmra/settings.py @@ -97,8 +97,13 @@ 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.getenv("SQLITE_PATH", BASE_DIR / 'qmra.db'), + }, + 'qmra': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.getenv("SQLITE_PATH", BASE_DIR / 'default_qmra_data.db'), } } +DATABASE_ROUTERS = ('qmra.risk_assessment.dbrouter.DBRouter',) AUTH_USER_MODEL = "user.User" AUTH_PASSWORD_VALIDATORS = [ diff --git a/qmra/static/data/default-inflows.json b/qmra/static/data/default-inflows.json index e396c39..95883b0 100644 --- a/qmra/static/data/default-inflows.json +++ b/qmra/static/data/default-inflows.json @@ -1 +1 @@ -{"groundwater": [{"source_name": "groundwater", "pathogen_name": "Rotavirus", "min": 0.0, "max": 2.0, "ReferenceID": "43"}, {"source_name": "groundwater", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 10.0, "ReferenceID": "43"}, {"source_name": "groundwater", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1.0, "ReferenceID": "43"}], "rainwater, rooftop harvesting": [{"source_name": "rainwater, rooftop harvesting", "pathogen_name": "Rotavirus", "min": 0.0, "max": 0.01, "ReferenceID": "44"}, {"source_name": "rainwater, rooftop harvesting", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 24.0, "ReferenceID": "44"}, {"source_name": "rainwater, rooftop harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 0.19, "ReferenceID": "44"}], "rainwater, stormwater harvesting": [{"source_name": "rainwater, stormwater harvesting", "pathogen_name": "Rotavirus", "min": 9.74510658007135, "max": 64.7460691472062, "ReferenceID": "45"}, {"source_name": "rainwater, stormwater harvesting", "pathogen_name": "Campylobacter jejuni", "min": 13.8694279635122, "max": 287.039358509118, "ReferenceID": "45"}, {"source_name": "rainwater, stormwater harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 4.52008261942372e-05, "max": 0.0880751977503127, "ReferenceID": "45"}], "sewage, raw": [{"source_name": "sewage, raw", "pathogen_name": "Rotavirus", "min": 50.0, "max": 5000.0, "ReferenceID": "39"}, {"source_name": "sewage, raw", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 1000000.0, "ReferenceID": "39"}, {"source_name": "sewage, raw", "pathogen_name": "Cryptosporidium parvum", "min": 1.0, "max": 10000.0, "ReferenceID": "39"}], "sewage, treated": [{"source_name": "sewage, treated", "pathogen_name": "Rotavirus", "min": 0.1, "max": 1000.0, "ReferenceID": "42"}, {"source_name": "sewage, treated", "pathogen_name": "Campylobacter jejuni", "min": 0.001, "max": 1000.0, "ReferenceID": "42"}, {"source_name": "sewage, treated", "pathogen_name": "Cryptosporidium parvum", "min": 0.01, "max": 10000.0, "ReferenceID": "42"}], "surface water, contaminated": [{"source_name": "surface water, contaminated", "pathogen_name": "Rotavirus", "min": 30.0, "max": 60.0, "ReferenceID": "43"}, {"source_name": "surface water, contaminated", "pathogen_name": "Campylobacter jejuni", "min": 90.0, "max": 2500.0, "ReferenceID": "43"}, {"source_name": "surface water, contaminated", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 480.0, "ReferenceID": "43"}], "surface water, general": [{"source_name": "surface water, general", "pathogen_name": "Rotavirus", "min": 0.01, "max": 100.0, "ReferenceID": "39"}, {"source_name": "surface water, general", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 10000.0, "ReferenceID": "39"}, {"source_name": "surface water, general", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1000.0, "ReferenceID": "39"}], "surface water, protected": [{"source_name": "surface water, protected", "pathogen_name": "Rotavirus", "min": 0.0, "max": 3.0, "ReferenceID": "43"}, {"source_name": "surface water, protected", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 1100.0, "ReferenceID": "43"}, {"source_name": "surface water, protected", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 240.0, "ReferenceID": "43"}]} \ No newline at end of file +{"groundwater": [{"id": 7, "source_name": "groundwater", "pathogen_name": "Rotavirus", "min": 0.0, "max": 2.0, "ReferenceID": "43"}, {"id": 15, "source_name": "groundwater", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 10.0, "ReferenceID": "43"}, {"id": 23, "source_name": "groundwater", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1.0, "ReferenceID": "43"}], "rainwater, rooftop harvesting": [{"id": 4, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Rotavirus", "min": 0.0, "max": 0.01, "ReferenceID": "44"}, {"id": 12, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 24.0, "ReferenceID": "44"}, {"id": 20, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 0.19, "ReferenceID": "44"}], "rainwater, stormwater harvesting": [{"id": 5, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Rotavirus", "min": 9.74510658007135, "max": 64.7460691472062, "ReferenceID": "45"}, {"id": 13, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Campylobacter jejuni", "min": 13.8694279635122, "max": 287.039358509118, "ReferenceID": "45"}, {"id": 21, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 4.52008261942372e-05, "max": 0.0880751977503127, "ReferenceID": "45"}], "sewage, raw": [{"id": 6, "source_name": "sewage, raw", "pathogen_name": "Rotavirus", "min": 50.0, "max": 5000.0, "ReferenceID": "39"}, {"id": 14, "source_name": "sewage, raw", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 1000000.0, "ReferenceID": "39"}, {"id": 22, "source_name": "sewage, raw", "pathogen_name": "Cryptosporidium parvum", "min": 1.0, "max": 10000.0, "ReferenceID": "39"}], "sewage, treated": [{"id": 0, "source_name": "sewage, treated", "pathogen_name": "Rotavirus", "min": 0.1, "max": 1000.0, "ReferenceID": "42"}, {"id": 8, "source_name": "sewage, treated", "pathogen_name": "Campylobacter jejuni", "min": 0.001, "max": 1000.0, "ReferenceID": "42"}, {"id": 16, "source_name": "sewage, treated", "pathogen_name": "Cryptosporidium parvum", "min": 0.01, "max": 10000.0, "ReferenceID": "42"}], "surface water, contaminated": [{"id": 2, "source_name": "surface water, contaminated", "pathogen_name": "Rotavirus", "min": 30.0, "max": 60.0, "ReferenceID": "43"}, {"id": 10, "source_name": "surface water, contaminated", "pathogen_name": "Campylobacter jejuni", "min": 90.0, "max": 2500.0, "ReferenceID": "43"}, {"id": 18, "source_name": "surface water, contaminated", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 480.0, "ReferenceID": "43"}], "surface water, general": [{"id": 1, "source_name": "surface water, general", "pathogen_name": "Rotavirus", "min": 0.01, "max": 100.0, "ReferenceID": "39"}, {"id": 9, "source_name": "surface water, general", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 10000.0, "ReferenceID": "39"}, {"id": 17, "source_name": "surface water, general", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1000.0, "ReferenceID": "39"}], "surface water, protected": [{"id": 3, "source_name": "surface water, protected", "pathogen_name": "Rotavirus", "min": 0.0, "max": 3.0, "ReferenceID": "43"}, {"id": 11, "source_name": "surface water, protected", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 1100.0, "ReferenceID": "43"}, {"id": 19, "source_name": "surface water, protected", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 240.0, "ReferenceID": "43"}]} \ No newline at end of file diff --git a/qmra/templates/layout.html b/qmra/templates/layout.html index dda83cc..b14c550 100644 --- a/qmra/templates/layout.html +++ b/qmra/templates/layout.html @@ -45,6 +45,11 @@
  • About
    • + {% if user.is_staff %} +
  • + Admin +
    • + {% endif %} diff --git a/qmra/urls.py b/qmra/urls.py index 17250d3..ceba860 100644 --- a/qmra/urls.py +++ b/qmra/urls.py @@ -9,7 +9,7 @@ path("dsgvo", views.dsgvo, name="dsgvo"), path("faqs", views.faqs, name="faqs"), path("imprint", views.imprint, name="imprint"), - path('admin/', admin.site.urls), + path('admin/', admin.site.urls, name="admin-site"), path("accounts/", include("django.contrib.auth.urls")), path('', include('qmra.risk_assessment.urls')), path('', include('qmra.user.urls')), diff --git a/qmra/user/models.py b/qmra/user/models.py index 254a98c..1d67b8d 100644 --- a/qmra/user/models.py +++ b/qmra/user/models.py @@ -1,7 +1,4 @@ -import uuid - from django.contrib.auth.models import AbstractUser -from django.db import models # Create your models here. From e9b6d51843c3c06199b1562ca646e35ada4dd707 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 10:19:10 +0200 Subject: [PATCH 02/11] fix tests --- .github/workflows/test.yaml | 2 ++ qmra/risk_assessment/tests/test_assess_risk.py | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e1a7ba4..d776067 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,6 +12,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' + - name: echo ref_name + run: echo ${{ github.ref_name }} - name: Install dependencies run: pip install -r requirements.txt && pip install -r requirements.test.txt - name: Test diff --git a/qmra/risk_assessment/tests/test_assess_risk.py b/qmra/risk_assessment/tests/test_assess_risk.py index a9e2dea..cf732de 100644 --- a/qmra/risk_assessment/tests/test_assess_risk.py +++ b/qmra/risk_assessment/tests/test_assess_risk.py @@ -1,6 +1,7 @@ """test computation of risk assessment""" import warnings +from django.core.management import call_command from django.test import TestCase from assertpy import assert_that @@ -11,6 +12,13 @@ class TestAssesRisk(TestCase): + databases = ["default", "qmra"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + call_command("seed_default_db") + def test_with_standard_pathogens_and_all_treatments(self): given_user = User.objects.create_user("test-user", "test-user@test.com", "password") given_user.save() @@ -106,7 +114,7 @@ def test_regression_test(self): given_inflows = [ Inflow.objects.create( risk_assessment=given_ra, - pathogen=inflow.pathogen_name, + pathogen=inflow.pathogen.name, min=inflow.min, max=inflow.max ) for inflow in QMRAInflows.get("groundwater") ] From 58361b1ffad8c78ddfe76ea8049814504af28d92 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 10:42:43 +0200 Subject: [PATCH 03/11] checkout the branch being deployed in the server --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 1c490ff..48cf3a0 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -25,7 +25,7 @@ jobs: username: ${{ secrets.DEPLOY_USER }} key: ${{ secrets.DEPLOY_SERVER_SSH_KEY }} script: | - cd ${{ secrets.DEPLOY_PATH }} && git pull + cd ${{ secrets.DEPLOY_PATH }} && git pull && git checkout ${{ github.ref_name }} microk8s ctr image import img.tar && rm img.tar cd infra/helm microk8s helm upgrade --install -f ./qmra/${{ inputs.environment }}.values.yaml qmra ./qmra -n qmra --set app_secret_key.value=${{ secrets.APP_SECRET_KEY }},image.tag=${{ env.sha_short }} \ No newline at end of file From 28dbedeecfe4dd80d746143ecf1826cf3db6ece3 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 10:50:58 +0200 Subject: [PATCH 04/11] fix wrong branche name --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 48cf3a0..b651671 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -25,7 +25,7 @@ jobs: username: ${{ secrets.DEPLOY_USER }} key: ${{ secrets.DEPLOY_SERVER_SSH_KEY }} script: | - cd ${{ secrets.DEPLOY_PATH }} && git pull && git checkout ${{ github.ref_name }} + cd ${{ secrets.DEPLOY_PATH }} && git pull && git checkout ${{ github.head_ref || github.ref_name }} microk8s ctr image import img.tar && rm img.tar cd infra/helm microk8s helm upgrade --install -f ./qmra/${{ inputs.environment }}.values.yaml qmra ./qmra -n qmra --set app_secret_key.value=${{ secrets.APP_SECRET_KEY }},image.tag=${{ env.sha_short }} \ No newline at end of file From 8c5136db8108f2bf3ae4a2043ff87925e9d18ddc Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 11:25:12 +0200 Subject: [PATCH 05/11] fix mustaches in deployment.yaml --- infra/helm/qmra/templates/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/helm/qmra/templates/deployment.yaml b/infra/helm/qmra/templates/deployment.yaml index 6f90230..dcb6d13 100644 --- a/infra/helm/qmra/templates/deployment.yaml +++ b/infra/helm/qmra/templates/deployment.yaml @@ -41,10 +41,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" envFrom: - configMapRef: - name: { { .Values.configmap_name } } + name: {{ .Values.configmap_name }} volumeMounts: - name: qmra_default - mountPath: { { .Values.qmra_default.mount_path } } + mountPath: {{ .Values.qmra_default.mount_path }} command: [ python, manage.py, migrate, --database, qmra ] containers: - name: {{ .Chart.Name }} @@ -87,7 +87,7 @@ spec: claimName: {{ include "app.fullname" . }}-sqlite-file-pvc - name: qmra_default persistentVolumeClaim: - claimName: { { include "app.fullname" . } }-qmra_default-file-pvc + claimName: {{ include "app.fullname" . }}-qmra_default-file-pvc - name: static persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-static-files-pvc From cb4855c6f377041879141e603ee5d444af087be2 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 11:32:39 +0200 Subject: [PATCH 06/11] fix pv & pvc value for qmra-default --- infra/helm/qmra/dev.values.yaml | 2 +- infra/helm/qmra/prod.values.yaml | 2 +- infra/helm/qmra/templates/deployment.yaml | 12 ++++++------ infra/helm/qmra/templates/volumes.yaml | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/infra/helm/qmra/dev.values.yaml b/infra/helm/qmra/dev.values.yaml index 0a97e88..313a449 100644 --- a/infra/helm/qmra/dev.values.yaml +++ b/infra/helm/qmra/dev.values.yaml @@ -17,7 +17,7 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db -qmra_default: +qmra-default: mount_path: /var/lib/qmra/default_qmra_data.db hostpath: /var/lib/qmra/default_qmra_data.db diff --git a/infra/helm/qmra/prod.values.yaml b/infra/helm/qmra/prod.values.yaml index ed3d45f..5538b6a 100644 --- a/infra/helm/qmra/prod.values.yaml +++ b/infra/helm/qmra/prod.values.yaml @@ -17,7 +17,7 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db -qmra_default: +qmra-default: mount_path: /var/lib/qmra/default_qmra_data.db hostpath: /var/lib/qmra/default_qmra_data.db diff --git a/infra/helm/qmra/templates/deployment.yaml b/infra/helm/qmra/templates/deployment.yaml index dcb6d13..3bb64ad 100644 --- a/infra/helm/qmra/templates/deployment.yaml +++ b/infra/helm/qmra/templates/deployment.yaml @@ -43,8 +43,8 @@ spec: - configMapRef: name: {{ .Values.configmap_name }} volumeMounts: - - name: qmra_default - mountPath: {{ .Values.qmra_default.mount_path }} + - name: qmra-default + mountPath: {{ .Values.qmra-default.mount_path }} command: [ python, manage.py, migrate, --database, qmra ] containers: - name: {{ .Chart.Name }} @@ -77,17 +77,17 @@ spec: volumeMounts: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} - - name: qmra_default - mountPath: {{ .Values.qmra_default.mount_path }} + - name: qmra-default + mountPath: {{ .Values.qmra-default.mount_path }} - name: static mountPath: {{ .Values.static.mount_path }} volumes: - name: sqlite persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-sqlite-file-pvc - - name: qmra_default + - name: qmra-default persistentVolumeClaim: - claimName: {{ include "app.fullname" . }}-qmra_default-file-pvc + claimName: {{ include "app.fullname" . }}-qmra-default-file-pvc - name: static persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-static-files-pvc diff --git a/infra/helm/qmra/templates/volumes.yaml b/infra/helm/qmra/templates/volumes.yaml index cf7578e..e48828a 100644 --- a/infra/helm/qmra/templates/volumes.yaml +++ b/infra/helm/qmra/templates/volumes.yaml @@ -33,7 +33,7 @@ spec: apiVersion: v1 kind: PersistentVolume metadata: - name: {{ include "app.fullname" . }}-qmra_default-file-pv + name: {{ include "app.fullname" . }}-qmra-default-file-pv spec: capacity: storage: 10Gi @@ -43,7 +43,7 @@ spec: persistentVolumeReclaimPolicy: Retain storageClassName: microk8s-hostpath hostPath: - path: {{ .Values.qmra_default.hostpath }} + path: {{ .Values.qmra-default.hostpath }} type: FileOrCreate --- apiVersion: v1 @@ -77,12 +77,12 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ include "app.fullname" . }}-qmra_default-file-pvc + name: {{ include "app.fullname" . }}-qmra-default-file-pvc spec: accessModes: - ReadWriteMany volumeMode: Filesystem - volumeName: {{ include "app.fullname" . }}-qmra_default-file-pv + volumeName: {{ include "app.fullname" . }}-qmra-default-file-pv resources: requests: storage: 1Gi From bc1fc06b849ded630fd561aa38a5c83ef97bbd20 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 11:41:50 +0200 Subject: [PATCH 07/11] other attempt --- infra/helm/qmra/dev.values.yaml | 2 +- infra/helm/qmra/prod.values.yaml | 2 +- infra/helm/qmra/templates/deployment.yaml | 10 +++++----- infra/helm/qmra/templates/volumes.yaml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/infra/helm/qmra/dev.values.yaml b/infra/helm/qmra/dev.values.yaml index 313a449..0a97e88 100644 --- a/infra/helm/qmra/dev.values.yaml +++ b/infra/helm/qmra/dev.values.yaml @@ -17,7 +17,7 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db -qmra-default: +qmra_default: mount_path: /var/lib/qmra/default_qmra_data.db hostpath: /var/lib/qmra/default_qmra_data.db diff --git a/infra/helm/qmra/prod.values.yaml b/infra/helm/qmra/prod.values.yaml index 5538b6a..ed3d45f 100644 --- a/infra/helm/qmra/prod.values.yaml +++ b/infra/helm/qmra/prod.values.yaml @@ -17,7 +17,7 @@ sqlite: mount_path: /var/lib/qmra/qmra.db hostpath: /var/lib/qmra/qmra.db -qmra-default: +qmra_default: mount_path: /var/lib/qmra/default_qmra_data.db hostpath: /var/lib/qmra/default_qmra_data.db diff --git a/infra/helm/qmra/templates/deployment.yaml b/infra/helm/qmra/templates/deployment.yaml index 3bb64ad..76a1f9d 100644 --- a/infra/helm/qmra/templates/deployment.yaml +++ b/infra/helm/qmra/templates/deployment.yaml @@ -43,8 +43,8 @@ spec: - configMapRef: name: {{ .Values.configmap_name }} volumeMounts: - - name: qmra-default - mountPath: {{ .Values.qmra-default.mount_path }} + - name: qmra_default + mountPath: {{ .Values.qmra_default.mount_path }} command: [ python, manage.py, migrate, --database, qmra ] containers: - name: {{ .Chart.Name }} @@ -77,15 +77,15 @@ spec: volumeMounts: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} - - name: qmra-default - mountPath: {{ .Values.qmra-default.mount_path }} + - name: qmra_default + mountPath: {{ .Values.qmra_default.mount_path }} - name: static mountPath: {{ .Values.static.mount_path }} volumes: - name: sqlite persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-sqlite-file-pvc - - name: qmra-default + - name: qmra_default persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-qmra-default-file-pvc - name: static diff --git a/infra/helm/qmra/templates/volumes.yaml b/infra/helm/qmra/templates/volumes.yaml index e48828a..a2669c1 100644 --- a/infra/helm/qmra/templates/volumes.yaml +++ b/infra/helm/qmra/templates/volumes.yaml @@ -43,7 +43,7 @@ spec: persistentVolumeReclaimPolicy: Retain storageClassName: microk8s-hostpath hostPath: - path: {{ .Values.qmra-default.hostpath }} + path: {{ .Values.qmra_default.hostpath }} type: FileOrCreate --- apiVersion: v1 From de6e628f9b099d98e6da8167bb4f7ded9e414d22 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 12:04:34 +0200 Subject: [PATCH 08/11] and another --- infra/helm/qmra/templates/deployment.yaml | 8 ++++---- infra/helm/qmra/templates/volumes.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/infra/helm/qmra/templates/deployment.yaml b/infra/helm/qmra/templates/deployment.yaml index 76a1f9d..e4b9c81 100644 --- a/infra/helm/qmra/templates/deployment.yaml +++ b/infra/helm/qmra/templates/deployment.yaml @@ -37,13 +37,13 @@ spec: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} command: [ python, manage.py, migrate ] - - name: migrate_qmra_default + - name: migrate-qmra-default image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" envFrom: - configMapRef: name: {{ .Values.configmap_name }} volumeMounts: - - name: qmra_default + - name: qmra-default mountPath: {{ .Values.qmra_default.mount_path }} command: [ python, manage.py, migrate, --database, qmra ] containers: @@ -77,7 +77,7 @@ spec: volumeMounts: - name: sqlite mountPath: {{ .Values.sqlite.mount_path }} - - name: qmra_default + - name: qmra-default mountPath: {{ .Values.qmra_default.mount_path }} - name: static mountPath: {{ .Values.static.mount_path }} @@ -85,7 +85,7 @@ spec: - name: sqlite persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-sqlite-file-pvc - - name: qmra_default + - name: qmra-default persistentVolumeClaim: claimName: {{ include "app.fullname" . }}-qmra-default-file-pvc - name: static diff --git a/infra/helm/qmra/templates/volumes.yaml b/infra/helm/qmra/templates/volumes.yaml index a2669c1..95859d5 100644 --- a/infra/helm/qmra/templates/volumes.yaml +++ b/infra/helm/qmra/templates/volumes.yaml @@ -36,7 +36,7 @@ metadata: name: {{ include "app.fullname" . }}-qmra-default-file-pv spec: capacity: - storage: 10Gi + storage: 2Gi volumeMode: Filesystem accessModes: - ReadWriteMany From 8ca944aa2eb05ee5236e56eea07b31f30be6b6f9 Mon Sep 17 00:00:00 2001 From: adaurat Date: Wed, 17 Sep 2025 13:34:06 +0200 Subject: [PATCH 09/11] fix path to default qmra db --- infra/helm/qmra/templates/configmap.yaml | 1 + qmra/settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/helm/qmra/templates/configmap.yaml b/infra/helm/qmra/templates/configmap.yaml index 3f85df0..270665e 100644 --- a/infra/helm/qmra/templates/configmap.yaml +++ b/infra/helm/qmra/templates/configmap.yaml @@ -7,6 +7,7 @@ data: DEBUG: "{{ .Values.debug }}" DOMAIN_NAME: {{ .Values.domain }} SQLITE_PATH: {{ .Values.sqlite.mount_path }} + DEFAULT_QMRA_PATH: {{ .Values.qmra_default.mount_path }} STATIC_ROOT: {{ .Values.static.mount_path }} --- apiVersion: v1 diff --git a/qmra/settings.py b/qmra/settings.py index 372b2fe..46c0181 100644 --- a/qmra/settings.py +++ b/qmra/settings.py @@ -100,7 +100,7 @@ }, 'qmra': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.getenv("SQLITE_PATH", BASE_DIR / 'default_qmra_data.db'), + 'NAME': os.getenv("DEFAULT_QMRA_PATH", BASE_DIR / 'default_qmra_data.db'), } } DATABASE_ROUTERS = ('qmra.risk_assessment.dbrouter.DBRouter',) From d862be6c4734b12d5c0b153850306457a243bd80 Mon Sep 17 00:00:00 2001 From: adaurat Date: Fri, 19 Sep 2025 14:44:24 +0200 Subject: [PATCH 10/11] cleanup code --- qmra/management/commands/export_default.py | 6 +-- qmra/risk_assessment/admin.py | 44 +++++++++++++++------- qmra/risk_assessment/forms.py | 9 ++++- qmra/risk_assessment/qmra_models.py | 38 +++++++++++++++---- qmra/static/data/default-exposures.json | 2 +- qmra/static/data/default-inflows.json | 2 +- qmra/static/data/default-pathogens.json | 2 +- qmra/static/data/default-treatments.json | 2 +- 8 files changed, 74 insertions(+), 31 deletions(-) diff --git a/qmra/management/commands/export_default.py b/qmra/management/commands/export_default.py index 94889ab..913f14f 100644 --- a/qmra/management/commands/export_default.py +++ b/qmra/management/commands/export_default.py @@ -13,8 +13,8 @@ def save_as_json(data, destination: str): class Command(BaseCommand): help = "export the default data of qmra to json files for serving them as statics" - def add_arguments(self, parser): - parser.add_argument('--format', type=str, help="'json' (default) or 'csv' ", default="json") + # def add_arguments(self, parser): + # parser.add_argument('--format', type=str, help="'json' (default) or 'csv' ", default="json") def handle(self, *args, **options): save_as_json( @@ -22,7 +22,7 @@ def handle(self, *args, **options): QMRASources.source ) save_as_json( - {pathogen.name: model_to_dict(pathogen) for pathogen in QMRAPathogen.objects.all()}, + {pathogen.name: pathogen.to_dict() for pathogen in QMRAPathogen.objects.all()}, QMRAPathogens.source ) save_as_json( diff --git a/qmra/risk_assessment/admin.py b/qmra/risk_assessment/admin.py index 806acc7..e554c13 100644 --- a/qmra/risk_assessment/admin.py +++ b/qmra/risk_assessment/admin.py @@ -1,42 +1,56 @@ from django.contrib import admin +from django.core.management import call_command + from qmra.risk_assessment.qmra_models import QMRASource, QMRAPathogen, QMRAInflow, \ - QMRATreatment, QMRAExposure + QMRATreatment, QMRAExposure, QMRAReference """ -- Separate QMRAModels from UserModels (and db) -- models.py must be able to load without an existing db -- db can be bootstrapped with csvs through seed_qmra_db -- served qmra_db can be downloaded as csv from the admin page -- run export_default and collectstatic every time an entity is saved - from django.core.management import call_command - call_command('my_command', 'foo', bar='baz') + """ -class DefaultInflowModelInline(admin.TabularInline): +def save_model(self, request, obj, form, change): + super(type(self), self).save_model(request, obj, form, change) + call_command("export_default") + call_command("collectstatic", "--no-input") + + +@admin.register(QMRAReference) +class QMRAReferenceAdmin(admin.ModelAdmin): + list_display = ["name", "link"] + save_model = save_model + + +class QMRAInflowInline(admin.TabularInline): model = QMRAInflow fields = ["pathogen", "min", "max", "reference"] @admin.register(QMRASource) -class DefaultSourceModelAdmin(admin.ModelAdmin): +class QMRASourceAdmin(admin.ModelAdmin): list_display = ["name", "description"] - inlines = [DefaultInflowModelInline] + inlines = [QMRAInflowInline] + + save_model = save_model @admin.register(QMRAExposure) -class DefaultExposureModelAdmin(admin.ModelAdmin): +class QMRAExposureAdmin(admin.ModelAdmin): list_display = ["name", "events_per_year", "volume_per_event"] # inlines = [ReferenceInline] + save_model = save_model + @admin.register(QMRAPathogen) -class DefaultPathogenModelAdmin(admin.ModelAdmin): +class QMRAPathogenAdmin(admin.ModelAdmin): list_display = ["name", "group"] + save_model = save_model + @admin.register(QMRATreatment) -class DefaultTreatmentModelAdmin(admin.ModelAdmin): +class QMRATreatmentAdmin(admin.ModelAdmin): list_display = [ "name", "group", "bacteria_min", @@ -55,3 +69,5 @@ class DefaultTreatmentModelAdmin(admin.ModelAdmin): ("protozoa_min", "protozoa_max"), "protozoa_reference" ] + + save_model = save_model diff --git a/qmra/risk_assessment/forms.py b/qmra/risk_assessment/forms.py index d11ec95..57b1bc7 100644 --- a/qmra/risk_assessment/forms.py +++ b/qmra/risk_assessment/forms.py @@ -208,10 +208,11 @@ def clean(self): class AddTreatmentForm(forms.Form): - select_treatment = forms.ChoiceField(choices=QMRATreatments.choices(), widget=forms.Select()) + select_treatment = forms.ChoiceField(choices=[], widget=forms.Select()) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.fields["select_treatment"].choices = QMRATreatments.choices() self.fields["select_treatment"].required = False self.fields["select_treatment"].label = "Select treatment to add" self.helper = FormHelper() @@ -235,9 +236,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.form_tag = False - self.form.base_fields["name"].choices = QMRATreatments.choices() + choices = QMRATreatments.choices() + self.form.base_fields["name"].choices = choices if not kwargs.get("queryset", False): self.queryset = Treatment.objects.none() + else: + # make sure the treatment name is still valid even it has been changed in the default + self.form.base_fields["name"].choices += [(t.name, t.name) for t in kwargs["queryset"] if t.name not in choices] def set_user(self, user: User): self.form.base_fields["name"].choices = [ diff --git a/qmra/risk_assessment/qmra_models.py b/qmra/risk_assessment/qmra_models.py index 3160b6a..9911bba 100644 --- a/qmra/risk_assessment/qmra_models.py +++ b/qmra/risk_assessment/qmra_models.py @@ -11,6 +11,29 @@ from django.utils.functional import classproperty +""" +IMPORTANT NOTE: + +qmra has 2 databases: +1. 'default' contains the users' defaults and risk_assessment data +2. 'qmra' contains the KWB's defaults and is managed only by KWB + +to bootstrap the 'qmra' db: +``` +python manage.py collect_default_static_entities # create data/default-*.json from the original QMRA data (.csv in raw_public_data/) +python manage.py seed_default_db # ingest the json into the 'qmra' db +``` +additionally, +``` +python manage.py export_default +``` +re-create the json **from** the 'qmra' db and is called every time an admin saves something in the admin page + +In order to provide the correct choices in server-side rendered forms, this module exposes `StaticEntities` models. +These classes read the .json files (not the 'qmra' db!) and scrape the keys for valid choices. + +""" + class ExponentialDistribution: def __init__(self, k): self.k = k @@ -48,9 +71,10 @@ def primary_key(self) -> str: @classproperty def raw_data(cls) -> dict[str, dict[str, Any]]: - if cls._raw_data is None: - with open(cls.source, "r") as f: - cls._raw_data = json.load(f) + # because an admin can change this data while the app runs, + # we need _raw_data to be loaded dynamically... + with open(cls.source, "r") as f: + cls._raw_data = json.load(f) return cls._raw_data @classproperty @@ -197,8 +221,6 @@ def choices(cls): class QMRAInflow(models.Model): - # pathogen_name: str - # source_name: str source = models.ForeignKey(QMRASource, on_delete=models.CASCADE) pathogen = models.ForeignKey(QMRAPathogen, on_delete=models.CASCADE) min: float = models.FloatField() @@ -208,11 +230,10 @@ class QMRAInflow(models.Model): @classmethod def from_dict(cls, data: dict): return QMRAInflow( - # pathogen_name=data["pathogen_name"], - # source_name=data["source_name"], pk=data["id"], source=QMRASource.objects.get(name=data["source_name"]), pathogen=QMRAPathogen.objects.get(name=data["pathogen_name"]), + reference_id=data.get("ReferenceID", None), min=data["min"], max=data["max"] ) @@ -221,6 +242,7 @@ def to_dict(self): data = model_to_dict(self, exclude={"source", "pathogen"}) data["source_name"] = self.source.name data["pathogen_name"] = self.pathogen.name + data["ReferenceID"] = str(self.reference.pk) if self.reference is not None else None data["id"] = self.pk return data @@ -319,7 +341,7 @@ def from_dict(cls, data): def to_dict(self): data = model_to_dict(self) data["id"] = self.pk - data["ReferenceId"] = str(self.reference.pk) + data["ReferenceID"] = str(self.reference.pk) if self.reference is not None else None return data diff --git a/qmra/static/data/default-exposures.json b/qmra/static/data/default-exposures.json index 2cfc5c4..261c552 100644 --- a/qmra/static/data/default-exposures.json +++ b/qmra/static/data/default-exposures.json @@ -1 +1 @@ -{"irrigation, unrestricted": {"id": 1, "name": "irrigation, unrestricted", "description": "100 g of lettuce leaves hold 10.8 mL water and cucumbers 0.4 mL at worst case (immediately post watering). A serve of lettuce (40 g) might hold 5 mL of recycled water and other produce might hold up to 1 mL per serve. Calculated frequencies are based on Autralian Bureau of Statistics (ABS) data", "events_per_year": 70, "volume_per_event": 0.005, "ReferenceID": "48"}, "domestic use, car washing": {"id": 2, "name": "domestic use, car washing", "description": "Assumed similar to garden watering estimated to typically occur every second day during dry months (half year). Exposure to aerosols occurs during watering.", "events_per_year": 25, "volume_per_event": 0.0001, "ReferenceID": null}, "irrigation, restricted": {"id": 3, "name": "irrigation, restricted", "description": "Based on unrestricted irrigation, but far less frequent due to restricted access", "events_per_year": 1, "volume_per_event": 0.005, "ReferenceID": null}, "domestic use, toilet flushing": {"id": 4, "name": "domestic use, toilet flushing", "description": "Frequency based on three uses of home toilet per day. Aerosol volumes are less than those produced by garden irrigation.", "events_per_year": 1100, "volume_per_event": 1e-05, "ReferenceID": "48"}, "drinking water": {"id": 5, "name": "drinking water", "description": "Assumption for ingestion of drinking water", "events_per_year": 365, "volume_per_event": 1.0, "ReferenceID": null}, "irrigation, public": {"id": 7, "name": "irrigation, public", "description": "Frequencies moderate as most people use municipal areas sparingly (estimate 1/2 - 3 weeks). People are unlikely to be directly exposed to large amounts of spray and therefore exposure is from indirect ingestion via contact with lawns, etc. Likely to be higher when used to irrigate facilities such as sports grounds or golf courses (estimate 1/week)\r\n\r\ngrounds and golf courses (estimate 1/week)", "events_per_year": 50, "volume_per_event": 0.001, "ReferenceID": "48"}, "irrigation, garden": {"id": 8, "name": "irrigation, garden", "description": "Garden watering estimated to typically occur every second day during dry months (half year). Routine exposure results from indirect ingestion via contact with plants, lawns, etc.", "events_per_year": 90, "volume_per_event": 0.001, "ReferenceID": "48"}, "domestic use, washing machine": {"id": 9, "name": "domestic use, washing machine", "description": "Assumes one member of household exposed. Calculated frequency based on Australian Bureau of Statistics (ABS) data. Aerosol volumes are less than those produced by garden irrigation (machines usually closed during operation).", "events_per_year": 100, "volume_per_event": 1e-05, "ReferenceID": "48"}} \ No newline at end of file +{"irrigation, unrestricted": {"id": 1, "name": "irrigation, unrestricted", "description": "100 g of lettuce leaves hold 10.8 mL water and cucumbers 0.4 mL at worst case (immediately post watering). A serve of lettuce (40 g) might hold 5 mL of recycled water and other produce might hold up to 1 mL per serve. Calculated frequencies are based on Autralian Bureau of Statistics (ABS) data", "events_per_year": 70, "volume_per_event": 0.005, "reference": 48, "ReferenceID": "48"}, "domestic use, car washing": {"id": 2, "name": "domestic use, car washing", "description": "Assumed similar to garden watering estimated to typically occur every second day during dry months (half year). Exposure to aerosols occurs during watering.", "events_per_year": 25, "volume_per_event": 0.0001, "reference": null, "ReferenceID": null}, "irrigation, restricted": {"id": 3, "name": "irrigation, restricted", "description": "Based on unrestricted irrigation, but far less frequent due to restricted access", "events_per_year": 1, "volume_per_event": 0.005, "reference": null, "ReferenceID": null}, "domestic use, toilet flushing": {"id": 4, "name": "domestic use, toilet flushing", "description": "Frequency based on three uses of home toilet per day. Aerosol volumes are less than those produced by garden irrigation.", "events_per_year": 1100, "volume_per_event": 1e-05, "reference": 48, "ReferenceID": "48"}, "drinking water": {"id": 5, "name": "drinking water", "description": "Assumption for ingestion of drinking water", "events_per_year": 365, "volume_per_event": 1.0, "reference": null, "ReferenceID": null}, "irrigation, public": {"id": 7, "name": "irrigation, public", "description": "Frequencies moderate as most people use municipal areas sparingly (estimate 1/2 - 3 weeks). People are unlikely to be directly exposed to large amounts of spray and therefore exposure is from indirect ingestion via contact with lawns, etc. Likely to be higher when used to irrigate facilities such as sports grounds or golf courses (estimate 1/week)\r\n\r\ngrounds and golf courses (estimate 1/week)", "events_per_year": 50, "volume_per_event": 0.001, "reference": 48, "ReferenceID": "48"}, "irrigation, garden": {"id": 8, "name": "irrigation, garden", "description": "Garden watering estimated to typically occur every second day during dry months (half year). Routine exposure results from indirect ingestion via contact with plants, lawns, etc.", "events_per_year": 90, "volume_per_event": 0.001, "reference": 48, "ReferenceID": "48"}, "domestic use, washing machine": {"id": 9, "name": "domestic use, washing machine", "description": "Assumes one member of household exposed. Calculated frequency based on Australian Bureau of Statistics (ABS) data. Aerosol volumes are less than those produced by garden irrigation (machines usually closed during operation).", "events_per_year": 100, "volume_per_event": 1e-05, "reference": 48, "ReferenceID": "48"}} \ No newline at end of file diff --git a/qmra/static/data/default-inflows.json b/qmra/static/data/default-inflows.json index 95883b0..293f3fe 100644 --- a/qmra/static/data/default-inflows.json +++ b/qmra/static/data/default-inflows.json @@ -1 +1 @@ -{"groundwater": [{"id": 7, "source_name": "groundwater", "pathogen_name": "Rotavirus", "min": 0.0, "max": 2.0, "ReferenceID": "43"}, {"id": 15, "source_name": "groundwater", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 10.0, "ReferenceID": "43"}, {"id": 23, "source_name": "groundwater", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1.0, "ReferenceID": "43"}], "rainwater, rooftop harvesting": [{"id": 4, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Rotavirus", "min": 0.0, "max": 0.01, "ReferenceID": "44"}, {"id": 12, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 24.0, "ReferenceID": "44"}, {"id": 20, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 0.19, "ReferenceID": "44"}], "rainwater, stormwater harvesting": [{"id": 5, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Rotavirus", "min": 9.74510658007135, "max": 64.7460691472062, "ReferenceID": "45"}, {"id": 13, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Campylobacter jejuni", "min": 13.8694279635122, "max": 287.039358509118, "ReferenceID": "45"}, {"id": 21, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Cryptosporidium parvum", "min": 4.52008261942372e-05, "max": 0.0880751977503127, "ReferenceID": "45"}], "sewage, raw": [{"id": 6, "source_name": "sewage, raw", "pathogen_name": "Rotavirus", "min": 50.0, "max": 5000.0, "ReferenceID": "39"}, {"id": 14, "source_name": "sewage, raw", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 1000000.0, "ReferenceID": "39"}, {"id": 22, "source_name": "sewage, raw", "pathogen_name": "Cryptosporidium parvum", "min": 1.0, "max": 10000.0, "ReferenceID": "39"}], "sewage, treated": [{"id": 0, "source_name": "sewage, treated", "pathogen_name": "Rotavirus", "min": 0.1, "max": 1000.0, "ReferenceID": "42"}, {"id": 8, "source_name": "sewage, treated", "pathogen_name": "Campylobacter jejuni", "min": 0.001, "max": 1000.0, "ReferenceID": "42"}, {"id": 16, "source_name": "sewage, treated", "pathogen_name": "Cryptosporidium parvum", "min": 0.01, "max": 10000.0, "ReferenceID": "42"}], "surface water, contaminated": [{"id": 2, "source_name": "surface water, contaminated", "pathogen_name": "Rotavirus", "min": 30.0, "max": 60.0, "ReferenceID": "43"}, {"id": 10, "source_name": "surface water, contaminated", "pathogen_name": "Campylobacter jejuni", "min": 90.0, "max": 2500.0, "ReferenceID": "43"}, {"id": 18, "source_name": "surface water, contaminated", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 480.0, "ReferenceID": "43"}], "surface water, general": [{"id": 1, "source_name": "surface water, general", "pathogen_name": "Rotavirus", "min": 0.01, "max": 100.0, "ReferenceID": "39"}, {"id": 9, "source_name": "surface water, general", "pathogen_name": "Campylobacter jejuni", "min": 100.0, "max": 10000.0, "ReferenceID": "39"}, {"id": 17, "source_name": "surface water, general", "pathogen_name": "Cryptosporidium parvum", "min": 0.0, "max": 1000.0, "ReferenceID": "39"}], "surface water, protected": [{"id": 3, "source_name": "surface water, protected", "pathogen_name": "Rotavirus", "min": 0.0, "max": 3.0, "ReferenceID": "43"}, {"id": 11, "source_name": "surface water, protected", "pathogen_name": "Campylobacter jejuni", "min": 0.0, "max": 1100.0, "ReferenceID": "43"}, {"id": 19, "source_name": "surface water, protected", "pathogen_name": "Cryptosporidium parvum", "min": 2.0, "max": 240.0, "ReferenceID": "43"}]} \ No newline at end of file +{"sewage, treated": [{"id": 0, "min": 0.1, "max": 1000.0, "reference": 42, "source_name": "sewage, treated", "pathogen_name": "Rotavirus", "ReferenceID": "42"}, {"id": 8, "min": 0.001, "max": 1000.0, "reference": 42, "source_name": "sewage, treated", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "42"}, {"id": 16, "min": 0.01, "max": 10000.0, "reference": 42, "source_name": "sewage, treated", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "42"}], "surface water, general": [{"id": 1, "min": 0.01, "max": 100.0, "reference": 39, "source_name": "surface water, general", "pathogen_name": "Rotavirus", "ReferenceID": "39"}, {"id": 9, "min": 100.0, "max": 10000.0, "reference": 39, "source_name": "surface water, general", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "39"}, {"id": 17, "min": 0.0, "max": 1000.0, "reference": 39, "source_name": "surface water, general", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "39"}], "surface water, contaminated": [{"id": 2, "min": 30.0, "max": 60.0, "reference": 43, "source_name": "surface water, contaminated", "pathogen_name": "Rotavirus", "ReferenceID": "43"}, {"id": 10, "min": 90.0, "max": 2500.0, "reference": 43, "source_name": "surface water, contaminated", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "43"}, {"id": 18, "min": 2.0, "max": 480.0, "reference": 43, "source_name": "surface water, contaminated", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "43"}], "surface water, protected": [{"id": 3, "min": 0.0, "max": 3.0, "reference": 43, "source_name": "surface water, protected", "pathogen_name": "Rotavirus", "ReferenceID": "43"}, {"id": 11, "min": 0.0, "max": 1100.0, "reference": 43, "source_name": "surface water, protected", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "43"}, {"id": 19, "min": 2.0, "max": 240.0, "reference": 43, "source_name": "surface water, protected", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "43"}], "rainwater, rooftop harvesting": [{"id": 4, "min": 0.0, "max": 0.01, "reference": 44, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Rotavirus", "ReferenceID": "44"}, {"id": 12, "min": 0.0, "max": 24.0, "reference": 44, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "44"}, {"id": 20, "min": 0.0, "max": 0.19, "reference": 44, "source_name": "rainwater, rooftop harvesting", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "44"}], "rainwater, stormwater harvesting": [{"id": 5, "min": 9.74510658007135, "max": 64.7460691472062, "reference": 45, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Rotavirus", "ReferenceID": "45"}, {"id": 13, "min": 13.8694279635122, "max": 287.039358509118, "reference": 45, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "45"}, {"id": 21, "min": 4.52008261942372e-05, "max": 0.0880751977503127, "reference": 45, "source_name": "rainwater, stormwater harvesting", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "45"}], "groundwater": [{"id": 7, "min": 0.0, "max": 2.0, "reference": 43, "source_name": "groundwater", "pathogen_name": "Rotavirus", "ReferenceID": "43"}, {"id": 15, "min": 0.0, "max": 10.0, "reference": 43, "source_name": "groundwater", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "43"}, {"id": 23, "min": 0.0, "max": 1.0, "reference": 43, "source_name": "groundwater", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "43"}], "sewage, raw": [{"id": 6, "min": 50.0, "max": 5000.0, "reference": 39, "source_name": "sewage, raw", "pathogen_name": "Rotavirus", "ReferenceID": "39"}, {"id": 14, "min": 100.0, "max": 1000000.0, "reference": 39, "source_name": "sewage, raw", "pathogen_name": "Campylobacter jejuni", "ReferenceID": "39"}, {"id": 22, "min": 1.0, "max": 10000.0, "reference": 39, "source_name": "sewage, raw", "pathogen_name": "Cryptosporidium parvum", "ReferenceID": "39"}]} \ No newline at end of file diff --git a/qmra/static/data/default-pathogens.json b/qmra/static/data/default-pathogens.json index 822cd4c..68414bd 100644 --- a/qmra/static/data/default-pathogens.json +++ b/qmra/static/data/default-pathogens.json @@ -1 +1 @@ -{"Campylobacter jejuni": {"id": 3, "name": "Campylobacter jejuni", "group": "Bacteria", "infection_to_illness": 0.3, "dalys_per_case": 0.0046, "best_fit_model": "beta-Poisson", "k": null, "alpha": 0.144, "n50": 890.0}, "Rotavirus": {"id": 32, "name": "Rotavirus", "group": "Viruses", "infection_to_illness": 0.5, "dalys_per_case": 0.014, "best_fit_model": "beta-Poisson", "k": null, "alpha": 0.253, "n50": 6.17}, "Cryptosporidium parvum": {"id": 34, "name": "Cryptosporidium parvum", "group": "Protozoa", "infection_to_illness": 0.7, "dalys_per_case": 0.0015, "best_fit_model": "exponential", "k": 0.0572, "alpha": null, "n50": null}} \ No newline at end of file +{"Campylobacter jejuni": {"id": 3, "group": "Bacteria", "name": "Campylobacter jejuni", "best_fit_model": "beta-Poisson", "k": null, "alpha": 0.144, "n50": 890.0, "infection_to_illness": 0.3, "dalys_per_case": 0.0046}, "Rotavirus": {"id": 32, "group": "Viruses", "name": "Rotavirus", "best_fit_model": "beta-Poisson", "k": null, "alpha": 0.253, "n50": 6.17, "infection_to_illness": 0.5, "dalys_per_case": 0.014}, "Cryptosporidium parvum": {"id": 34, "group": "Protozoa", "name": "Cryptosporidium parvum", "best_fit_model": "exponential", "k": 0.0572, "alpha": null, "n50": null, "infection_to_illness": 0.7, "dalys_per_case": 0.0015}} \ No newline at end of file diff --git a/qmra/static/data/default-treatments.json b/qmra/static/data/default-treatments.json index 7ada446..1703b84 100644 --- a/qmra/static/data/default-treatments.json +++ b/qmra/static/data/default-treatments.json @@ -1 +1 @@ -{"UV disinfection 40 mJ/cm2, drinking": {"id": 29, "name": "UV disinfection 40 mJ/cm2, drinking", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Log reduction for drinking water UV is based on closed UV-reactors wich have been validated according to appropriate standards (e.g. USEPA or DVGW). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Excessive turbidity and certain dissolved species inhibit this process; hence, turbidity should be kept below 1 NTU to support effective disinfection.", "bacteria_min": 4.6, "bacteria_max": 6.0, "bacteria_reference": "50", "protozoa_min": 2.5, "protozoa_max": 3.0, "protozoa_reference": "50", "viruses_min": 4.1, "viruses_max": 5.9, "viruses_reference": "50"}, "Conventional clarification": {"id": 1, "name": "Conventional clarification", "group": "Coagulation, flocculation and sedimentation", "description": "Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, rapid mixing, slow mixing and sedimentation. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings.", "bacteria_min": 0.2, "bacteria_max": 2.0, "bacteria_reference": "40", "protozoa_min": 1.0, "protozoa_max": 2.0, "protozoa_reference": "40", "viruses_min": 0.1, "viruses_max": 3.4, "viruses_reference": "40"}, "High-rate clarification": {"id": 3, "name": "High-rate clarification", "group": "Coagulation, flocculation and sedimentation", "description": "Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, mixing and enhanced sedimentation by flock blankets, lamellae- or tube settlers. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings.", "bacteria_min": null, "bacteria_max": null, "bacteria_reference": null, "protozoa_min": 2.0, "protozoa_max": 2.8, "protozoa_reference": "40", "viruses_min": null, "viruses_max": null, "viruses_reference": null}, "Slow sand filtration": {"id": 8, "name": "Slow sand filtration", "group": "Filtration", "description": "Water is filtered through a fixed bed sand operatied down flow with rates of 0.1 to 1 m/h and contact times of 3 to 6 hours. The filter is not backwashed. In weeks to months a 'schmutzdecke' will develop on the filter which enhances log removal. Grain size, flow rate and temperature also affect log removal. Consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU) are associated higher log removal of pathogens\r\n\r\nassociated with 1 - 2 log reduction of viruses and 2.5 - 3 log reduction of Cryptosporidiuma", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "40", "protozoa_min": 0.3, "protozoa_max": 5.0, "protozoa_reference": "40", "viruses_min": 0.25, "viruses_max": 4.0, "viruses_reference": "40"}, "Bank filtration": {"id": 9, "name": "Bank filtration", "group": "Pretreatment", "description": "Water is abstracted through wells located close to surface water, thus the bank serves as a natural filter. Log removal depends on travel distance and time, soil type (grain size),\r\n and geochemicl conditions (oxygen level, pH)", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "40", "protozoa_min": 1.0, "protozoa_max": 2.0, "protozoa_reference": "40", "viruses_min": 2.1, "viruses_max": 8.3, "viruses_reference": "40"}, "Storage reservoirs": {"id": 11, "name": "Storage reservoirs", "group": "Pretreatment", "description": "Water is protected from human recontamination in reservoirs, however wildlife and waterfoul may introduce pathogens. Log reduction occurs due to sedimentation, UV radiation from sunlight and die-off in time, depending on construction (mixing) and temperature. Reporded reduction based on residence time > 40 days (bacteria), 160 days (protozoa)", "bacteria_min": 0.7, "bacteria_max": 2.2, "bacteria_reference": "40", "protozoa_min": 1.4, "protozoa_max": 2.3, "protozoa_reference": "40", "viruses_min": null, "viruses_max": null, "viruses_reference": null}, "Chlorination, wastewater": {"id": 12, "name": "Chlorination, wastewater", "group": "Primary disinfection", "description": "Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported protozoan log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, effect in wastewater is limited since free chlorine will rapidly decay. \r\n\r\nEffective disinfection. Where this is not practical, turbidities should be kept below 5 NTU with higher chlorine doses or contact times. In addition to initial disinfection, the benefits of maintaining free chlorine residuals throughout distribution systems at or above 0.2 mg/l should be considered", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40"}, "Chlorine dioxide": {"id": 13, "name": "Chlorine dioxide", "group": "Primary disinfection", "description": "Log inactivation depends on chlorine dioxide concentration and contact time (CT); Turbidity and organics inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n effective disinfection Chlorine dioxide degrades rapidly and doesn't leave a disinfectand residual for distribution.", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40"}, "Ozonation, drinking water": {"id": 14, "name": "Ozonation, drinking water", "group": "Primary disinfection", "description": "Log inactivation depends on dissolved ozone concentration and contact time (CT); Turbidity and organics inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n\r\n effective disinfection. Ozone degrades rapidly and doesn't leave a disinfectand residual for distribution. Effective mixing and consistent contact time are crucial for disinfection due to the rapid degradation of ozone.\r\n\r\nCryptosporidium varies widely", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40"}, "UV disinfection 20 mJ/cm2, drinking": {"id": 15, "name": "UV disinfection 20 mJ/cm2, drinking", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Log reduction for drinking water UV is based on closed UV-reactors wich have been validated according to appropriate standards (e.g. USEPA or DVGW). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Excessive turbidity and certain dissolved species inhibit this process; hence, turbidity should be kept below 1 NTU to support effective disinfection.", "bacteria_min": 4.6, "bacteria_max": 6.0, "bacteria_reference": "50", "protozoa_min": 2.4, "protozoa_max": 3.0, "protozoa_reference": "50", "viruses_min": 2.0, "viruses_max": 3.1, "viruses_reference": "50"}, "Primary treatment": {"id": 16, "name": "Primary treatment", "group": "Pretreatment", "description": "Primary treatment consists of temporarily holding the sewage in a quiescent basin where heavy solids can settle to the bottom while oil, grease and lighter solids float to the surface. The settled and floating materials are removed and the remaining liquid may be discharged or subjected to secondary treatment", "bacteria_min": 0.0, "bacteria_max": 0.5, "bacteria_reference": "8", "protozoa_min": 0.0, "protozoa_max": 1.0, "protozoa_reference": "8", "viruses_min": 0.0, "viruses_max": 0.1, "viruses_reference": "8"}, "Secondary treatment": {"id": 17, "name": "Secondary treatment", "group": "Pretreatment", "description": "Secondary treatment consists of an activated sludge process to break down organics in the wastewater and a settling stage to separate the biologiscal sludge from the water.", "bacteria_min": 1.0, "bacteria_max": 3.0, "bacteria_reference": "8", "protozoa_min": 0.5, "protozoa_max": 1.5, "protozoa_reference": "8", "viruses_min": 0.5, "viruses_max": 2.0, "viruses_reference": "8"}, "Dual media filtration": {"id": 18, "name": "Dual media filtration", "group": "Filtration", "description": "Water is filtered through a fixed bed consisting of two layers of granular media (e.g. antracite and sand) generally operatied down flow with rates of 5 to 20 m/h and contact times of 4 to 15 minutes. They are regularly backwashed to remove built up solids in the filter. Log removal depends on filter media and coagulation pretreatment;consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU)\r\n are associated higher log removal of pathogens", "bacteria_min": 0.0, "bacteria_max": 1.0, "bacteria_reference": "8", "protozoa_min": 1.5, "protozoa_max": 2.5, "protozoa_reference": "8", "viruses_min": 0.5, "viruses_max": 3.0, "viruses_reference": "8"}, "Chlorination, drinking water": {"id": 20, "name": "Chlorination, drinking water", "group": "Primary disinfection", "description": "Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n effective disinfection. Where this is not practical, turbidities should be kept below 5 NTU with higher chlorine doses or contact times. In addition to initial disinfection, the benefits of maintaining free chlorine residuals throughout distribution systems at or above 0.2 mg/l should be considered", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "8", "protozoa_min": 0.0, "protozoa_max": 1.5, "protozoa_reference": "8", "viruses_min": 1.0, "viruses_max": 3.0, "viruses_reference": "8"}, "Reverse osmosis": {"id": 21, "name": "Reverse osmosis", "group": "Filtration", "description": "A reverse osmosis membrane is a thin sheet with small openings that removes solids and most soluble molecules, including salts (< 0,004 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound membranes, hollow fibers or sheets. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.44, "bacteria_max": 6.0, "bacteria_reference": "47", "protozoa_min": 5.75, "protozoa_max": 6.32, "protozoa_reference": "47", "viruses_min": 5.44, "viruses_max": 6.0, "viruses_reference": "47"}, "Ozonation, wastewater": {"id": 22, "name": "Ozonation, wastewater", "group": "Primary disinfection", "description": "Log inactivation depends on dissolved ozone concentration and contact time (CT); Turbidity and organics inhibit this process; Since wastewater is often turbidity and contains high organics that consume ozone, the actual CT cannot be determined accurately and therefore inactivation cannot be determined accurately. Still, effective mixing and consistent contact time are crucial for disinfection due to the rapid degradation of ozone.", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "8", "protozoa_min": null, "protozoa_max": null, "protozoa_reference": null, "viruses_min": 3.0, "viruses_max": 6.0, "viruses_reference": "8"}, "Wetlands, surface flow": {"id": 23, "name": "Wetlands, surface flow", "group": "Wetlands", "description": "An artificial wetland to treat municipal or industrial wastewater, greywater or stormwater runoff by a combination of sedimentation and biological processes including plants. Effect depends on design and climate, especially les log reduction at lower temperatures.", "bacteria_min": 1.5, "bacteria_max": 2.5, "bacteria_reference": "8", "protozoa_min": 0.5, "protozoa_max": 1.5, "protozoa_reference": "8", "viruses_min": null, "viruses_max": null, "viruses_reference": null}, "Wetlands, subsurface flow": {"id": 24, "name": "Wetlands, subsurface flow", "group": "Wetlands", "description": "An artificial wetland to treat municipal or industrial wastewater, greywater or stormwater runoff by a combination of sedimentation, filtration and biological processes including plants. Effect depends on design, soil/filter media and climate, especially les log reduction at lower temperatures.", "bacteria_min": 0.5, "bacteria_max": 3.0, "bacteria_reference": "8", "protozoa_min": 0.5, "protozoa_max": 2.0, "protozoa_reference": "8", "viruses_min": null, "viruses_max": null, "viruses_reference": null}, "UV disinfection, wastewater": {"id": 25, "name": "UV disinfection, wastewater", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Wastewater UV-reactors are generally open-channel reactors in which UV lamps are placed. Excessive turbidity and certain dissolved species inhibit this process; hence the effect in wastewater highly depends on the water quality an is generally lower than in drinking water at the same dose.", "bacteria_min": 2.0, "bacteria_max": 4.0, "bacteria_reference": "8", "protozoa_min": 3.0, "protozoa_max": 3.0, "protozoa_reference": "8", "viruses_min": 1.0, "viruses_max": 3.0, "viruses_reference": "8"}, "Microfiltration": {"id": 26, "name": "Microfiltration", "group": "Filtration", "description": "A microfiltration membrane is a thin sheet with small openings that removes solids (0.1-10 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 0.0, "bacteria_max": 4.3, "bacteria_reference": "46", "protozoa_min": 2.3, "protozoa_max": 6.0, "protozoa_reference": "46", "viruses_min": 0.0, "viruses_max": 3.7, "viruses_reference": "46"}, "Ultrafiltration": {"id": 27, "name": "Ultrafiltration", "group": "Filtration", "description": "An ultrafiltration membrane is a thin sheet with small openings that removes solids (0.005-0,2 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers, spiral wound or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.5, "bacteria_max": 6.0, "bacteria_reference": "47", "protozoa_min": 5.3, "protozoa_max": 6.5, "protozoa_reference": "47", "viruses_min": 2.69, "viruses_max": 5.14, "viruses_reference": "47"}, "Nanofiltration": {"id": 28, "name": "Nanofiltration", "group": "Filtration", "description": "An nanofiltration membrane is a thin sheet with small openings that removes solids and larger soluble molecules (0.001-0,03 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound or hollow fiber membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.44, "bacteria_max": 6.0, "bacteria_reference": "47", "protozoa_min": 5.75, "protozoa_max": 6.32, "protozoa_reference": "47", "viruses_min": 5.44, "viruses_max": 6.0, "viruses_reference": "47"}} \ No newline at end of file +{"Conventional clarification": {"id": 1, "name": "Conventional clarification", "group": "Coagulation, flocculation and sedimentation", "description": "Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, rapid mixing, slow mixing and sedimentation. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings.", "bacteria_min": 0.2, "bacteria_max": 2.0, "bacteria_reference": "40", "viruses_min": 0.1, "viruses_max": 3.4, "viruses_reference": "40", "protozoa_min": 1.0, "protozoa_max": 2.0, "protozoa_reference": "40"}, "High-rate clarification": {"id": 3, "name": "High-rate clarification", "group": "Coagulation, flocculation and sedimentation", "description": "Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, mixing and enhanced sedimentation by flock blankets, lamellae- or tube settlers. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings.", "bacteria_min": null, "bacteria_max": null, "bacteria_reference": null, "viruses_min": null, "viruses_max": null, "viruses_reference": null, "protozoa_min": 2.0, "protozoa_max": 2.8, "protozoa_reference": "40"}, "Slow sand filtration": {"id": 8, "name": "Slow sand filtration", "group": "Filtration", "description": "Water is filtered through a fixed bed sand operatied down flow with rates of 0.1 to 1 m/h and contact times of 3 to 6 hours. The filter is not backwashed. In weeks to months a 'schmutzdecke' will develop on the filter which enhances log removal. Grain size, flow rate and temperature also affect log removal. Consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU) are associated higher log removal of pathogens\r\n\r\nassociated with 1 - 2 log reduction of viruses and 2.5 - 3 log reduction of Cryptosporidiuma", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "40", "viruses_min": 0.25, "viruses_max": 4.0, "viruses_reference": "40", "protozoa_min": 0.3, "protozoa_max": 5.0, "protozoa_reference": "40"}, "Bank filtration-456": {"id": 9, "name": "Bank filtration-456", "group": "Pretreatment", "description": "Water is abstracted through wells located close to surface water, thus the bank serves as a natural filter. Log removal depends on travel distance and time, soil type (grain size),\r\n and geochemicl conditions (oxygen level, pH)", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "40", "viruses_min": 2.1, "viruses_max": 8.3, "viruses_reference": "40", "protozoa_min": 1.0, "protozoa_max": 2.0, "protozoa_reference": "40"}, "Storage reservoirs": {"id": 11, "name": "Storage reservoirs", "group": "Pretreatment", "description": "Water is protected from human recontamination in reservoirs, however wildlife and waterfoul may introduce pathogens. Log reduction occurs due to sedimentation, UV radiation from sunlight and die-off in time, depending on construction (mixing) and temperature. Reporded reduction based on residence time > 40 days (bacteria), 160 days (protozoa)", "bacteria_min": 0.7, "bacteria_max": 2.2, "bacteria_reference": "40", "viruses_min": null, "viruses_max": null, "viruses_reference": null, "protozoa_min": 1.4, "protozoa_max": 2.3, "protozoa_reference": "40"}, "Chlorination, wastewater": {"id": 12, "name": "Chlorination, wastewater", "group": "Primary disinfection", "description": "Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported protozoan log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, effect in wastewater is limited since free chlorine will rapidly decay. \r\n\r\nEffective disinfection. Where this is not practical, turbidities should be kept below 5 NTU with higher chlorine doses or contact times. In addition to initial disinfection, the benefits of maintaining free chlorine residuals throughout distribution systems at or above 0.2 mg/l should be considered", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40"}, "Chlorine dioxide": {"id": 13, "name": "Chlorine dioxide", "group": "Primary disinfection", "description": "Log inactivation depends on chlorine dioxide concentration and contact time (CT); Turbidity and organics inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n effective disinfection Chlorine dioxide degrades rapidly and doesn't leave a disinfectand residual for distribution.", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40"}, "Ozonation, drinking water": {"id": 14, "name": "Ozonation, drinking water", "group": "Primary disinfection", "description": "Log inactivation depends on dissolved ozone concentration and contact time (CT); Turbidity and organics inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n\r\n effective disinfection. Ozone degrades rapidly and doesn't leave a disinfectand residual for distribution. Effective mixing and consistent contact time are crucial for disinfection due to the rapid degradation of ozone.\r\n\r\nCryptosporidium varies widely", "bacteria_min": 2.0, "bacteria_max": 2.0, "bacteria_reference": "40", "viruses_min": 2.0, "viruses_max": 2.0, "viruses_reference": "40", "protozoa_min": 2.0, "protozoa_max": 2.0, "protozoa_reference": "40"}, "UV disinfection 20 mJ/cm2, drinking": {"id": 15, "name": "UV disinfection 20 mJ/cm2, drinking", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Log reduction for drinking water UV is based on closed UV-reactors wich have been validated according to appropriate standards (e.g. USEPA or DVGW). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Excessive turbidity and certain dissolved species inhibit this process; hence, turbidity should be kept below 1 NTU to support effective disinfection.", "bacteria_min": 4.6, "bacteria_max": 6.0, "bacteria_reference": "50", "viruses_min": 2.0, "viruses_max": 3.1, "viruses_reference": "50", "protozoa_min": 2.4, "protozoa_max": 3.0, "protozoa_reference": "50"}, "Primary treatment": {"id": 16, "name": "Primary treatment", "group": "Pretreatment", "description": "Primary treatment consists of temporarily holding the sewage in a quiescent basin where heavy solids can settle to the bottom while oil, grease and lighter solids float to the surface. The settled and floating materials are removed and the remaining liquid may be discharged or subjected to secondary treatment", "bacteria_min": 0.0, "bacteria_max": 0.5, "bacteria_reference": "8", "viruses_min": 0.0, "viruses_max": 0.1, "viruses_reference": "8", "protozoa_min": 0.0, "protozoa_max": 1.0, "protozoa_reference": "8"}, "Secondary treatment": {"id": 17, "name": "Secondary treatment", "group": "Pretreatment", "description": "Secondary treatment consists of an activated sludge process to break down organics in the wastewater and a settling stage to separate the biologiscal sludge from the water.", "bacteria_min": 1.0, "bacteria_max": 3.0, "bacteria_reference": "8", "viruses_min": 0.5, "viruses_max": 2.0, "viruses_reference": "8", "protozoa_min": 0.5, "protozoa_max": 1.5, "protozoa_reference": "8"}, "Dual media filtration": {"id": 18, "name": "Dual media filtration", "group": "Filtration", "description": "Water is filtered through a fixed bed consisting of two layers of granular media (e.g. antracite and sand) generally operatied down flow with rates of 5 to 20 m/h and contact times of 4 to 15 minutes. They are regularly backwashed to remove built up solids in the filter. Log removal depends on filter media and coagulation pretreatment;consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU)\r\n are associated higher log removal of pathogens", "bacteria_min": 0.0, "bacteria_max": 1.0, "bacteria_reference": "8", "viruses_min": 0.5, "viruses_max": 3.0, "viruses_reference": "8", "protozoa_min": 1.5, "protozoa_max": 2.5, "protozoa_reference": "8"}, "Chlorination, drinking water": {"id": 20, "name": "Chlorination, drinking water", "group": "Primary disinfection", "description": "Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, turbidity should be kept below 1 NTU to support\r\n effective disinfection. Where this is not practical, turbidities should be kept below 5 NTU with higher chlorine doses or contact times. In addition to initial disinfection, the benefits of maintaining free chlorine residuals throughout distribution systems at or above 0.2 mg/l should be considered", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "8", "viruses_min": 1.0, "viruses_max": 3.0, "viruses_reference": "8", "protozoa_min": 0.0, "protozoa_max": 1.5, "protozoa_reference": "8"}, "Reverse osmosis": {"id": 21, "name": "Reverse osmosis", "group": "Filtration", "description": "A reverse osmosis membrane is a thin sheet with small openings that removes solids and most soluble molecules, including salts (< 0,004 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound membranes, hollow fibers or sheets. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.44, "bacteria_max": 6.0, "bacteria_reference": "47", "viruses_min": 5.44, "viruses_max": 6.0, "viruses_reference": "47", "protozoa_min": 5.75, "protozoa_max": 6.32, "protozoa_reference": "47"}, "Ozonation, wastewater": {"id": 22, "name": "Ozonation, wastewater", "group": "Primary disinfection", "description": "Log inactivation depends on dissolved ozone concentration and contact time (CT); Turbidity and organics inhibit this process; Since wastewater is often turbidity and contains high organics that consume ozone, the actual CT cannot be determined accurately and therefore inactivation cannot be determined accurately. Still, effective mixing and consistent contact time are crucial for disinfection due to the rapid degradation of ozone.", "bacteria_min": 2.0, "bacteria_max": 6.0, "bacteria_reference": "8", "viruses_min": 3.0, "viruses_max": 6.0, "viruses_reference": "8", "protozoa_min": null, "protozoa_max": null, "protozoa_reference": null}, "Wetlands, surface flow": {"id": 23, "name": "Wetlands, surface flow", "group": "Wetlands", "description": "An artificial wetland to treat municipal or industrial wastewater, greywater or stormwater runoff by a combination of sedimentation and biological processes including plants. Effect depends on design and climate, especially les log reduction at lower temperatures.", "bacteria_min": 1.5, "bacteria_max": 2.5, "bacteria_reference": "8", "viruses_min": null, "viruses_max": null, "viruses_reference": null, "protozoa_min": 0.5, "protozoa_max": 1.5, "protozoa_reference": "8"}, "Wetlands, subsurface flow": {"id": 24, "name": "Wetlands, subsurface flow", "group": "Wetlands", "description": "An artificial wetland to treat municipal or industrial wastewater, greywater or stormwater runoff by a combination of sedimentation, filtration and biological processes including plants. Effect depends on design, soil/filter media and climate, especially les log reduction at lower temperatures.", "bacteria_min": 0.5, "bacteria_max": 3.0, "bacteria_reference": "8", "viruses_min": null, "viruses_max": null, "viruses_reference": null, "protozoa_min": 0.5, "protozoa_max": 2.0, "protozoa_reference": "8"}, "UV disinfection, wastewater": {"id": 25, "name": "UV disinfection, wastewater", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Wastewater UV-reactors are generally open-channel reactors in which UV lamps are placed. Excessive turbidity and certain dissolved species inhibit this process; hence the effect in wastewater highly depends on the water quality an is generally lower than in drinking water at the same dose.", "bacteria_min": 2.0, "bacteria_max": 4.0, "bacteria_reference": "8", "viruses_min": 1.0, "viruses_max": 3.0, "viruses_reference": "8", "protozoa_min": 3.0, "protozoa_max": 3.0, "protozoa_reference": "8"}, "Microfiltration": {"id": 26, "name": "Microfiltration", "group": "Filtration", "description": "A microfiltration membrane is a thin sheet with small openings that removes solids (0.1-10 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 0.0, "bacteria_max": 4.3, "bacteria_reference": "46", "viruses_min": 0.0, "viruses_max": 3.7, "viruses_reference": "46", "protozoa_min": 2.3, "protozoa_max": 6.0, "protozoa_reference": "46"}, "Ultrafiltration": {"id": 27, "name": "Ultrafiltration", "group": "Filtration", "description": "An ultrafiltration membrane is a thin sheet with small openings that removes solids (0.005-0,2 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers, spiral wound or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.5, "bacteria_max": 6.0, "bacteria_reference": "47", "viruses_min": 2.69, "viruses_max": 5.14, "viruses_reference": "47", "protozoa_min": 5.3, "protozoa_max": 6.5, "protozoa_reference": "47"}, "Nanofiltration": {"id": 28, "name": "Nanofiltration", "group": "Filtration", "description": "An nanofiltration membrane is a thin sheet with small openings that removes solids and larger soluble molecules (0.001-0,03 \u043f\u0457\u0405m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound or hollow fiber membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing.", "bacteria_min": 5.44, "bacteria_max": 6.0, "bacteria_reference": "47", "viruses_min": 5.44, "viruses_max": 6.0, "viruses_reference": "47", "protozoa_min": 5.75, "protozoa_max": 6.32, "protozoa_reference": "47"}, "UV disinfection 40 mJ/cm2, drinking": {"id": 29, "name": "UV disinfection 40 mJ/cm2, drinking", "group": "Primary disinfection", "description": "UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Log reduction for drinking water UV is based on closed UV-reactors wich have been validated according to appropriate standards (e.g. USEPA or DVGW). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Excessive turbidity and certain dissolved species inhibit this process; hence, turbidity should be kept below 1 NTU to support effective disinfection.", "bacteria_min": 4.6, "bacteria_max": 6.0, "bacteria_reference": "50", "viruses_min": 4.1, "viruses_max": 5.9, "viruses_reference": "50", "protozoa_min": 2.5, "protozoa_max": 3.0, "protozoa_reference": "50"}} \ No newline at end of file From 0cfcd2bc287139c47f3803531982efa404f6233c Mon Sep 17 00:00:00 2001 From: adaurat Date: Thu, 4 Dec 2025 10:17:49 +0100 Subject: [PATCH 11/11] add change notice modal --- .github/workflows/test.yaml | 2 - .gitignore | 3 +- .../default-changes-notification.html | 51 +++++++++++++++++++ .../templates/risk-assessment-list.html | 2 + qmra/user/views.py | 2 +- qmra/views.py | 2 +- 6 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 qmra/risk_assessment/templates/default-changes-notification.html diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d776067..e1a7ba4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,8 +12,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: echo ref_name - run: echo ${{ github.ref_name }} - name: Install dependencies run: pip install -r requirements.txt && pip install -r requirements.test.txt - name: Test diff --git a/.gitignore b/.gitignore index 1478d93..0241599 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ default_qmra_data.db qmra-prod.db dump* prod-migrations/ -*.tar \ No newline at end of file +*.tar +.vscode \ No newline at end of file diff --git a/qmra/risk_assessment/templates/default-changes-notification.html b/qmra/risk_assessment/templates/default-changes-notification.html new file mode 100644 index 0000000..7fcf757 --- /dev/null +++ b/qmra/risk_assessment/templates/default-changes-notification.html @@ -0,0 +1,51 @@ +
    + + + +
    \ No newline at end of file diff --git a/qmra/risk_assessment/templates/risk-assessment-list.html b/qmra/risk_assessment/templates/risk-assessment-list.html index 6f805f0..e86ebcd 100644 --- a/qmra/risk_assessment/templates/risk-assessment-list.html +++ b/qmra/risk_assessment/templates/risk-assessment-list.html @@ -32,6 +32,8 @@

    Comparison of infection risk

    + {% include "default-changes-notification.html" %} +
    diff --git a/qmra/user/views.py b/qmra/user/views.py index 466ca54..e6e5f3c 100644 --- a/qmra/user/views.py +++ b/qmra/user/views.py @@ -51,7 +51,7 @@ def login_view(request): # Check if authentication successful if user is not None: login(request, user) - return HttpResponseRedirect(reverse("index")) + return HttpResponseRedirect(f"{reverse('index')}?isLogin=1") else: return render( request, diff --git a/qmra/views.py b/qmra/views.py index e46ce0a..fbeaee7 100644 --- a/qmra/views.py +++ b/qmra/views.py @@ -9,7 +9,7 @@ def index(request): if request.user.is_authenticated: - return HttpResponseRedirect(reverse("assessments")) + return HttpResponseRedirect(f"{reverse('assessments')}?isLogin={request.GET.get('isLogin', 0)}") else: return render(request, "index.html")