add QMRA models for the admin page

.github/workflows/ci.yaml

@@ -5,6 +5,9 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   test:
@@ -25,13 +28,15 @@ jobs:
       environment: dev
     secrets: inherit
   build-prod:
+    if: github.ref_name == 'main'
     needs: test
     name: build-prod
     uses: ./.github/workflows/build.yaml
     with:
       environment: prod
     secrets: inherit
   deploy-prod:
+    if: github.ref_name == 'main'
     needs:
       - build-prod
       - deploy-dev

.github/workflows/deploy.yaml

@@ -25,7 +25,7 @@ jobs:
         username: ${{ secrets.DEPLOY_USER }}
         key: ${{ secrets.DEPLOY_SERVER_SSH_KEY }}
         script: |
-          cd ${{ secrets.DEPLOY_PATH }} && git pull
+          cd ${{ secrets.DEPLOY_PATH }} && git pull && git checkout ${{ github.head_ref || github.ref_name }}
           microk8s ctr image import img.tar && rm img.tar
           cd infra/helm
           microk8s helm upgrade --install -f ./qmra/${{ inputs.environment }}.values.yaml qmra ./qmra -n qmra --set app_secret_key.value=${{ secrets.APP_SECRET_KEY }},image.tag=${{ env.sha_short }}

.gitignore

@@ -4,7 +4,9 @@ venv/
 .idea
 .static
 qmra.db
+default_qmra_data.db
 qmra-prod.db
 dump*
 prod-migrations/
-*.tar
+*.tar
+.vscode

infra/helm/qmra/dev.values.yaml

@@ -17,6 +17,10 @@ sqlite:
   mount_path: /var/lib/qmra/qmra.db
   hostpath: /var/lib/qmra/qmra.db
 
+qmra_default:
+  mount_path: /var/lib/qmra/default_qmra_data.db
+  hostpath: /var/lib/qmra/default_qmra_data.db
+
 static:
   mount_path: /var/cache/qmra/static
   hostpath: /var/cache/qmra/static

infra/helm/qmra/prod.values.yaml

@@ -17,6 +17,10 @@ sqlite:
   mount_path: /var/lib/qmra/qmra.db
   hostpath: /var/lib/qmra/qmra.db
 
+qmra_default:
+  mount_path: /var/lib/qmra/default_qmra_data.db
+  hostpath: /var/lib/qmra/default_qmra_data.db
+
 static:
   mount_path: /var/cache/qmra/static
   hostpath: /var/cache/qmra/static

infra/helm/qmra/templates/configmap.yaml

@@ -7,6 +7,7 @@ data:
   DEBUG: "{{ .Values.debug }}"
   DOMAIN_NAME: {{ .Values.domain }}
   SQLITE_PATH: {{ .Values.sqlite.mount_path }}
+  DEFAULT_QMRA_PATH: {{ .Values.qmra_default.mount_path }}
   STATIC_ROOT: {{ .Values.static.mount_path }}
 ---
 apiVersion: v1

infra/helm/qmra/templates/deployment.yaml

@@ -37,6 +37,15 @@ spec:
             - name: sqlite
               mountPath: {{ .Values.sqlite.mount_path }}
           command: [ python, manage.py, migrate ]
+        - name: migrate-qmra-default
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          envFrom:
+            - configMapRef:
+                name: {{ .Values.configmap_name }}
+          volumeMounts:
+            - name: qmra-default
+              mountPath: {{ .Values.qmra_default.mount_path }}
+          command: [ python, manage.py, migrate, --database, qmra ]
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -68,12 +77,17 @@ spec:
           volumeMounts:
               - name: sqlite
                 mountPath: {{ .Values.sqlite.mount_path }}
+              - name: qmra-default
+                mountPath: {{ .Values.qmra_default.mount_path }}
               - name: static
                 mountPath: {{ .Values.static.mount_path }}
       volumes:
         - name: sqlite
           persistentVolumeClaim:
             claimName: {{ include "app.fullname" . }}-sqlite-file-pvc
+        - name: qmra-default
+          persistentVolumeClaim:
+            claimName: {{ include "app.fullname" . }}-qmra-default-file-pvc
         - name: static
           persistentVolumeClaim:
             claimName: {{ include "app.fullname" . }}-static-files-pvc

infra/helm/qmra/templates/volumes.yaml

@@ -31,6 +31,22 @@ spec:
     type: FileOrCreate
 ---
 apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "app.fullname" . }}-qmra-default-file-pv
+spec:
+  capacity:
+    storage: 2Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: microk8s-hostpath
+  hostPath:
+    path: {{ .Values.qmra_default.hostpath }}
+    type: FileOrCreate
+---
+apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: {{ include "app.fullname" . }}-static-files-pvc
@@ -53,6 +69,20 @@ spec:
     - ReadWriteMany
   volumeMode: Filesystem
   volumeName: {{ include "app.fullname" . }}-sqlite-file-pv
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: microk8s-hostpath
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "app.fullname" . }}-qmra-default-file-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  volumeMode: Filesystem
+  volumeName: {{ include "app.fullname" . }}-qmra-default-file-pv
   resources:
     requests:
       storage: 1Gi

qmra/management/commands/collect_static_default_entities.py

@@ -30,7 +30,8 @@ def get_default_inflows():
     inflows = pd.merge(inflows, sources, left_on="source_id", right_on="id", how="left").rename(columns={"name": "source_name"})
     inflows = pd.merge(inflows, pathogens, left_on="pathogen_id", right_on="id", how="left").rename(columns={"name": "pathogen_name"})
     inflows = inflows[inflows.pathogen_id.isin((3, 32, 34))]
-    return inflows.loc[:, ["source_name", "pathogen_name", "min", "max", "ReferenceID"]]
+    inflows["id"] = list(range(len(inflows.index)))
+    return inflows.loc[:, ["id", "source_name", "pathogen_name", "min", "max", "ReferenceID"]]
 
 
 def get_default_treatments():

qmra/management/commands/export_default.py

@@ -0,0 +1,48 @@
+import json
+from django.core.management.base import BaseCommand
+from qmra.risk_assessment.qmra_models import QMRAReference, QMRAReferences, QMRASource, \
+    QMRASources, QMRAPathogen, QMRAPathogens, QMRAInflow, QMRAInflows, QMRATreatment, \
+    QMRATreatments, QMRAExposure, QMRAExposures
+
+
+def save_as_json(data, destination: str):
+    with open(destination, "w") as f:
+        json.dump(data, f)
+
+
+class Command(BaseCommand):
+    help = "export the default data of qmra to json files for serving them as statics"
+
+    # def add_arguments(self, parser):
+    #     parser.add_argument('--format', type=str, help="'json' (default) or 'csv' ", default="json")
+
+    def handle(self, *args, **options):
+        save_as_json(
+            {src.name: src.to_dict() for src in QMRASource.objects.all()},
+            QMRASources.source
+        )
+        save_as_json(
+            {pathogen.name: pathogen.to_dict() for pathogen in QMRAPathogen.objects.all()},
+            QMRAPathogens.source
+        )
+        save_as_json(
+            {src.name: [inflow.to_dict() for inflow in QMRAInflow.objects.filter(source__name=src.name).all()]
+             for src in QMRASource.objects.all()},
+            QMRAInflows.source
+        )
+        save_as_json(
+            {t.name: t.to_dict() for t in QMRATreatment.objects.all()},
+            QMRATreatments.source
+        )
+        save_as_json(
+            {e.name: e.to_dict() for e in QMRAExposure.objects.all()},
+            QMRAExposures.source
+        )
+        save_as_json(
+            {str(ref.pk): ref.to_dict() for ref in QMRAReference.objects.all()},
+            QMRAReferences.source
+        )
+
+
+if __name__ == '__main__':
+    Command().handle()

qmra/management/commands/seed_default_db.py

@@ -0,0 +1,26 @@
+from django.core.management.base import BaseCommand
+from qmra.risk_assessment.qmra_models import QMRAReferences, QMRASources, QMRAPathogens, QMRAInflows, \
+    QMRATreatments, QMRAExposures
+
+
+class Command(BaseCommand):
+    help = "Create the default static data of qmra"
+
+    def handle(self, *args, **options):
+        for _, ref in QMRAReferences.data.items():
+            ref.save()
+        for _, pat in QMRAPathogens.data.items():
+            pat.save()
+        for _, source in QMRASources.data.items():
+            source.save()
+        for _, inflows in QMRAInflows.data.items():
+            for inflow in inflows:
+                inflow.save()
+        for _, treatment in QMRATreatments.data.items():
+            treatment.save()
+        for _, exposure in QMRAExposures.data.items():
+            exposure.save()
+
+
+if __name__ == '__main__':
+    Command().handle()

qmra/risk_assessment/admin.py

@@ -1,8 +1,73 @@
 from django.contrib import admin
+from django.core.management import call_command
 
-# Register your models here.
-# @admin.register(Health)
-# class HealthAdmin(admin.ModelAdmin):
-#     pass
+from qmra.risk_assessment.qmra_models import QMRASource, QMRAPathogen, QMRAInflow, \
+    QMRATreatment, QMRAExposure, QMRAReference
 
-# admin.site.register(RiskAssessment)
+"""
+
+"""
+
+
+def save_model(self, request, obj, form, change):
+    super(type(self), self).save_model(request, obj, form, change)
+    call_command("export_default")
+    call_command("collectstatic", "--no-input")
+
+
+@admin.register(QMRAReference)
+class QMRAReferenceAdmin(admin.ModelAdmin):
+    list_display = ["name", "link"]
+    save_model = save_model
+
+
+class QMRAInflowInline(admin.TabularInline):
+    model = QMRAInflow
+    fields = ["pathogen", "min", "max", "reference"]
+
+
+@admin.register(QMRASource)
+class QMRASourceAdmin(admin.ModelAdmin):
+    list_display = ["name", "description"]
+    inlines = [QMRAInflowInline]
+
+    save_model = save_model
+
+
+@admin.register(QMRAExposure)
+class QMRAExposureAdmin(admin.ModelAdmin):
+    list_display = ["name", "events_per_year", "volume_per_event"]
+    # inlines = [ReferenceInline]
+
+    save_model = save_model
+
+
+@admin.register(QMRAPathogen)
+class QMRAPathogenAdmin(admin.ModelAdmin):
+    list_display = ["name", "group"]
+
+    save_model = save_model
+
+
+@admin.register(QMRATreatment)
+class QMRATreatmentAdmin(admin.ModelAdmin):
+    list_display = [
+        "name", "group",
+        "bacteria_min",
+        "bacteria_max",
+        "viruses_min",
+        "viruses_max",
+        "protozoa_min",
+        "protozoa_max",
+    ]
+    fields = [
+        ("name", "group"),
+        ("bacteria_min", "bacteria_max"),
+        "bacteria_reference",
+        ("viruses_min", "viruses_max"),
+        "viruses_reference",
+        ("protozoa_min", "protozoa_max"),
+        "protozoa_reference"
+    ]
+
+    save_model = save_model

qmra/risk_assessment/dbrouter.py

@@ -0,0 +1,20 @@
+class DBRouter(object):
+    """Default* entities are managed by admins, everything else is for the users"""
+
+    def db_for_read(self, model, **hints):
+        if "QMRA" in model.__name__:
+            return 'qmra'
+        return "default"
+
+    def db_for_write(self, model, **hints):
+        if "QMRA" in model.__name__:
+            return 'qmra'
+        return "default"
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        """
+        Make sure the qmra data (and only this data!) is in its db
+        """
+        if model_name is not None and "qmra" in model_name:
+            return db == "qmra"
+        return db == "default"

qmra/risk_assessment/forms.py

@@ -5,8 +5,9 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Layout, Field, Row, Column, HTML
 
-from qmra.risk_assessment.models import Inflow, DefaultTreatments, Treatment, \
-    RiskAssessment, DefaultExposures, DefaultSources
+from qmra.risk_assessment.models import Inflow, Treatment, \
+    RiskAssessment
+from qmra.risk_assessment.qmra_models import QMRASources, QMRATreatments, QMRAExposures
 from qmra.user.models import User
 
 
@@ -31,8 +32,8 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields["source_name"].label = "Select a source water type to add pathogen concentrations"
-        self.fields["exposure_name"].choices = DefaultExposures.choices()
-        self.fields["source_name"].choices = DefaultSources.choices()
+        self.fields["exposure_name"].choices = QMRAExposures.choices()
+        self.fields["source_name"].choices = QMRASources.choices()
         self.fields['events_per_year'].widget.attrs['min'] = 0
         self.fields['volume_per_event'].widget.attrs['min'] = 0
         self.fields['volume_per_event'].label = "Volume per event in liters"
@@ -207,10 +208,11 @@ def clean(self):
 
 
 class AddTreatmentForm(forms.Form):
-    select_treatment = forms.ChoiceField(choices=DefaultTreatments.choices(), widget=forms.Select())
+    select_treatment = forms.ChoiceField(choices=[], widget=forms.Select())
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.fields["select_treatment"].choices = QMRATreatments.choices()
         self.fields["select_treatment"].required = False
         self.fields["select_treatment"].label = "Select treatment to add"
         self.helper = FormHelper()
@@ -234,9 +236,13 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.helper = FormHelper()
         self.helper.form_tag = False
-        self.form.base_fields["name"].choices = DefaultTreatments.choices()
+        choices = QMRATreatments.choices()
+        self.form.base_fields["name"].choices = choices
         if not kwargs.get("queryset", False):
             self.queryset = Treatment.objects.none()
+        else:
+            # make sure the treatment name is still valid even it has been changed in the default
+            self.form.base_fields["name"].choices += [(t.name, t.name) for t in kwargs["queryset"] if t.name not in choices]
 
     def set_user(self, user: User):
         self.form.base_fields["name"].choices = [