Gt1180 apel loading validator

monitoring/validator/templates/validator/validator_index.html

@@ -41,18 +41,28 @@ <h1>Record Validator</h2>
             wrap="wrap"
           >{{ input_record|default:''|escape }}</textarea>
           <br><br>
-          <button type="submit" id="validate-button">Validate!</button>
+          <p>Select 'Validate' if you want to check only the formatting of the record(s).</p>
+          <p>Select 'Load' if you want to also check whether the record(s) will successfully load into the database.</p>
+          <input type="radio" id="submission_type" name="submission_type" value="validate"> Validate
+          <br>
+          <input type="radio" id="submission_type" name="submission_type" value="load"> Load
+          <br><br><br>
+          <button type="submit" id="validate-button">Submit Record(s)!</button>
           <br><br>
         </form>
 
         <script>
-          // Setting select option back to what was selected (as it gets reset on form submission)
+          // Setting select and radio option back to what was selected (as they get reset on form submission)
           document.getElementById("record_type").value = "{{ record_type }}";
+          if ("{{ submission_type }}" != "") {
+              const checkRadio = document.querySelector(`input[name=submission_type][value="{{ submission_type }}"]`);
+              checkRadio.checked = true;
+          }
         </script>
 
         <div id="output">
           {% if output %}
-            <h3>Validation Output</h3>
+            <h3>Submission Output</h3>
             {{ output }}
           {% endif %}
         </div>

monitoring/validator/views.py

@@ -1,7 +1,9 @@
 from django.shortcuts import render
 from django.views.decorators.http import require_http_methods
 
-# Apel record-checking class imports
+# Apel loader and record-checking class imports
+from apel.db import ApelDbException
+from apel.db.loader.loader import Loader, LoaderException
 from apel.db.loader.record_factory import RecordFactory, RecordFactoryException
 from apel.db.records.record import InvalidRecordException
 
@@ -13,30 +15,56 @@
 from apel.db.records.cloud import CloudRecord
 from apel.db.records.cloud_summary import CloudSummaryRecord
 
+# Validator db configuration
+from monitoring import validatorSettings
+
+# Python tempfile for temporary apel queues
+import tempfile
+
 
 @require_http_methods(["GET", "POST"])
 def index(request):
     """
-    Validates inputted records using the Apel record validation methods.
-    It either validates a record against a specific type or against all types, depending on what record_type
+    Validates inputted records using the Apel record validation methods, or tests they load correctly using an
+        instance of the Apel loader.
+    For validation:
+        It either validates a record against a specific type or against all types, depending on what record_type
         option was chosen on the html template. The default is `All`.
-    The input record, record type and validation output are then returned to the html template as context on get
-        request, so that the html page retains its information/context when refreshing the page or submitting the form.
+    For loading:
+        It both validates the record's syntax against all types, and checks it can load into a database correctly
+        using the Apel loader class. The database uses a blackhole engine, so no data is stored.
+    The input record, record type, submission type and validation output are then returned to the html template
+        as context on get request, so that the html page retains its information/context when refreshing the page
+        or submitting the form.
     """
+
     template_name = "validator/validator_index.html"
     input_record = ""
     record_type = "All"
+    submission_type = ""
     output = ""
 
-    # On form submission, trigger record validation
+    # On form submission, check record isnt empty.
+    # Then trigger record validation or record loading, based on submission type
     if request.method == "POST":
         input_record = request.POST.get("input_record", "")
         record_type = request.POST.get("record_type", "")
-        output = validate(input_record, record_type)
+        submission_type = request.POST.get("submission_type", "")
+
+        if input_record:
+            input_record = input_record.strip()
+
+            if submission_type == "load":
+                output = load(input_record)
+            else:
+                output = validate(input_record, record_type)
+        else:
+            output = "Please enter a record to be validated."
 
     context = {
         "input_record": input_record,
         "record_type": record_type,
+        "submission_type": submission_type,
         "output": output,
     }
 
@@ -45,17 +73,13 @@ def index(request):
 
 def validate(record: str, record_type: str) -> str:
     """
-    Validated record(s) and record_type passed in from the html page template.
-    If record type is all, make use of the create_records apel method (expects a record header).
-    Else, make use of the _create_record_objects apel method (expects there to be no record header).
+    Record(s) and record_type passed in from the html page template.
+    If record type is all, make use of the `create_records` apel method (expects a record header).
+    Else, make use of the `_create_record_objects` apel method (expects there to be no record header).
     If the record is valid, return a "valid record" string.
     If the record is invalid, an InvalidRecordException or RecordFactoryException is raised by the Apel methods.
     Catch these exceptions and return the exception information.
     """
-    if not record:
-        return "Please enter a record to be validated."
-
-    record = record.strip()
 
     # Map record_type string to record_type class
     # String is always exact as determined through html form selection option
@@ -80,10 +104,50 @@ def validate(record: str, record_type: str) -> str:
             record_class = record_map[record_type]
             result = recordFactory._create_record_objects(record, record_class)
 
-        if "Record object at" in str(result):
+        if "object at" in str(result):
             return "Record(s) valid!"
 
         return str(result)
 
     except (InvalidRecordException, RecordFactoryException) as e:
         return str(e)
+
+def load(record: str) -> str:
+    """
+    Record passed in from the html page template.
+    Create a loader instance, making use of a tempfile directory for the queues and pidfile creation
+    Startup loader and then pass record(s) and blank signer into the `load_msg` method
+    Make use of a blackhole database, which loads but doesn't store any data.
+    If the record loads successfully, return a "valid record" string.
+    If the record fails to load, an exception is raised by the Apel methods.
+    Catch these exceptions and return the exception information.
+    """
+
+    try:
+        validatorDB = validatorSettings.VALIDATOR_DB
+
+        # Set the tempfile temporary directory within /tmp
+        tempfile.tempdir = "/tmp"
+
+        qpath = tempfile.gettempdir()
+        db_backend = validatorDB.get("ENGINE")
+        db_host = validatorDB.get("HOST")
+        db_port = int(validatorDB.get("PORT"))
+        db_name = validatorDB.get("NAME")
+        db_username = validatorDB.get("USER")
+        db_password = validatorDB.get("PASSWORD")
+        pidfile = ""
+        signer = ""
+
+        loader = Loader(qpath, record, db_backend, db_host, db_port, db_name, db_username, db_password, pidfile)
+
+        loader.startup()
+
+        loader.load_msg(record, signer)
+
+        loader.shutdown()
+
+        return("Record(s) will load successfully!")
+
+    except (ApelDbException, InvalidRecordException, LoaderException, RecordFactoryException) as e:
+        return str(e)

monitoring/validatorSettings.py

@@ -0,0 +1,33 @@
+"""
+Settings for the validator app, part of the monitoring project.
+
+These settings are for the apel modules, so they need to be kept separate
+from Django's control. Because Django was having issues with some of
+the defined configuration, such as the database engine.
+"""
+
+import configparser
+import os
+import sys
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    # Read configuration from the file
+    cp = configparser.ConfigParser(interpolation=None)
+    file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini')
+    cp.read(file_path)
+
+    VALIDATOR_DB = {
+        'ENGINE': cp.get('db_validator', 'backend'),
+        'HOST': cp.get('db_validator', 'hostname'),
+        'PORT': cp.get('db_validator', 'port'),
+        'NAME': cp.get('db_validator', 'name'),
+        'USER': cp.get('db_validator', 'username'),
+        'PASSWORD': cp.get('db_validator', 'password'),
+    }
+
+except (configparser.NoSectionError) as err:
+    print("Error in configuration file. Check that file exists first: %s" % err)
+    sys.exit(1)