diff --git a/docs/blackhole_db_setup.md b/docs/blackhole_db_setup.md
new file mode 100644
index 0000000..c0c1bf3
--- /dev/null
+++ b/docs/blackhole_db_setup.md
@@ -0,0 +1,84 @@
+## Steps to set up a blackhole database for local testing of the loading validator
+
+As part of the validator app, records are sent to a local instance of the apel
+loader class, to test if they load into an apel server database correctly.
+This is important as some errors in a record won't be detected by the syntax
+validator, but will still cause a record to fail to load.
+
+These records are loaded into a database with blackhole engined tables - these
+tables allow insert commands, but don't store any rows or data, as data is
+discarded on write. This allows complete checking that a record can be
+successfully loaded to a database, without having to deal with data being stored.
+
+The local instance of the apel loader class is within `monitoring/views.py`, and
+uses `monitoring/validatorSettings.py` to pull configuration settings from
+`monitoring/settings.ini` about the blackhole validator database.
+
+Steps to set up a blackhole-engined version of the apel server database:
+
+1. Ensure maraidb is started and enabled:
+ - `sudo su`
+ - `sudo systemctl start mariadb`
+ - `sudo systemctl enable mariadb`
+
+2. Login to mariadb with root:
+ - `mysql -u root -p`
+
+3. Install the blackhole plugin, and then verify it is installed:
+ - `INSTALL SONAME 'ha_blackhole';`
+ - `SHOW ENGINES;` (should be a row with BLACKHOLE and support as YES).
+
+4. Exit mariadb:
+ - `exit;`
+
+5. Set the global default storage engine to blackhole, so that when the database
+ schema gets applied, tables are created with the blackhole engine:
+ - find where your mariadb settings are stored (for me it was `/etc/my.cnf.d/`).
+ - either edit `server.cnf` or create a new `blackhole.cnf` file (what I did).
+ - in that file:
+ ```
+ [mysqld]
+ default-storage-engine=BLACKHOLE
+ ```
+ - This config setting means that any create table statements without an engine
+ defined will be set to a blackhole engine by default.
+
+6. Restart mariadb:
+ - `sudo systemctl restart mariadb`
+ - Running `SHOW ENGINES;` within mariadb at this point should show the Blackhole
+ row with support as DEFAULT;
+
+7. Create the database:
+ - `mysql -u root -p`
+ - `CREATE DATABASE validator_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;`
+ - `CREATE USER 'your_name'@'localhost' IDENTIFIED BY 'your_password';`
+ - `GRANT ALL PRIVILEGES ON validator_db.* TO 'your_name'@'localhost';`
+ - `FLUSH PRIVILEGES;`
+ - `exit;`
+
+8. Apply the apel server schema to the database:
+ - the schema is at https://github.com/apel/apel/blob/dev/schemas/server.sql.
+ - to do this step, I used my locally cloned version of apel as the
+ schema file path.
+ - `mysql -u root validator_db < path_to_apel/schemas/server.sql`
+
+9. Verify the schema applied correctly, and that the correct tables use a
+ blackhole engine:
+ - `mysql -u your_name -p`
+ - `SHOW DATABASES;`
+ - `USE validator_db;`
+ - `SHOW TABLES;` (check all tables are there)
+ - `SHOW TABLE STATUS;` (check that all tables either have a BLACKHOLE or
+ NULL engine)
+
+10. Populate settings.ini with the following, adding in the correct values:
+ - ```
+ [db_validator]
+ backend=mysql
+ hostname=localhost
+ name=validator_db
+ password=
+ port=3306
+ username=
+ ```
+ - these config options are picked up by the `validatorSettings.py` file.
diff --git a/monitoring/validator/templates/validator/validator_index.html b/monitoring/validator/templates/validator/validator_index.html
index 748741a..031266a 100644
--- a/monitoring/validator/templates/validator/validator_index.html
+++ b/monitoring/validator/templates/validator/validator_index.html
@@ -41,18 +41,28 @@
Record Validator
wrap="wrap"
>{{ input_record|default:''|escape }}
-
+
Select 'Validate' if you want to check only the formatting of the record(s).
+
Select 'Load' if you want to also check whether the record(s) will successfully load into the database.
+ Validate
+
+ Load
+
+
{% if output %}
-
Validation Output
+
Submission Output
{{ output }}
{% endif %}
diff --git a/monitoring/validator/views.py b/monitoring/validator/views.py
index 1490d8a..5b0fb64 100644
--- a/monitoring/validator/views.py
+++ b/monitoring/validator/views.py
@@ -1,7 +1,9 @@
from django.shortcuts import render
from django.views.decorators.http import require_http_methods
-# Apel record-checking class imports
+# Apel loader and record-checking class imports
+from apel.db import ApelDbException
+from apel.db.loader.loader import Loader, LoaderException
from apel.db.loader.record_factory import RecordFactory, RecordFactoryException
from apel.db.records.record import InvalidRecordException
@@ -13,30 +15,56 @@
from apel.db.records.cloud import CloudRecord
from apel.db.records.cloud_summary import CloudSummaryRecord
+# Validator db configuration
+from monitoring import validatorSettings
+
+# Python tempfile for temporary apel queues
+import tempfile
+
@require_http_methods(["GET", "POST"])
def index(request):
"""
- Validates inputted records using the Apel record validation methods.
- It either validates a record against a specific type or against all types, depending on what record_type
+ Validates inputted records using the Apel record validation methods, or tests they load correctly using an
+ instance of the Apel loader.
+ For validation:
+ It either validates a record against a specific type or against all types, depending on what record_type
option was chosen on the html template. The default is `All`.
- The input record, record type and validation output are then returned to the html template as context on get
- request, so that the html page retains its information/context when refreshing the page or submitting the form.
+ For loading:
+ It both validates the record's syntax against all types, and checks it can load into a database correctly
+ using the Apel loader class. The database uses a blackhole engine, so no data is stored.
+ The input record, record type, submission type and validation output are then returned to the html template
+ as context on get request, so that the html page retains its information/context when refreshing the page
+ or submitting the form.
"""
+
template_name = "validator/validator_index.html"
input_record = ""
record_type = "All"
+ submission_type = ""
output = ""
- # On form submission, trigger record validation
+ # On form submission, check record isnt empty.
+ # Then trigger record validation or record loading, based on submission type
if request.method == "POST":
input_record = request.POST.get("input_record", "")
record_type = request.POST.get("record_type", "")
- output = validate(input_record, record_type)
+ submission_type = request.POST.get("submission_type", "")
+
+ if input_record:
+ input_record = input_record.strip()
+
+ if submission_type == "load":
+ output = load(input_record)
+ else:
+ output = validate(input_record, record_type)
+ else:
+ output = "Please enter a record to be validated."
context = {
"input_record": input_record,
"record_type": record_type,
+ "submission_type": submission_type,
"output": output,
}
@@ -45,17 +73,13 @@ def index(request):
def validate(record: str, record_type: str) -> str:
"""
- Validated record(s) and record_type passed in from the html page template.
- If record type is all, make use of the create_records apel method (expects a record header).
- Else, make use of the _create_record_objects apel method (expects there to be no record header).
+ Record(s) and record_type passed in from the html page template.
+ If record type is all, make use of the `create_records` apel method (expects a record header).
+ Else, make use of the `_create_record_objects` apel method (expects there to be no record header).
If the record is valid, return a "valid record" string.
If the record is invalid, an InvalidRecordException or RecordFactoryException is raised by the Apel methods.
Catch these exceptions and return the exception information.
"""
- if not record:
- return "Please enter a record to be validated."
-
- record = record.strip()
# Map record_type string to record_type class
# String is always exact as determined through html form selection option
@@ -80,10 +104,50 @@ def validate(record: str, record_type: str) -> str:
record_class = record_map[record_type]
result = recordFactory._create_record_objects(record, record_class)
- if "Record object at" in str(result):
+ if "object at" in str(result):
return "Record(s) valid!"
return str(result)
except (InvalidRecordException, RecordFactoryException) as e:
return str(e)
+
+def load(record: str) -> str:
+ """
+ Record passed in from the html page template.
+ Create a loader instance, making use of a tempfile directory for the queues and pidfile creation
+ Startup loader and then pass record(s) and blank signer into the `load_msg` method
+ Make use of a blackhole database, which loads but doesn't store any data.
+ If the record loads successfully, return a "valid record" string.
+ If the record fails to load, an exception is raised by the Apel methods.
+ Catch these exceptions and return the exception information.
+ """
+
+ try:
+ validatorDB = validatorSettings.VALIDATOR_DB
+
+ # Set the tempfile temporary directory within /tmp
+ tempfile.tempdir = "/tmp"
+
+ qpath = tempfile.gettempdir()
+ db_backend = validatorDB.get("ENGINE")
+ db_host = validatorDB.get("HOST")
+ db_port = int(validatorDB.get("PORT"))
+ db_name = validatorDB.get("NAME")
+ db_username = validatorDB.get("USER")
+ db_password = validatorDB.get("PASSWORD")
+ pidfile = ""
+ signer = ""
+
+ loader = Loader(qpath, record, db_backend, db_host, db_port, db_name, db_username, db_password, pidfile)
+
+ loader.startup()
+
+ loader.load_msg(record, signer)
+
+ loader.shutdown()
+
+ return("Record(s) will load successfully!")
+
+ except (ApelDbException, InvalidRecordException, LoaderException, RecordFactoryException) as e:
+ return str(e)
diff --git a/monitoring/validatorSettings.py b/monitoring/validatorSettings.py
new file mode 100644
index 0000000..7194ba8
--- /dev/null
+++ b/monitoring/validatorSettings.py
@@ -0,0 +1,33 @@
+"""
+Settings for the validator app, part of the monitoring project.
+
+These settings are for the apel modules, so they need to be kept separate
+from Django's control. Because Django was having issues with some of
+the defined configuration, such as the database engine.
+"""
+
+import configparser
+import os
+import sys
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+ # Read configuration from the file
+ cp = configparser.ConfigParser(interpolation=None)
+ file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini')
+ cp.read(file_path)
+
+ VALIDATOR_DB = {
+ 'ENGINE': cp.get('db_validator', 'backend'),
+ 'HOST': cp.get('db_validator', 'hostname'),
+ 'PORT': cp.get('db_validator', 'port'),
+ 'NAME': cp.get('db_validator', 'name'),
+ 'USER': cp.get('db_validator', 'username'),
+ 'PASSWORD': cp.get('db_validator', 'password'),
+ }
+
+except (configparser.NoSectionError) as err:
+ print("Error in configuration file. Check that file exists first: %s" % err)
+ sys.exit(1)
diff --git a/requirements.txt b/requirements.txt
index b8bae58..69f5922 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,4 @@
# Pin packages to support and work with py3.6.
-apel
Django==3.2.25
djangorestframework==3.15.1
pytz==2025.2