diff --git a/docs/blackhole_db_setup.md b/docs/blackhole_db_setup.md new file mode 100644 index 0000000..c0c1bf3 --- /dev/null +++ b/docs/blackhole_db_setup.md @@ -0,0 +1,84 @@ +## Steps to set up a blackhole database for local testing of the loading validator + +As part of the validator app, records are sent to a local instance of the apel +loader class, to test if they load into an apel server database correctly. +This is important as some errors in a record won't be detected by the syntax +validator, but will still cause a record to fail to load. + +These records are loaded into a database with blackhole engined tables - these +tables allow insert commands, but don't store any rows or data, as data is +discarded on write. This allows complete checking that a record can be +successfully loaded to a database, without having to deal with data being stored. + +The local instance of the apel loader class is within `monitoring/views.py`, and +uses `monitoring/validatorSettings.py` to pull configuration settings from +`monitoring/settings.ini` about the blackhole validator database. + +Steps to set up a blackhole-engined version of the apel server database: + +1. Ensure maraidb is started and enabled: + - `sudo su` + - `sudo systemctl start mariadb` + - `sudo systemctl enable mariadb` + +2. Login to mariadb with root: + - `mysql -u root -p` + +3. Install the blackhole plugin, and then verify it is installed: + - `INSTALL SONAME 'ha_blackhole';` + - `SHOW ENGINES;` (should be a row with BLACKHOLE and support as YES). + +4. Exit mariadb: + - `exit;` + +5. Set the global default storage engine to blackhole, so that when the database + schema gets applied, tables are created with the blackhole engine: + - find where your mariadb settings are stored (for me it was `/etc/my.cnf.d/`). + - either edit `server.cnf` or create a new `blackhole.cnf` file (what I did). + - in that file: + ``` + [mysqld] + default-storage-engine=BLACKHOLE + ``` + - This config setting means that any create table statements without an engine + defined will be set to a blackhole engine by default. + +6. Restart mariadb: + - `sudo systemctl restart mariadb` + - Running `SHOW ENGINES;` within mariadb at this point should show the Blackhole + row with support as DEFAULT; + +7. Create the database: + - `mysql -u root -p` + - `CREATE DATABASE validator_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;` + - `CREATE USER 'your_name'@'localhost' IDENTIFIED BY 'your_password';` + - `GRANT ALL PRIVILEGES ON validator_db.* TO 'your_name'@'localhost';` + - `FLUSH PRIVILEGES;` + - `exit;` + +8. Apply the apel server schema to the database: + - the schema is at https://github.com/apel/apel/blob/dev/schemas/server.sql. + - to do this step, I used my locally cloned version of apel as the + schema file path. + - `mysql -u root validator_db < path_to_apel/schemas/server.sql` + +9. Verify the schema applied correctly, and that the correct tables use a + blackhole engine: + - `mysql -u your_name -p` + - `SHOW DATABASES;` + - `USE validator_db;` + - `SHOW TABLES;` (check all tables are there) + - `SHOW TABLE STATUS;` (check that all tables either have a BLACKHOLE or + NULL engine) + +10. Populate settings.ini with the following, adding in the correct values: + - ``` + [db_validator] + backend=mysql + hostname=localhost + name=validator_db + password= + port=3306 + username= + ``` + - these config options are picked up by the `validatorSettings.py` file. diff --git a/monitoring/validator/templates/validator/validator_index.html b/monitoring/validator/templates/validator/validator_index.html index 748741a..031266a 100644 --- a/monitoring/validator/templates/validator/validator_index.html +++ b/monitoring/validator/templates/validator/validator_index.html @@ -41,18 +41,28 @@

Record Validator

wrap="wrap" >{{ input_record|default:''|escape }}

- +

Select 'Validate' if you want to check only the formatting of the record(s).

+

Select 'Load' if you want to also check whether the record(s) will successfully load into the database.

+ Validate +
+ Load +


+

{% if output %} -

Validation Output

+

Submission Output

{{ output }} {% endif %}
diff --git a/monitoring/validator/views.py b/monitoring/validator/views.py index 1490d8a..5b0fb64 100644 --- a/monitoring/validator/views.py +++ b/monitoring/validator/views.py @@ -1,7 +1,9 @@ from django.shortcuts import render from django.views.decorators.http import require_http_methods -# Apel record-checking class imports +# Apel loader and record-checking class imports +from apel.db import ApelDbException +from apel.db.loader.loader import Loader, LoaderException from apel.db.loader.record_factory import RecordFactory, RecordFactoryException from apel.db.records.record import InvalidRecordException @@ -13,30 +15,56 @@ from apel.db.records.cloud import CloudRecord from apel.db.records.cloud_summary import CloudSummaryRecord +# Validator db configuration +from monitoring import validatorSettings + +# Python tempfile for temporary apel queues +import tempfile + @require_http_methods(["GET", "POST"]) def index(request): """ - Validates inputted records using the Apel record validation methods. - It either validates a record against a specific type or against all types, depending on what record_type + Validates inputted records using the Apel record validation methods, or tests they load correctly using an + instance of the Apel loader. + For validation: + It either validates a record against a specific type or against all types, depending on what record_type option was chosen on the html template. The default is `All`. - The input record, record type and validation output are then returned to the html template as context on get - request, so that the html page retains its information/context when refreshing the page or submitting the form. + For loading: + It both validates the record's syntax against all types, and checks it can load into a database correctly + using the Apel loader class. The database uses a blackhole engine, so no data is stored. + The input record, record type, submission type and validation output are then returned to the html template + as context on get request, so that the html page retains its information/context when refreshing the page + or submitting the form. """ + template_name = "validator/validator_index.html" input_record = "" record_type = "All" + submission_type = "" output = "" - # On form submission, trigger record validation + # On form submission, check record isnt empty. + # Then trigger record validation or record loading, based on submission type if request.method == "POST": input_record = request.POST.get("input_record", "") record_type = request.POST.get("record_type", "") - output = validate(input_record, record_type) + submission_type = request.POST.get("submission_type", "") + + if input_record: + input_record = input_record.strip() + + if submission_type == "load": + output = load(input_record) + else: + output = validate(input_record, record_type) + else: + output = "Please enter a record to be validated." context = { "input_record": input_record, "record_type": record_type, + "submission_type": submission_type, "output": output, } @@ -45,17 +73,13 @@ def index(request): def validate(record: str, record_type: str) -> str: """ - Validated record(s) and record_type passed in from the html page template. - If record type is all, make use of the create_records apel method (expects a record header). - Else, make use of the _create_record_objects apel method (expects there to be no record header). + Record(s) and record_type passed in from the html page template. + If record type is all, make use of the `create_records` apel method (expects a record header). + Else, make use of the `_create_record_objects` apel method (expects there to be no record header). If the record is valid, return a "valid record" string. If the record is invalid, an InvalidRecordException or RecordFactoryException is raised by the Apel methods. Catch these exceptions and return the exception information. """ - if not record: - return "Please enter a record to be validated." - - record = record.strip() # Map record_type string to record_type class # String is always exact as determined through html form selection option @@ -80,10 +104,50 @@ def validate(record: str, record_type: str) -> str: record_class = record_map[record_type] result = recordFactory._create_record_objects(record, record_class) - if "Record object at" in str(result): + if "object at" in str(result): return "Record(s) valid!" return str(result) except (InvalidRecordException, RecordFactoryException) as e: return str(e) + +def load(record: str) -> str: + """ + Record passed in from the html page template. + Create a loader instance, making use of a tempfile directory for the queues and pidfile creation + Startup loader and then pass record(s) and blank signer into the `load_msg` method + Make use of a blackhole database, which loads but doesn't store any data. + If the record loads successfully, return a "valid record" string. + If the record fails to load, an exception is raised by the Apel methods. + Catch these exceptions and return the exception information. + """ + + try: + validatorDB = validatorSettings.VALIDATOR_DB + + # Set the tempfile temporary directory within /tmp + tempfile.tempdir = "/tmp" + + qpath = tempfile.gettempdir() + db_backend = validatorDB.get("ENGINE") + db_host = validatorDB.get("HOST") + db_port = int(validatorDB.get("PORT")) + db_name = validatorDB.get("NAME") + db_username = validatorDB.get("USER") + db_password = validatorDB.get("PASSWORD") + pidfile = "" + signer = "" + + loader = Loader(qpath, record, db_backend, db_host, db_port, db_name, db_username, db_password, pidfile) + + loader.startup() + + loader.load_msg(record, signer) + + loader.shutdown() + + return("Record(s) will load successfully!") + + except (ApelDbException, InvalidRecordException, LoaderException, RecordFactoryException) as e: + return str(e) diff --git a/monitoring/validatorSettings.py b/monitoring/validatorSettings.py new file mode 100644 index 0000000..7194ba8 --- /dev/null +++ b/monitoring/validatorSettings.py @@ -0,0 +1,33 @@ +""" +Settings for the validator app, part of the monitoring project. + +These settings are for the apel modules, so they need to be kept separate +from Django's control. Because Django was having issues with some of +the defined configuration, such as the database engine. +""" + +import configparser +import os +import sys + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +try: + # Read configuration from the file + cp = configparser.ConfigParser(interpolation=None) + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + cp.read(file_path) + + VALIDATOR_DB = { + 'ENGINE': cp.get('db_validator', 'backend'), + 'HOST': cp.get('db_validator', 'hostname'), + 'PORT': cp.get('db_validator', 'port'), + 'NAME': cp.get('db_validator', 'name'), + 'USER': cp.get('db_validator', 'username'), + 'PASSWORD': cp.get('db_validator', 'password'), + } + +except (configparser.NoSectionError) as err: + print("Error in configuration file. Check that file exists first: %s" % err) + sys.exit(1) diff --git a/requirements.txt b/requirements.txt index b8bae58..69f5922 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ # Pin packages to support and work with py3.6. -apel Django==3.2.25 djangorestframework==3.15.1 pytz==2025.2