diff --git a/ESSArch_Core/api/fields.py b/ESSArch_Core/api/fields.py new file mode 100644 index 000000000..0d4ea4bec --- /dev/null +++ b/ESSArch_Core/api/fields.py @@ -0,0 +1,21 @@ +import os + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + + +class FilePathField(serializers.CharField): + default_error_messages = { + 'invalid_path': _('{input} is not a valid path.'), + } + + def __init__(self, path, **kwargs): + self.path = path + super().__init__(**kwargs) + + def to_internal_value(self, data): + data = super().to_internal_value(data) + if not os.path.exists(os.path.join(self.path, data)): + self.fail('invalid_path', input=data) + + return os.path.join(self.path, data) diff --git a/ESSArch_Core/api/tests/test_fields.py b/ESSArch_Core/api/tests/test_fields.py new file mode 100644 index 000000000..f02110d4e --- /dev/null +++ b/ESSArch_Core/api/tests/test_fields.py @@ -0,0 +1,20 @@ +import os + +from rest_framework import serializers +from rest_framework.test import APITestCase + +from ESSArch_Core.api.fields import FilePathField + + +class FilePathFieldTests(APITestCase): + @classmethod + def setUpTestData(cls): + cls.field = FilePathField(os.path.abspath(os.path.dirname(__file__))) + + def test_valid_path(self): + self.assertEqual(self.field.run_validation(__file__), __file__) + self.assertEqual(self.field.run_validation(os.path.basename(__file__)), __file__) + + def test_invalid_path(self): + with self.assertRaises(serializers.ValidationError): + self.field.run_validation('invalid_file') diff --git a/ESSArch_Core/config/settings.py b/ESSArch_Core/config/settings.py index 174fad4ed..bdc083ac6 100644 --- a/ESSArch_Core/config/settings.py +++ b/ESSArch_Core/config/settings.py @@ -412,6 +412,7 @@ RABBITMQ_URL = os.environ.get('RABBITMQ_URL_ESSARCH', 'amqp://guest:guest@localhost:5672') CELERY_BROKER_URL = RABBITMQ_URL CELERY_IMPORTS = ( + "ESSArch_Core.fixity.validation.tasks", "ESSArch_Core.ip.tasks", "ESSArch_Core.maintenance.tasks", "ESSArch_Core.preingest.tasks", diff --git a/ESSArch_Core/config/urls.py b/ESSArch_Core/config/urls.py index f5e6505ed..1c04e0220 100644 --- a/ESSArch_Core/config/urls.py +++ b/ESSArch_Core/config/urls.py @@ -38,6 +38,7 @@ ConversionToolViewSet, ValidationFilesViewSet, ValidationViewSet, + ValidatorViewSet, ) from ESSArch_Core.ip.views import ( ConsignMethodViewSet, @@ -295,7 +296,6 @@ router.register(r'organizations', OrganizationViewSet, basename='organizations') - router.register(r'appraisal-jobs', AppraisalJobViewSet).register( r'information-packages', AppraisalJobInformationPackageViewSet, @@ -321,6 +321,7 @@ router.register(r'conversion-templates', ConversionTemplateViewSet) router.register(r'conversion-tools', ConversionToolViewSet) router.register(r'features', FeatureViewSet, basename='features') +router.register(r'validators', ValidatorViewSet, basename='validators') router.register(r'validations', ValidationViewSet) router.register(r'events', EventIPViewSet) router.register(r'event-types', EventTypeViewSet) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 06d28c3a1..7f08fd877 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -1,6 +1,8 @@ from rest_framework import serializers from ESSArch_Core.fixity.models import ConversionTool, Validation +from ESSArch_Core.fixity.validation import get_backend as get_validator +from ESSArch_Core.ip.models import InformationPackage class ConversionToolSerializer(serializers.ModelSerializer): @@ -11,6 +13,46 @@ class Meta: fields = ('name', 'form',) +class ValidatorWorkflowSerializer(serializers.Serializer): + purpose = serializers.CharField(default='Validation') + information_package = serializers.PrimaryKeyRelatedField(queryset=InformationPackage.objects.all()) + validators = serializers.ListField(min_length=1, child=serializers.JSONField()) + + def validate_validators(self, validators): + new_data = [] + ip = self.context['request'].data.get('information_package', None) + sub_context = {'information_package': ip} + sub_context.update(self.context) + + for validator in validators: + name = validator.pop('name') + klass = get_validator(name) + + serializer = klass.get_serializer_class()( + data=validator, context=sub_context, + ) + serializer.is_valid(True) + data = serializer.validated_data + data['name'] = name + + options_data = validator.pop('options', {}) + options_context = { + 'information_package': ip, + 'base_data': data, + } + options_serializer = klass.get_options_serializer_class()( + data=options_data, + context=options_context, + ) + + options_serializer.is_valid(True) + data['options'] = options_serializer.validated_data + + new_data.append(data) + + return new_data + + class ValidationSerializer(serializers.ModelSerializer): specification = serializers.JSONField(read_only=True) diff --git a/ESSArch_Core/fixity/tests/test_views.py b/ESSArch_Core/fixity/tests/test_views.py new file mode 100644 index 000000000..2b59ca694 --- /dev/null +++ b/ESSArch_Core/fixity/tests/test_views.py @@ -0,0 +1,13 @@ +from django.contrib.auth import get_user_model +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +User = get_user_model() + + +class ValidatorViewSetTests(APITestCase): + def test_list(self): + url = reverse('validators-list') + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) diff --git a/ESSArch_Core/fixity/validation/__init__.py b/ESSArch_Core/fixity/validation/__init__.py index 222ef2eb1..b107b8252 100644 --- a/ESSArch_Core/fixity/validation/__init__.py +++ b/ESSArch_Core/fixity/validation/__init__.py @@ -34,6 +34,15 @@ PATH_VARIABLE = "_PATH" +def get_backend(name): + try: + module_name, klass = AVAILABLE_VALIDATORS[name].rsplit('.', 1) + except KeyError: + raise ValueError('Validator "%s" not found' % name) + + return getattr(importlib.import_module(module_name), klass) + + def _validate_file(path, validators, task=None, ip=None, stop_at_failure=True, responsible=None): for validator in validators: included = False @@ -94,13 +103,7 @@ def validate_path(path, validators, profile, data=None, task=None, ip=None, stop validator_instances = [] for name in validators: - try: - module_name, validator_class = AVAILABLE_VALIDATORS[name].rsplit('.', 1) - except KeyError: - raise ValueError('Validator "%s" not found' % name) - - validator = getattr(importlib.import_module(module_name), validator_class) - + validator_klass = get_backend(name) for specification in profile.specification.get(name, []): required = specification.get('required', True) context = specification.get('context') @@ -108,7 +111,8 @@ def validate_path(path, validators, profile, data=None, task=None, ip=None, stop exclude = [os.path.join(path, excluded) for excluded in specification.get('exclude', [])] options = specification.get('options', {}) - validator_instance = validator( + validator_instance = validator_klass( + name, context=context, include=include, exclude=exclude, diff --git a/ESSArch_Core/fixity/validation/backends/base.py b/ESSArch_Core/fixity/validation/backends/base.py index b52aafead..76f77734b 100644 --- a/ESSArch_Core/fixity/validation/backends/base.py +++ b/ESSArch_Core/fixity/validation/backends/base.py @@ -1,4 +1,7 @@ import click +from rest_framework import serializers + +from ESSArch_Core.api.fields import FilePathField class BaseValidator: @@ -19,6 +22,28 @@ def __init__(self, context=None, include=None, exclude=None, options=None, self.ip = ip self.responsible = responsible + class Serializer(serializers.Serializer): + context = serializers.CharField() + + def __init__(self, *args, **kwargs): + from ESSArch_Core.ip.models import InformationPackage + + super().__init__(*args, **kwargs) + ip_pk = kwargs['context']['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') + + class OptionsSerializer(serializers.Serializer): + pass + + @classmethod + def get_serializer_class(cls): + return cls.Serializer + + @classmethod + def get_options_serializer_class(cls): + return cls.OptionsSerializer + def validate(self, filepath, expected=None): raise NotImplementedError('subclasses of BaseValidator must provide a validate() method') diff --git a/ESSArch_Core/fixity/validation/backends/checksum.py b/ESSArch_Core/fixity/validation/backends/checksum.py index ac2661654..298ce49cb 100644 --- a/ESSArch_Core/fixity/validation/backends/checksum.py +++ b/ESSArch_Core/fixity/validation/backends/checksum.py @@ -1,7 +1,8 @@ import logging -import traceback +import os from django.utils import timezone +from rest_framework import serializers from ESSArch_Core.essxml.util import find_file from ESSArch_Core.exceptions import ValidationError @@ -25,6 +26,59 @@ class ChecksumValidator(BaseValidator): * ``block_size``: Defaults to 65536 """ + label = 'Checksum Validator' + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'Path to validate', + 'required': True, + } + }, + { + 'key': 'options.algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + { + 'key': 'options.expected', + 'type': 'input', + 'templateOptions': { + 'label': 'Checksum', + 'required': True, + } + }, + ] + + class Serializer(BaseValidator.Serializer): + context = serializers.CharField(default='checksum_str') + block_size = serializers.IntegerField(default=65536) + + class OptionsSerializer(BaseValidator.OptionsSerializer): + expected = serializers.CharField() + algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -36,8 +90,14 @@ def __init__(self, *args, **kwargs): def validate(self, filepath, expected=None): logger.debug('Validating checksum of %s' % filepath) + + if self.ip is not None: + relpath = os.path.relpath(filepath, self.ip.object_path) + else: + relpath = filepath + val_obj = Validation.objects.create( - filename=filepath, + filename=relpath, time_started=timezone.now(), validator=self.__class__.__name__, required=self.required, @@ -66,14 +126,14 @@ def validate(self, filepath, expected=None): actual_checksum = calculate_checksum(filepath, algorithm=self.algorithm, block_size=self.block_size) if actual_checksum != checksum: raise ValidationError("checksum for %s is not valid (%s != %s)" % ( - filepath, checksum, actual_checksum + relpath, checksum, actual_checksum )) passed = True - except Exception: - val_obj.message = traceback.format_exc() + except Exception as e: + val_obj.message = str(e) raise else: - message = 'Successfully validated checksum of %s' % filepath + message = 'Successfully validated checksum of %s' % relpath val_obj.message = message logger.info(message) finally: diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 5b1ee6357..2becc173a 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -6,7 +6,9 @@ import click from django.utils import timezone from lxml import etree, isoschematron +from rest_framework import serializers +from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.essxml.util import ( find_files, find_pointers, @@ -31,6 +33,86 @@ class DiffCheckValidator(BaseValidator): """ file_validator = False + label = "Diff-check validator" + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'Path to validate', + } + }, + { + 'key': 'context', + 'type': 'input', + 'templateOptions': { + 'label': 'Metadata file', + 'required': True, + } + }, + { + 'key': 'options.recursive', + 'defaultValue': True, + 'type': 'checkbox', + 'templateOptions': { + 'label': 'Recursive', + } + }, + { + 'key': 'options.default_algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Default checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + ] + + class Serializer(BaseValidator.Serializer): + context = serializers.CharField() + + def __init__(self, *args, **kwargs): + from ESSArch_Core.ip.models import InformationPackage + + super().__init__(*args, **kwargs) + ip_pk = kwargs['context']['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + self.fields['context'] = FilePathField(ip.object_path, allow_blank=True, default='') + + class OptionsSerializer(BaseValidator.OptionsSerializer): + rootdir = serializers.CharField(required=False) + recursive = serializers.BooleanField(default=True) + default_algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) + + def validate(self, data): + if 'rootdir' not in data: + if self.context['base_data']['path']: + data['rootdir'] = self.context['base_data']['path'] + else: + from ESSArch_Core.ip.models import InformationPackage + + ip_pk = self.context['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + data['rootdir'] = ip.object_path + + return data def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -90,7 +172,7 @@ def _create_obj(self, filename, passed, msg): validator=self.__class__.__name__, required=self.required, task=self.task, - information_package_id=self.ip, + information_package=self.ip, responsible=self.responsible, message=msg, passed=passed, @@ -242,6 +324,56 @@ def validate(self, path, expected=None): class XMLComparisonValidator(DiffCheckValidator): + label = "XML Comparison validator" + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'First XML', + 'required': True, + } + }, + { + 'key': 'context', + 'type': 'input', + 'templateOptions': { + 'label': 'Second XML', + 'required': True, + } + }, + { + 'key': 'options.recursive', + 'defaultValue': True, + 'type': 'checkbox', + 'templateOptions': { + 'label': 'Recursive', + } + }, + { + 'key': 'options.default_algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Default checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + ] + def _get_files(self): skip_files = [p.path for p in find_pointers(self.context)] self.logical_files = find_files( @@ -387,6 +519,20 @@ def cli(path, schema): class XMLSyntaxValidator(BaseValidator): + label = "XML syntax validator" + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'Path to validate', + } + }, + ] + def validate(self, filepath, expected=None): logger.debug('Validating syntax of {xml}'.format(xml=filepath)) diff --git a/ESSArch_Core/fixity/validation/tasks.py b/ESSArch_Core/fixity/validation/tasks.py new file mode 100644 index 000000000..d592cf989 --- /dev/null +++ b/ESSArch_Core/fixity/validation/tasks.py @@ -0,0 +1,20 @@ +from ESSArch_Core.fixity.models import Validation +from ESSArch_Core.fixity.validation import get_backend +from ESSArch_Core.WorkflowEngine.dbtask import DBTask + + +class Validate(DBTask): + def run(self, name, path, context=None, options=None): + # delete validations from previous attempts + Validation.objects.filter(task=self.get_processtask()).delete() + + options = {} if options is None else options + klass = get_backend(name) + + validator = klass( + context=context, + ip=self.get_information_package(), + task=self.get_processtask(), + options=options, + ) + validator.validate(path) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 1c28b26cb..edef59cff 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -2,6 +2,7 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework import filters, viewsets from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response from rest_framework_extensions.mixins import NestedViewSetMixin from ESSArch_Core.api.filters import SearchFilter @@ -12,6 +13,10 @@ ValidationFilesSerializer, ValidationSerializer, ) +from ESSArch_Core.fixity.validation import ( + AVAILABLE_VALIDATORS, + get_backend as get_validator, +) class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): @@ -20,6 +25,32 @@ class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = ConversionToolSerializer +class ValidatorViewSet(viewsets.ViewSet): + permission_classes = () + + def list(self, request, format=None): + validators = {} + for k, _ in AVAILABLE_VALIDATORS.items(): + klass = get_validator(k) + try: + label = klass.label + except AttributeError: + label = klass.__name__ + + try: + form = klass.get_form() + except AttributeError: + form = [] + + validator = { + 'label': label, + 'form': form, + } + validators[k] = validator + + return Response(validators) + + class ValidationViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet): queryset = Validation.objects.all().order_by('filename', 'validator') serializer_class = ValidationSerializer diff --git a/ESSArch_Core/frontend/static/frontend/lang/en/index.ts b/ESSArch_Core/frontend/static/frontend/lang/en/index.ts index 60d8e57ee..86ce74619 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/en/index.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/en/index.ts @@ -17,6 +17,7 @@ import stateTree from './stateTree'; import sysInfo from './sysinfo'; import upload from './upload'; import userSettings from './userSettings'; +import validation from './validation'; export default [ access, @@ -38,4 +39,5 @@ export default [ sysInfo, upload, userSettings, + validation, ]; diff --git a/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts b/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts new file mode 100644 index 000000000..038139057 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts @@ -0,0 +1,13 @@ +/*@ngInject*/ +export default ($translateProvider: ng.translate.ITranslateProvider) => { + $translateProvider.translations('en', { + VALIDATION_VIEW: { + ADD_VALIDATOR: 'Add validator', + VALIDATOR: 'Validator', + REMOVE_VALIDATION: 'Remove validation', + RUN_VALIDATIONS: 'Run validations', + }, + VALIDATION: 'Validation', + NONE_SELECTED: 'None selected', + }); +}; diff --git a/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts b/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts index 60d8e57ee..86ce74619 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts @@ -17,6 +17,7 @@ import stateTree from './stateTree'; import sysInfo from './sysinfo'; import upload from './upload'; import userSettings from './userSettings'; +import validation from './validation'; export default [ access, @@ -38,4 +39,5 @@ export default [ sysInfo, upload, userSettings, + validation, ]; diff --git a/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts b/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts new file mode 100644 index 000000000..0db05109f --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts @@ -0,0 +1,13 @@ +/*@ngInject*/ +export default ($translateProvider: ng.translate.ITranslateProvider) => { + $translateProvider.translations('sv', { + VALIDATION_VIEW: { + ADD_VALIDATOR: 'Lägg till validator', + VALIDATOR: 'Validator', + REMOVE_VALIDATION: 'Ta bort validering', + RUN_VALIDATIONS: 'Kör valideringar', + }, + VALIDATION: 'Validering', + NONE_SELECTED: 'Ingen vald', + }); +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js new file mode 100644 index 000000000..20fe2b7c0 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js @@ -0,0 +1,11 @@ +import ValidationCtrl from '../controllers/ValidationCtrl'; + +export default { + templateUrl: 'static/frontend/views/validation_view.html', + controller: ['$scope', '$rootScope', 'appConfig', '$translate', '$http', '$timeout', '$uibModal', ValidationCtrl], + controllerAs: 'vm', + bindings: { + ip: '<', + baseUrl: '@', + }, +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js new file mode 100644 index 000000000..6b6c53ad4 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js @@ -0,0 +1,12 @@ +export default class RemoveValidationModalInstanceCtrl { + constructor($uibModalInstance, data) { + const $ctrl = this; + $ctrl.data = data; + $ctrl.remove = () => { + $uibModalInstance.close('remove'); + }; + $ctrl.cancel = function() { + $uibModalInstance.dismiss('cancel'); + }; + } +} diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js new file mode 100644 index 000000000..90f1dc525 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js @@ -0,0 +1,122 @@ +export default class ValidationCtrl { + constructor($scope, $rootScope, appConfig, $translate, $http, $timeout, $uibModal) { + const vm = this; + vm.flowOptions = {}; + vm.options = {validators: []}; + vm.fields = $scope.mockedValidations; + vm.activeTab = 'validation0'; + vm.purposeField = [ + { + key: 'purpose', + type: 'input', + templateOptions: { + label: $translate.instant('PURPOSE'), + }, + }, + ]; + + let tabNumber = 0; + vm.validations = [ + { + id: 0, + label: $translate.instant('VALIDATION') + ' 1', + validator: null, + data: {}, + }, + ]; + vm.currentValidation = vm.validations[0]; + vm.updateValidatorForm = validation => { + vm.currentValidation = validation; + if (validation.validator) { + vm.fields = validation.validator.form; + } else { + vm.fields = []; + } + }; + + vm.getValidators = function(search) { + return $http({ + url: appConfig.djangoUrl + 'validators/', + method: 'GET', + params: {search: search}, + }).then(function(response) { + let validators = []; + Object.keys(response.data).forEach(key => { + validators.push(angular.extend(response.data[key], {name: key})); + }); + vm.options.validators = validators; + return vm.options.validators; + }); + }; + + vm.addValidator = () => { + tabNumber++; + let val = { + id: tabNumber, + label: $translate.instant('VALIDATION') + ' ' + (tabNumber + 1), + validator: null, + data: {}, + }; + vm.validations.push(val); + $timeout(() => { + vm.activeTab = 'validation' + tabNumber; + vm.updateValidatorForm(val); + }); + }; + + vm.removeValidationModal = validation => { + var modalInstance = $uibModal.open({ + animation: true, + ariaLabelledBy: 'modal-title', + ariaDescribedBy: 'modal-body', + templateUrl: 'static/frontend/views/remove_validation_modal.html', + scope: $scope, + controller: 'RemoveValidationModalInstanceCtrl', + controllerAs: '$ctrl', + resolve: { + data: { + validation, + }, + }, + }); + modalInstance.result.then( + () => { + vm.validations.forEach((x, index, array) => { + if (x.id === validation.id) { + array.splice(index, 1); + tabNumber--; + } + }); + }, + function() {} + ); + }; + + vm.startValidation = () => { + if (vm.form.$invalid) { + vm.form.$setSubmitted(); + return; + } + let validations = vm.validations.filter(a => { + return a.validator !== null; + }); + if (validations.length > 0) { + vm.validations = validations; + } + if (!angular.isUndefined(vm.flowOptions.purpose) && vm.flowOptions.purpose === '') { + delete vm.flowOptions.purpose; + } + let data = angular.extend(vm.flowOptions, { + validators: vm.validations.map(x => { + let item = x.data; + item.name = x.validator.name; + return item; + }), + }); + let id = vm.baseUrl === 'workareas' ? vm.ip.workarea[0].id : vm.ip.id; + $http.post(appConfig.djangoUrl + vm.baseUrl + '/' + id + '/validate/', data).then(() => { + $rootScope.$broadcast('REFRESH_LIST_VIEW', {}); + }); + }; + } +} diff --git a/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js b/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js new file mode 100644 index 000000000..48f0556c7 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js @@ -0,0 +1,84 @@ +export default ($timeout, $document) => { + return { + link: function(scope, element, attrs, controller) { + // Attempt to integrate with ngRepeat + var match = attrs.ngRepeat.match(/^\s*([\s\S]+?)\s+in\s+([\s\S]+?)(?:\s+track\s+by\s+([\s\S]+?))?\s*$/); + var tabs; + scope.$watch(match[2], function(newTabs) { + tabs = newTabs; + }); + + var index = scope.$index; + scope.$watch('$index', function(newIndex) { + index = newIndex; + }); + + attrs.$set('draggable', true); + + // Wrapped in $apply so Angular reacts to changes + var wrappedListeners = { + // On item being dragged + dragstart: function(e) { + e.originalEvent.dataTransfer.effectAllowed = 'move'; + e.originalEvent.dataTransfer.dropEffect = 'move'; + e.originalEvent.dataTransfer.setData('application/json', index); + element.addClass('dragging'); + }, + dragend: function(e) { + //e.stopPropagation(); + element.removeClass('dragging'); + }, + + // On item being dragged over / dropped onto + dragenter: function(e) {}, + dragleave: function(e) { + element.removeClass('hover'); + }, + drop: function(e) { + e.preventDefault(); + e.stopPropagation(); + var sourceIndex = e.originalEvent.dataTransfer.getData('application/json'); + move(sourceIndex, index); + element.removeClass('hover'); + }, + }; + + // For performance purposes, do not + // call $apply for these + var unwrappedListeners = { + dragover: function(e) { + e.preventDefault(); + element.addClass('hover'); + }, + /* Use .hover instead of :hover. :hover doesn't play well with + moving DOM from under mouse when hovered */ + mouseenter: function() { + element.addClass('hover'); + }, + mouseleave: function() { + element.removeClass('hover'); + }, + }; + + angular.forEach(wrappedListeners, function(listener, event) { + element.on(event, wrap(listener)); + }); + + angular.forEach(unwrappedListeners, function(listener, event) { + element.on(event, listener); + }); + + function wrap(fn) { + return function(e) { + scope.$apply(function() { + fn(e); + }); + }; + } + + function move(fromIndex, toIndex) { + tabs.splice(toIndex, 0, tabs.splice(fromIndex, 1)[0]); + } + }, + }; +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js index 853fd0939..693761f38 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js @@ -24,6 +24,7 @@ import SaEditorComponent from '../components/SaEditorComponent'; import searchFilter from '../components/SearchFilterComponent'; import search from '../components/SearchComponent'; import StateTreeView from '../components/StateTreeViewComponent'; +import Validation from '../components/ValidationComponent'; export default angular .module('essarch.components', ['essarch.controllers']) @@ -52,4 +53,5 @@ export default angular .component('sysInfoComponent', sysInfoComponent) .component('search', search) .component('searchFilter', searchFilter) - .component('userDropdown', UserDropdownComponent).name; + .component('userDropdown', UserDropdownComponent) + .component('validationView', Validation).name; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js index bf4775fc1..d1aa74c5b 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js @@ -86,6 +86,7 @@ import ReceptionCtrl from '../controllers/ReceptionCtrl'; import RemoveNodeModalInstanceCtrl from '../controllers/RemoveNodeModalInstanceCtrl'; import RemoveStructureModalInstanceCtrl from '../controllers/RemoveStructureModalInstanceCtrl'; import RemoveStructureUnitModalInstanceCtrl from '../controllers/RemoveStructureUnitModalInstanceCtrl'; +import RemoveValidationModalInstanceCtrl from '../controllers/RemoveValidationModalInstanceCtrl'; import RequestModalInstanceCtrl from '../controllers/RequestModalInstanceCtrl'; import RestrictedCtrl from '../controllers/RestrictedCtrl'; import RobotInformationCtrl from '../controllers/RobotInformationCtrl'; @@ -963,6 +964,7 @@ export default angular '$translate', RemoveStructureUnitModalInstanceCtrl, ]) + .controller('RemoveValidationModalInstanceCtrl', ['$uibModalInstance', 'data', RemoveValidationModalInstanceCtrl]) .controller('RestrictedCtrl', ['$scope', RestrictedCtrl]) .controller('SavedSearchModalInstanceCtrl', [ '$uibModalInstance', diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js index 649512392..f077486ee 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js @@ -2,10 +2,12 @@ import focused from '../directives/focused'; import ngEnter from '../directives/ngEnter'; import treednd from '../directives/dragNdropDirective'; import fileread from '../directives/fileread'; +import sortableTab from '../directives/sortableTab'; export default angular .module('essarch.directives', ['essarch.services']) .directive('fileread', [fileread]) .directive('ngEnter', [ngEnter]) .directive('treednd', ['myService', treednd]) + .directive('sortableTab', ['$timeout', '$document', sortableTab]) .directive('focused', ['$timeout', '$parse', focused]).name; diff --git a/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss b/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss index dd3d9e342..f319c365d 100644 --- a/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss +++ b/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss @@ -3,3 +3,16 @@ .no-tabs-available { @include container(10px); } + +.nav-pills > li:hover > a { + background: transparent; + border-color: transparent; +} + +.nav-pills > li.hover > a { + background: #eeeeee; +} + +.nav-pills > li.dragging { + opacity: 0.5; +} diff --git a/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss b/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss new file mode 100644 index 000000000..40f9af184 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss @@ -0,0 +1,3 @@ +.validation-view { + padding: $padding-base-vertical $padding-base-horizontal; +} diff --git a/ESSArch_Core/frontend/static/frontend/styles/styles.scss b/ESSArch_Core/frontend/static/frontend/styles/styles.scss index a40d03cab..2d83c98b9 100644 --- a/ESSArch_Core/frontend/static/frontend/styles/styles.scss +++ b/ESSArch_Core/frontend/static/frontend/styles/styles.scss @@ -80,3 +80,4 @@ $icon-font-path: '~bootstrap-sass/assets/fonts/bootstrap/'; @import 'modules/search_filter'; @import 'modules/dashboard'; @import 'modules/form_errors'; +@import 'modules/validation'; diff --git a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html index f87e049f0..6c75cc51f 100644 --- a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html +++ b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html @@ -104,6 +104,11 @@ + +
+ +
+
diff --git a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html index f91d56794..c042cdf92 100644 --- a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html +++ b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html @@ -105,6 +105,11 @@
+ +
+ +
+
diff --git a/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html b/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html new file mode 100644 index 000000000..d8528a2ce --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html @@ -0,0 +1,30 @@ +
+ +
+
+

{{'VALIDATION' | translate}}

+
+ + + + + + + + + + + + + +
{{'LABEL' | translate}}{{'VALIDATION_VIEW.VALIDATOR' | translate}}
{{$ctrl.data.validation.label}} + {{$ctrl.data.validation.validator ? $ctrl.data.validation.validator.name : ('NONE_SELECTED' | translate)}} +
+
+
+
+ + +
diff --git a/ESSArch_Core/frontend/static/frontend/views/validation_view.html b/ESSArch_Core/frontend/static/frontend/views/validation_view.html new file mode 100644 index 000000000..24739859a --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/views/validation_view.html @@ -0,0 +1,89 @@ +
+
+ + + +
+ + {{'VALIDATION' | translate}} + +
+ +
+ + {{validation.validator.label}} + +
+ + +
{{'NO_RESULTS_FOUND' | translate}}
+ + +
+
+
+ +
+
+
+ + + + + + + + +
+ +
+ + +
+
+
diff --git a/ESSArch_Core/ip/tests/test_views.py b/ESSArch_Core/ip/tests/test_views.py index ff7cd75f6..a98bdf58c 100644 --- a/ESSArch_Core/ip/tests/test_views.py +++ b/ESSArch_Core/ip/tests/test_views.py @@ -2,12 +2,13 @@ import shutil import tempfile +from celery import states as celery_states from django.contrib.auth import get_user_model from django.contrib.auth.models import Permission from django.test import TestCase from django.urls import reverse from rest_framework import status -from rest_framework.test import APIClient +from rest_framework.test import APIClient, APITestCase from ESSArch_Core.auth.models import ( Group, @@ -16,6 +17,8 @@ GroupType, ) from ESSArch_Core.configuration.models import Path, StoragePolicy +from ESSArch_Core.exceptions import ValidationError +from ESSArch_Core.fixity.checksum import calculate_checksum from ESSArch_Core.ip.models import InformationPackage, Workarea from ESSArch_Core.profiles.models import SubmissionAgreement from ESSArch_Core.storage.models import ( @@ -28,6 +31,8 @@ StorageObject, StorageTarget, ) +from ESSArch_Core.testing.runner import TaskRunner +from ESSArch_Core.WorkflowEngine.models import ProcessTask User = get_user_model() @@ -316,3 +321,165 @@ def test_get_migratable_new_storage_target(self): ip_exists = InformationPackage.objects.migratable().exists() self.assertFalse(ip_exists) + + +class InformationPackageValidationTests(APITestCase): + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create(username='superuser', is_superuser=True) + + def setUp(self): + self.datadir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.datadir) + + Path.objects.create(entity='temp', value=tempfile.mkdtemp(dir=self.datadir)) + + self.ip = InformationPackage.objects.create( + object_path=tempfile.mkdtemp(dir=self.datadir), + ) + + @TaskRunner() + def test_validate(self): + self.client.force_authenticate(self.user) + url = reverse('informationpackage-validate', args=(str(self.ip.pk),)) + + os.makedirs(os.path.join(self.ip.object_path, 'content')) + + with open(os.path.join(self.ip.object_path, 'content', 'foo.txt'), 'w') as f: + f.write('hello') + + chksum = calculate_checksum(os.path.join(self.ip.object_path, 'content', 'foo.txt'), 'SHA-224') + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write('') + + with self.subTest('invalid file path'): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(ProcessTask.objects.exists()) + + with self.subTest('valid path and expected value'): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 0) + + with self.subTest('valid path and unexpected value'): + with self.assertRaises(ValidationError): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': 'incorrect', + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) + + with self.subTest('multiple validators'): + with self.assertRaises(ValidationError): + response = self.client.post(url, { + 'information_package': str(self.ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': '', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 2) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) + + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write(f'') + + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': '', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 4) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) + + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write(f'') + + response = self.client.post(url, { + 'validators': [ + { + 'name': 'diff_check', + 'path': 'content', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 5) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) diff --git a/ESSArch_Core/ip/views.py b/ESSArch_Core/ip/views.py index 2fbbc14e6..5885a3383 100644 --- a/ESSArch_Core/ip/views.py +++ b/ESSArch_Core/ip/views.py @@ -64,8 +64,12 @@ from ESSArch_Core.essxml.util import get_objectpath, parse_submit_description from ESSArch_Core.exceptions import Conflict, NoFileChunksFound from ESSArch_Core.fixity.format import FormatIdentifier +from ESSArch_Core.fixity.serializers import ValidatorWorkflowSerializer from ESSArch_Core.fixity.transformation import AVAILABLE_TRANSFORMERS -from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS +from ESSArch_Core.fixity.validation import ( + AVAILABLE_VALIDATORS, + get_backend as get_validator, +) from ESSArch_Core.fixity.validation.backends.checksum import ChecksumValidator from ESSArch_Core.ip.filters import ( AgentFilter, @@ -1924,46 +1928,48 @@ def unlock_profile(self, request, pk=None): ) }) - @transaction.atomic @action(detail=True, methods=['post'], url_path='validate') def validate(self, request, pk=None): ip = self.get_object() - prepare = Path.objects.get(entity="ingest_workarea").value - xmlfile = os.path.join(prepare, "%s.xml" % pk) + request.data['information_package'] = str(ip.pk) + serializer = ValidatorWorkflowSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + workflow_spec = [] + ip = serializer.validated_data['information_package'] - step = ProcessStep.objects.create( - name="Validation", - information_package=ip - ) + for validator in serializer.validated_data['validators']: + name = validator['name'] + klass = get_validator(name) - step.add_tasks( - ProcessTask.objects.create( - name="ESSArch_Core.tasks.ValidateXMLFile", - params={ - "xml_filename": xmlfile - }, - log=EventIP, - information_package=ip, - responsible=self.request.user, - ), - ProcessTask.objects.create( - name="ESSArch_Core.tasks.ValidateFiles", - params={ - "mets_path": xmlfile, - "validate_fileformat": True, - "validate_integrity": True, - }, - log=EventIP, - processstep_pos=0, - information_package=ip, - responsible=self.request.user, - ) - ) + options = validator['options'] + if validator['path']: + path = os.path.join(ip.object_path, validator['path']) + else: + path = ip.object_path - step.run() + task_spec = { + 'name': 'ESSArch_Core.fixity.validation.tasks.Validate', + 'label': 'Validate using {}'.format(klass.label), + 'args': [name, path], + 'params': {'context': validator['context'], 'options': options}, + } - return Response("Validating IP") + workflow_spec.append(task_spec) + + with transaction.atomic(): + step = { + 'step': True, + 'name': serializer.validated_data['purpose'], + 'parallel': True, + 'children': workflow_spec + } + workflow = create_workflow([step], ip=ip, name='Validation') + + workflow.run() + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) def update(self, request, *args, **kwargs): ip = self.get_object() diff --git a/ESSArch_Core/tasks.py b/ESSArch_Core/tasks.py index a3c51d14e..acddd1ad7 100644 --- a/ESSArch_Core/tasks.py +++ b/ESSArch_Core/tasks.py @@ -33,7 +33,6 @@ import requests from django.conf import settings from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError from django.core.mail import EmailMessage from django.db import transaction from django.utils import timezone @@ -53,7 +52,6 @@ findElementWithoutNamespace, ) from ESSArch_Core.essxml.util import find_pointers, get_premis_ref -from ESSArch_Core.fixity import validation from ESSArch_Core.fixity.models import Validation from ESSArch_Core.fixity.transformation import get_backend as get_transformer from ESSArch_Core.fixity.validation.backends.xml import ( @@ -316,66 +314,6 @@ def event_outcome_success(self, result, filename=None, format_name=None, ) -class ValidateWorkarea(DBTask): - queue = 'validation' - - def create_notification(self, ip): - errcount = Validation.objects.filter(information_package=ip, passed=False, required=True).count() - - if errcount: - Notification.objects.create( - message='Validation of "{ip}" failed with {errcount} error(s)'.format( - ip=ip.object_identifier_value, errcount=errcount - ), - level=logging.ERROR, - user_id=self.responsible, - refresh=True - ) - else: - Notification.objects.create( - message='"{ip}" was successfully validated'.format( - ip=ip.object_identifier_value - ), - level=logging.INFO, - user_id=self.responsible, - refresh=True - ) - - def run(self, workarea, validators, stop_at_failure=True): - workarea = Workarea.objects.get(pk=workarea) - workarea.successfully_validated = {} - - for validator in validators: - workarea.successfully_validated[validator] = None - - workarea.save(update_fields=['successfully_validated']) - ip = workarea.ip - sa = ip.submission_agreement - validation_profile = ip.get_profile('validation') - profile_data = fill_specification_data(data=ip.get_profile_data('validation'), sa=sa, ip=ip) - responsible = User.objects.get(pk=self.responsible) - - try: - validation.validate_path(workarea.path, validators, validation_profile, data=profile_data, ip=ip, - task=self.get_processtask(), stop_at_failure=stop_at_failure, - responsible=responsible) - except ValidationError: - self.create_notification(ip) - else: - self.create_notification(ip) - finally: - validations = ip.validation_set.all() - failed_validators = validations.values('validator').filter( - passed=False, required=True - ).values_list('validator', flat=True) - - for k, _v in workarea.successfully_validated.items(): - class_name = validation.AVAILABLE_VALIDATORS[k].split('.')[-1] - workarea.successfully_validated[k] = class_name not in failed_validators - - workarea.save(update_fields=['successfully_validated']) - - class TransformWorkarea(DBTask): def run(self, backend, workarea): workarea = Workarea.objects.select_related('ip__submission_agreement').get(pk=workarea) @@ -440,9 +378,9 @@ def run(self, path, xmlfile, skip_files=None, relpath=None): else: rootdir = os.path.dirname(path) - ip = InformationPackage.objects.get(pk=self.ip) + ip = self.get_information_package() validator = DiffCheckValidator(context=xmlfile, exclude=skip_files, options={'rootdir': rootdir}, - task=self.get_processtask(), ip=self.ip, responsible=ip.responsible) + task=self.get_processtask(), ip=ip, responsible=ip.responsible) validator.validate(path) def event_outcome_success(self, result, path, xmlfile, skip_files=None, relpath=None): @@ -459,7 +397,7 @@ class CompareXMLFiles(DBTask): def run(self, first, second, rootdir=None, recursive=True): Validation.objects.filter(task=self.get_processtask()).delete() first, second = self.parse_params(first, second) - ip = InformationPackage.objects.get(pk=self.ip) + ip = self.get_information_package() if rootdir is None: rootdir = ip.object_path else: @@ -469,7 +407,7 @@ def run(self, first, second, rootdir=None, recursive=True): context=first, options={'rootdir': rootdir, 'recursive': recursive}, task=self.get_processtask(), - ip=self.ip, + ip=ip, responsible=ip.responsible, ) validator.validate(second)