diff --git a/backend/gn_module_monitoring/command/imports/constant.py b/backend/gn_module_monitoring/command/imports/constant.py index fa1294a82..53401dc5f 100644 --- a/backend/gn_module_monitoring/command/imports/constant.py +++ b/backend/gn_module_monitoring/command/imports/constant.py @@ -116,9 +116,10 @@ TOOLTIPS = { "id_base_site_origin": "Identifiant alphanumérique permettant de faire le lien entre les sites et leurs visites si aucun UUID est fourni", "id_base_visit_origin": "Identifiant alphanumérique permettant de faire le lien entre les visites et leurs observations si aucun UUID est fourni", + "id_sites_group_origin": "Identifiant alphanumérique permettant de faire le lien entre les groupes de sites et leurs sites si aucun UUID est fourni", } -ENTITIES_NOT_AVAILABLE = ["sites_group", "observation_detail"] +ENTITIES_NOT_AVAILABLE = ["observation_detail", "individual", "marking"] # Valeur pouvant être retourné suite à l'analyse de la modification du protocole # cas de la mise à jour d'un module diff --git a/backend/gn_module_monitoring/command/imports/entity.py b/backend/gn_module_monitoring/command/imports/entity.py index 1f45e1774..e8eec424f 100644 --- a/backend/gn_module_monitoring/command/imports/entity.py +++ b/backend/gn_module_monitoring/command/imports/entity.py @@ -32,6 +32,17 @@ def get_entities_protocol(module_code: str) -> list: list Liste des entités du module. """ + + entities_order = [ + "sites_group", + "site", + "visit", + "observation", + "observation_detail", # Pas encore géré par l'import + "individual", # Pas encore géré par l'import + "marking", # Pas encore géré par l'import + ] + module_path = monitoring_module_config_path(module_code) if not (module_path / "config.json").is_file(): @@ -40,7 +51,9 @@ def get_entities_protocol(module_code: str) -> list: data_config = json_from_file(module_path / "config.json") tree = data_config.get("tree", {}).get("module", {}) keys = extract_keys(tree) - return [key for key in list(dict.fromkeys(keys)) if key not in ENTITIES_NOT_AVAILABLE] + unique_keys = [key for key in list(dict.fromkeys(keys)) if key not in ENTITIES_NOT_AVAILABLE] + + return [entity for entity in entities_order if entity in unique_keys] def get_entity_parent(tree: dict, entity_code: str): @@ -58,6 +71,10 @@ def find_parent(node, target, parent=None): return found return None + # Handle protocol which have sites and sites groups at the same level in config + if entity_code == "site" and "site" in tree and "sites_group" in tree: + return "sites_group" + parent_entity = find_parent(tree, entity_code) return parent_entity @@ -114,6 +131,7 @@ def insert_entities( "obs_detail" if entity_code == "observation_detail" else entity_code ) mapping_entity_object_code = { + "sites_group": "MONITORINGS_GRP_SITES", "site": "MONITORINGS_SITES", "visit": "MONITORINGS_VISITES", "observation": "MONITORINGS_OBSERVATIONS", @@ -248,47 +266,43 @@ def insert_entity_field_relations( bib_themes=bib_themes, is_parent_link=True, ) - get_cor_entity_field( - entity_id=entity_id, - field_name=f"uuid_base_{parent_code}", - id_destination=id_destination, - bib_themes=bib_themes, - is_parent_link=True, - ) - get_cor_entity_field( - entity_id=entity_id, - field_name=f"id_base_{parent_code}_origin", - id_destination=id_destination, - bib_themes=bib_themes, - is_parent_link=True, - ) + if parent_code == "sites_group": + get_cor_entity_field( + entity_id=entity_id, + field_name=f"uuid_{parent_code}", + id_destination=id_destination, + bib_themes=bib_themes, + is_parent_link=True, + ) + get_cor_entity_field( + entity_id=entity_id, + field_name=f"id_{parent_code}_origin", + id_destination=id_destination, + bib_themes=bib_themes, + is_parent_link=True, + ) + else: + get_cor_entity_field( + entity_id=entity_id, + field_name=f"uuid_base_{parent_code}", + id_destination=id_destination, + bib_themes=bib_themes, + is_parent_link=True, + ) + get_cor_entity_field( + entity_id=entity_id, + field_name=f"id_base_{parent_code}_origin", + id_destination=id_destination, + bib_themes=bib_themes, + is_parent_link=True, + ) def get_cor_entity_field( entity_id, field_name, id_destination, bib_themes, order=None, is_parent_link=False ): """ - Crée une relation entre une entité et un champ dans cor_entity_field. - - Parameters - ---------- - entity_id : int - ID de l'entité - field_name : str - Nom du champ - id_destination : int - ID de la destination (table) où chercher les IDs des entités - bib_themes : dict - Dictionnaire contenant les thèmes - order : int - Ordre du champ dans la hiérarchie des entités - is_parent_link : bool - Indique si le champ est une relation parent - - Returns - ------- - bool - True si la relation a été créée, False sinon + Crée une relation entre une entité et un champ dans cor_entity_field """ id_field = DB.session.execute( diff --git a/backend/gn_module_monitoring/command/imports/fields.py b/backend/gn_module_monitoring/command/imports/fields.py index ff593d40f..80a162784 100644 --- a/backend/gn_module_monitoring/command/imports/fields.py +++ b/backend/gn_module_monitoring/command/imports/fields.py @@ -54,21 +54,24 @@ def prepare_fields( ignored_fields = [ "id_module", # handled manually + "modules", # handled manually (cor_sites_group_module rempli avec le module de la destination) "id_dataset", # handled manually "id_digitiser", # handled manually + "uuid_sites_group", # handled manually "uuid_base_visit", # handled manually "uuid_base_site", # handled manually "uuid_observation", # handled manually + "id_sites_group", # generated "id_base_site", # generated "id_base_visit", # generated "id_observation", # generated "id_nomenclature_type_site", # handled manually "last_visit", # dynamic + "nb_sites", # dynamic "nb_visits", # dynamic "nb_observations", # dynamic "nb_individuals", # dynamic "medias", # not importable - "id_sites_group", # ignored for now ] field_set_manually = [] diff --git a/backend/gn_module_monitoring/command/imports/protocol.py b/backend/gn_module_monitoring/command/imports/protocol.py index 02cfcf699..370c74b6c 100644 --- a/backend/gn_module_monitoring/command/imports/protocol.py +++ b/backend/gn_module_monitoring/command/imports/protocol.py @@ -91,7 +91,7 @@ def get_protocol_data(module_code: str, id_destination: int): "generic_data": generic_data, } - # Now we can iterate safetly over confs + # Now we can iterate safely over confs for entity_code in entity_confs: entity_conf = entity_confs[entity_code] parent_entity = get_entity_parent(tree, entity_code) @@ -129,16 +129,15 @@ def get_protocol_data(module_code: str, id_destination: int): [ { "name_field": name_field, - "fr_label": "Géometrie (WKT)", + "fr_label": "Géométrie (WKT)", "type_field": "textarea", "type_column": "text", - "mandatory": True, + "mandatory": False, "autogenerated": False, "display": True, "source_field": f"src_{name_field}", "multi": False, "id_destination": id_destination, - "optional_conditions": ["x", "y"], }, { "name_field": f"{name_field}_4326", @@ -164,39 +163,51 @@ def get_protocol_data(module_code: str, id_destination: int): "multi": False, "id_destination": id_destination, }, - { - "name_field": "x", - "fr_label": "Longitude (x)", - "type_field": "number", - "type_column": "number", - "mandatory": True, - "autogenerated": False, - "display": True, - "source_field": "src_x", - "multi": False, - "id_destination": id_destination, - "optional_conditions": [name_field], - }, - { - "name_field": "y", - "fr_label": "Latitude (y)", - "type_field": "number", - "type_column": "number", - "mandatory": True, - "autogenerated": False, - "display": True, - "source_field": "src_y", - "multi": False, - "id_destination": id_destination, - "optional_conditions": [name_field], - }, ] ) - for name in ["site", "visit", "observation"]: + + if entity_code == "site": + for dict in protocol_data["site"]["generic"]: + if dict["name_field"] == name_field: + dict["mandatory"] = True + dict["optional_conditions"] = ["x", "y"] + + protocol_data["site"]["generic"].extend( + [ + { + "name_field": "x", + "fr_label": "Longitude (x)", + "type_field": "number", + "type_column": "number", + "mandatory": True, + "autogenerated": False, + "display": True, + "source_field": "src_x", + "multi": False, + "id_destination": id_destination, + "optional_conditions": [name_field], + }, + { + "name_field": "y", + "fr_label": "Latitude (y)", + "type_field": "number", + "type_column": "number", + "mandatory": True, + "autogenerated": False, + "display": True, + "source_field": "src_y", + "multi": False, + "id_destination": id_destination, + "optional_conditions": [name_field], + }, + ] + ) + + for name in ["sites_group", "site", "visit", "observation"]: label_entity = f"{name.capitalize()}e" if name == "visit" else f"{name.capitalize()}" if name not in entities: continue - suffix = name if name == "observation" else f"base_{name}" + suffix = name if name in ["sites_group", "observation"] else f"base_{name}" protocol_data[name]["generic"].extend( [ { @@ -271,7 +282,7 @@ def get_protocol_data(module_code: str, id_destination: int): ] ) - # Add observation_detail the file exists + # Add observation_detail if the file exists if "observation_detail" in entities: observation_detail_specific_path = module_config_dir_path / "observation_detail.json" observation_detail_generic_path = os.path.join( diff --git a/backend/gn_module_monitoring/command/utils.py b/backend/gn_module_monitoring/command/utils.py index 6ad37f4c8..d944d077a 100644 --- a/backend/gn_module_monitoring/command/utils.py +++ b/backend/gn_module_monitoring/command/utils.py @@ -148,15 +148,14 @@ def validate_json_file_protocol(module_code: str): try: entities = get_entities_protocol(module_code) for entity_code in entities: - if not entity_code == "sites_group": - # Valid specific file - specific_path = module_config_dir / f"{entity_code}.json" - errors.extend(validate_json_file(specific_path, valid_type_widgets)) - - # Valid generic file - project_root = Path(__file__).parent.parent - generic_path = project_root / "config" / "generic" / f"{entity_code}.json" - errors.extend(validate_json_file(generic_path, valid_type_widgets)) + # Valid specific file + specific_path = module_config_dir / f"{entity_code}.json" + errors.extend(validate_json_file(specific_path, valid_type_widgets)) + + # Valid generic file + project_root = Path(__file__).parent.parent + generic_path = project_root / "config" / "generic" / f"{entity_code}.json" + errors.extend(validate_json_file(generic_path, valid_type_widgets)) except Exception as e: errors.append(f"Erreur lors de la lecture des entités: {str(e)}") diff --git a/backend/gn_module_monitoring/config/generic/sites_group.json b/backend/gn_module_monitoring/config/generic/sites_group.json index ec06a72df..3aee86515 100644 --- a/backend/gn_module_monitoring/config/generic/sites_group.json +++ b/backend/gn_module_monitoring/config/generic/sites_group.json @@ -65,11 +65,11 @@ "schema_dot_table": "gn_monitoring.t_sites_groups" }, "altitude_min": { - "type_widget": "integer", + "type_widget": "number", "attribut_label": "Altitude (min)" }, "altitude_max": { - "type_widget": "integer", + "type_widget": "number", "attribut_label": "Altitude (max)" }, "id_digitiser": { diff --git a/backend/gn_module_monitoring/migrations/974hpr70u7hd_add_id_import_column_for_sites_groups.py b/backend/gn_module_monitoring/migrations/974hpr70u7hd_add_id_import_column_for_sites_groups.py new file mode 100644 index 000000000..6da9b16ba --- /dev/null +++ b/backend/gn_module_monitoring/migrations/974hpr70u7hd_add_id_import_column_for_sites_groups.py @@ -0,0 +1,36 @@ +"""[import] add id_import column for sites groups + +Revision ID: 974hpr70u7hd +Revises: 461b82ee737a +Create Date: 2026-02-11 16:00:05.425708 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "974hpr70u7hd" +down_revision = "3d39820c9ab7" +branch_labels = None +depends_on = None + +import_column_name = "id_import" +schema = "gn_monitoring" +table = "t_sites_groups" + + +def upgrade(): + op.add_column( + schema=schema, + table_name=table, + column=sa.Column(import_column_name, sa.Integer, nullable=True), + ) + + +def downgrade(): + op.drop_column( + schema=schema, + table_name=table, + column_name=import_column_name, + ) diff --git a/backend/gn_module_monitoring/monitoring/import_actions/actions.py b/backend/gn_module_monitoring/monitoring/import_actions/actions.py index 142ab6a93..527d2c32d 100644 --- a/backend/gn_module_monitoring/monitoring/import_actions/actions.py +++ b/backend/gn_module_monitoring/monitoring/import_actions/actions.py @@ -9,12 +9,16 @@ ) from gn_module_monitoring.monitoring.models import ( - TMonitoringObservations, + TMonitoringSitesGroups, TMonitoringSites, TMonitoringVisits, + TMonitoringObservations, + cor_sites_group_module, ) import sqlalchemy as sa from sqlalchemy.orm import aliased, joinedload +from sqlalchemy.inspection import inspect +from werkzeug.exceptions import Conflict from geonature.core.imports.actions import ImportActions, ImportStatisticsLabels from geonature.core.imports.checks.sql.core import check_orphan_rows, init_rows_validity from geonature.core.imports.models import Entity, TImports @@ -30,6 +34,9 @@ from .entity_import_actions_utils import EntityImportActionsUtils +from gn_module_monitoring.monitoring.import_actions.sites_group_actions import ( + SitesGroupImportActions, +) from gn_module_monitoring.monitoring.import_actions.site_actions import SiteImportActions from gn_module_monitoring.monitoring.import_actions.visit_actions import VisitImportActions from gn_module_monitoring.monitoring.import_actions.observation_actions import ( @@ -48,6 +55,8 @@ def get_entities(imprt: TImports) -> typing.Tuple[Entity, Entity, Entity]: def get_entity_model(entity: Entity): + if entity.code == "sites_group": + return TMonitoringSitesGroups if entity.code == "site": return TBaseSites elif entity.code == "visit": @@ -74,6 +83,7 @@ class MonitoringImportActions(ImportActions): @staticmethod def statistics_labels() -> typing.List[ImportStatisticsLabels]: return [ + {"key": "sites_group_count", "value": "Nombre de groupes de sites importés"}, {"key": "site_count", "value": "Nombre de sites importés"}, {"key": "visit_count", "value": "Nombre de visites importées"}, {"key": "observation_count", "value": "Nombre d'observations importées"}, @@ -120,6 +130,14 @@ def check_transient_data(task, logger, imprt: TImports) -> None: config = get_config(imprt.destination.code) + isSitesGroup = EntityImportActionsUtils.is_entity_defined_in_import( + imprt, SitesGroupImportActions.ENTITY_CODE + ) + isSitesGroupMandatory = False + if isSitesGroup: + # Some protocols make sites group optionals by setting a site entity at the same level than sites group in tree config + isSitesGroupMandatory = "site" not in config["tree"]["module"] + isVisit = EntityImportActionsUtils.is_entity_defined_in_import( imprt, VisitImportActions.ENTITY_CODE ) @@ -135,19 +153,25 @@ def check_transient_data(task, logger, imprt: TImports) -> None: # We first check site and visit consistency in order to avoid checking # incoherent data + if isSitesGroup: + SitesGroupImportActions.check_entity_data_consistency(imprt) SiteImportActions.check_entity_data_consistency(imprt) if isVisit: VisitImportActions.check_entity_data_consistency(imprt) # We run dataframes checks before SQL checks in order to avoid # check_types overriding generated values during SQL checks. + if isSitesGroup: + SitesGroupImportActions.check_dataframe(imprt, config) SiteImportActions.check_dataframe(imprt, config) if isVisit: VisitImportActions.check_dataframe(imprt) if isObservation: ObservationImportActions.check_dataframe(imprt) - SiteImportActions.check_sql(imprt) + if isSitesGroup: + SitesGroupImportActions.check_sql(imprt) + SiteImportActions.check_sql(imprt, isSitesGroup, isSitesGroupMandatory) if isVisit: VisitImportActions.check_sql(imprt) if isObservation: @@ -155,6 +179,9 @@ def check_transient_data(task, logger, imprt: TImports) -> None: @staticmethod def import_data_to_destination(imprt: TImports) -> None: + isSitesGroup = EntityImportActionsUtils.is_entity_defined_in_import( + imprt, SitesGroupImportActions.ENTITY_CODE + ) isVisit = EntityImportActionsUtils.is_entity_defined_in_import( imprt, VisitImportActions.ENTITY_CODE ) @@ -176,12 +203,17 @@ def import_data_to_destination(imprt: TImports) -> None: .all() ) } + + if isSitesGroup: + SitesGroupImportActions.generate_id(imprt) SiteImportActions.generate_id(imprt) if isVisit: VisitImportActions.generate_id(imprt) if isObservation: ObservationImportActions.generate_id(imprt) + if isSitesGroup: + SiteImportActions.set_parent_id_from_line_no(imprt) if isVisit: VisitImportActions.set_parent_id_from_line_no(imprt) if isObservation: @@ -190,7 +222,9 @@ def import_data_to_destination(imprt: TImports) -> None: for entity in entities.values(): print(f"--------- {entity.code}") - entity_fields = EntityImportActionsUtils.get_destination_fields(imprt, entity) + entity_fields = EntityImportActionsUtils.get_destination_fields( + imprt, entity, isSitesGroup + ) core_dest_col_names = ["id_import", "id_digitiser"] core_select_cols = [ @@ -234,6 +268,20 @@ def import_data_to_destination(imprt: TImports) -> None: core_dest_col_names.append("id_module") core_select_cols.append(sa.literal(imprt.destination.id_module).label("id_module")) + # sites_group n'a pas de table complément séparée : ses champs spécifiques + # sont stockés dans sa propre colonne data (parité avec site/visit/observation). + if entity.code == "sites_group" and complement_fields: + sg_json_args = [] + for field in complement_fields: + sg_json_args.extend( + [ + EntityImportActionsUtils.get_destination_column_name(field.dest_field), + transient_table.c[field.dest_field], + ] + ) + core_dest_col_names.append("data") + core_select_cols.append(sa.func.json_build_object(*sg_json_args).label("data")) + core_select_stmt = ( sa.select(*core_select_cols) .where(transient_table.c.id_import == imprt.id_import) @@ -250,13 +298,19 @@ def import_data_to_destination(imprt: TImports) -> None: id_col_name = f"id_base_{entity.code}" json_args = [] + # Fields not to insert in data column + SITES_GROUP_ID_FIELDS = ["id_sites_group", "uuid_sites_group"] + is_sites_group_id_fields = False for field in complement_fields: - json_args.extend( - [ - EntityImportActionsUtils.get_destination_column_name(field.dest_field), - transient_table.c[field.dest_field], - ] - ) + if field.name_field not in SITES_GROUP_ID_FIELDS: + json_args.extend( + [ + EntityImportActionsUtils.get_destination_column_name(field.dest_field), + transient_table.c[field.dest_field], + ] + ) + else: + is_sites_group_id_fields = True complement_select_stmt = None model_complements = get_entity_model_complements(entity) @@ -266,14 +320,22 @@ def import_data_to_destination(imprt: TImports) -> None: cols = [sa.func.json_build_object(*json_args).label("data")] if entity.code != "observation": cols.insert(0, transient_table.c[id_col_name]) - complement_select_stmt = ( - sa.select(*cols) - .where(transient_table.c.id_import == imprt.id_import) - .where(transient_table.c[entity.validity_column] == True) - .order_by( - transient_table.c.line_no - ) # Required for the process of inserting observation complements - ) + if entity.code == "site" and is_sites_group_id_fields: + complement_select_stmt = ( + sa.select(*cols, transient_table.c["id_sites_group"]) + .where(transient_table.c.id_import == imprt.id_import) + .where(transient_table.c[entity.validity_column] == True) + .order_by(transient_table.c.line_no) + ) + else: + complement_select_stmt = ( + sa.select(*cols) + .where(transient_table.c.id_import == imprt.id_import) + .where(transient_table.c[entity.validity_column] == True) + .order_by( + transient_table.c.line_no + ) # Required for the process of inserting observation complements + ) types_site_select_stmt = None if entity.code == "site": @@ -286,6 +348,17 @@ def import_data_to_destination(imprt: TImports) -> None: .where(transient_table.c[entity.validity_column] == True) ) + cor_sites_group_module_select = None + if entity.code == "sites_group": + cor_sites_group_module_select = ( + sa.select( + transient_table.c["id_sites_group"], + sa.literal(imprt.destination.id_module).label("id_module"), + ) + .where(transient_table.c.id_import == imprt.id_import) + .where(transient_table.c[entity.validity_column] == True) + ) + batch_size = current_app.config["IMPORT"]["INSERT_BATCH_SIZE"] batch_count = ceil(imprt.source_count / batch_size) row_count = 0 @@ -350,17 +423,39 @@ def import_data_to_destination(imprt: TImports) -> None: ) row_count += db.session.execute(core_insert_stmt).rowcount - if complement_select_stmt is not None: + if cor_sites_group_module_select is not None: db.session.execute( - sa.insert(model_complements).from_select( - names=[id_col_name, "data"], - select=complement_select_stmt.filter( + sa.insert(cor_sites_group_module).from_select( + ["id_sites_group", "id_module"], + cor_sites_group_module_select.filter( transient_table.c["line_no"] >= min_line_no, transient_table.c["line_no"] < max_line_no, ), ) ) + if complement_select_stmt is not None: + if entity.code == "site" and is_sites_group_id_fields: + db.session.execute( + sa.insert(model_complements).from_select( + names=[id_col_name, "data", "id_sites_group"], + select=complement_select_stmt.filter( + transient_table.c["line_no"] >= min_line_no, + transient_table.c["line_no"] < max_line_no, + ), + ) + ) + else: + db.session.execute( + sa.insert(model_complements).from_select( + names=[id_col_name, "data"], + select=complement_select_stmt.filter( + transient_table.c["line_no"] >= min_line_no, + transient_table.c["line_no"] < max_line_no, + ), + ) + ) + if types_site_select_stmt is not None: db.session.execute( sa.insert(cor_site_type).from_select( @@ -415,12 +510,87 @@ def import_data_to_destination(imprt: TImports) -> None: for key in statistics_0_keys: imprt.statistics.pop(key) + @staticmethod + def remove_data_from_destination(imprt: TImports): + """ + Remove data from destination database for a given import. + + Parameters + ---------- + imprt : TImports + The import to remove data from. + + Notes + ----- + This method is called when an import is deleted. + It removes from the destination database all data that was created + by the import. + + If a child entity (e.g. Habitat) was created later on an imported + parent entity (e.g. Station), deleting the imported entity will + be refused ! + """ + entities = db.session.scalars( + sa.select(Entity) + .where(Entity.destination == imprt.destination) + .order_by(sa.desc(Entity.order)) + ).all() + for entity in entities: + parent_table = entity.get_destination_table() + if entity.childs: + for child in entity.childs: + child_table = child.get_destination_table() + (parent_pk,) = inspect(parent_table).primary_key.columns + (child_pk,) = inspect(child_table).primary_key.columns + # Looking for parent rows belonging to this import with child rows + # not belonging to this import. + # We use is_distinct_from to match rows with NULL id_import. + + if parent_table.name == "t_sites_groups": + query = ( + sa.select(parent_pk, sa.func.array_agg(child_pk)) + .select_from(parent_table.join(TMonitoringSites)) + .where( + parent_table.c.id_import == imprt.id_import, + TMonitoringSites.id_import.is_distinct_from(imprt.id_import), + ) + .group_by(parent_pk) + ) + else: + query = ( + sa.select(parent_pk, sa.func.array_agg(child_pk)) + .select_from(parent_table.join(child_table)) + .where( + parent_table.c.id_import == imprt.id_import, + child_table.c.id_import.is_distinct_from(imprt.id_import), + ) + .group_by(parent_pk) + ) + orphans = db.session.execute(query).fetchall() + if orphans: + description = "L’import ne peut pas être supprimé car cela provoquerait la suppression de données ne provenant pas de cet import :" + description += "" + raise Conflict(description) + db.session.execute( + sa.delete(parent_table).where(parent_table.c.id_import == imprt.id_import) + ) + @staticmethod def report_plot(imprt: TImports) -> StandaloneEmbedJson: return None @staticmethod def compute_bounding_box(imprt: TImports): + isSitesGroup = EntityImportActionsUtils.is_entity_defined_in_import( + imprt, SitesGroupImportActions.ENTITY_CODE + ) + # Problem with bounding box: the field doesn't have the same name between the transient table and the destination table # It might be the problem - return SiteImportActions.compute_bounding_box(imprt) + if isSitesGroup: + return SitesGroupImportActions.compute_bounding_box(imprt) + else: + return SiteImportActions.compute_bounding_box(imprt) diff --git a/backend/gn_module_monitoring/monitoring/import_actions/entity_import_actions_utils.py b/backend/gn_module_monitoring/monitoring/import_actions/entity_import_actions_utils.py index d957b2300..81839cffb 100644 --- a/backend/gn_module_monitoring/monitoring/import_actions/entity_import_actions_utils.py +++ b/backend/gn_module_monitoring/monitoring/import_actions/entity_import_actions_utils.py @@ -28,7 +28,7 @@ def dataframe_checks(imprt, df, entity, fields): return updated_cols @staticmethod - def get_destination_fields(imprt: TImports, entity: Entity) -> None: + def get_destination_fields(imprt: TImports, entity: Entity, isSitesGroups) -> None: fields = { ef.field.name_field: ef.field for ef in entity.fields if ef.field.dest_field != None } @@ -46,7 +46,18 @@ def get_destination_fields(imprt: TImports, entity: Entity) -> None: if column_src in imprt.columns or mapping.get("constant_value", None) is not None: entity_fields |= {field} - if entity.code == "site": + if entity.code == "sites_group": + entity_fields |= { + fields["id_sites_group"], + fields["g__geom_4326"], + fields["g__geom_local"], + fields["uuid_sites_group"], + } + elif entity.code == "site": + if isSitesGroups: + entity_fields |= { + fields["id_sites_group"], + } entity_fields |= { fields["id_base_site"], fields["s__geom_4326"], diff --git a/backend/gn_module_monitoring/monitoring/import_actions/observation_actions.py b/backend/gn_module_monitoring/monitoring/import_actions/observation_actions.py index 16daf1da6..3563beaea 100644 --- a/backend/gn_module_monitoring/monitoring/import_actions/observation_actions.py +++ b/backend/gn_module_monitoring/monitoring/import_actions/observation_actions.py @@ -2,13 +2,14 @@ from .entity_import_actions_utils import EntityImportActionsUtils from geonature.core.imports.checks.sql.parent import set_parent_line_no -from geonature.core.imports.models import TImports +from geonature.core.imports.models import Entity, TImports from geonature.core.gn_monitoring.models import TObservations from geonature.utils.env import db import sqlalchemy as sa from geonature.core.imports.checks.sql.extra import ( + check_entity_data_consistency, disable_duplicated_rows, generate_entity_id, generate_missing_uuid, diff --git a/backend/gn_module_monitoring/monitoring/import_actions/site_actions.py b/backend/gn_module_monitoring/monitoring/import_actions/site_actions.py index d1addfa64..944476774 100644 --- a/backend/gn_module_monitoring/monitoring/import_actions/site_actions.py +++ b/backend/gn_module_monitoring/monitoring/import_actions/site_actions.py @@ -1,23 +1,33 @@ +import sqlalchemy as sa from flask import current_app from .entity_import_actions_utils import EntityImportActionsUtils from geonature.core.imports.models import TImports +from geonature.core.imports.checks.errors import ImportCodeError +from geonature.core.imports.checks.sql.utils import report_erroneous_rows +from geonature.core.imports.checks.sql.parent import ( + set_parent_line_no, + check_no_parent_entity, + set_id_parent_from_destination, + check_erroneous_parent_entities, +) + from geonature.core.imports.checks.sql.extra import ( check_entity_data_consistency, disable_duplicated_rows, generate_entity_id, generate_missing_uuid, -) - -from geonature.core.imports.checks.sql import ( - check_altitudes, + generate_missing_uuid_for_id_origin, check_duplicate_uuid, check_existing_uuid, - convert_geom_columns, - do_nomenclatures_mapping, - generate_missing_uuid_for_id_origin, + check_altitudes, + set_parent_id_from_line_no, ) + +from geonature.core.imports.checks.sql.geo import convert_geom_columns +from geonature.core.imports.checks.sql.nomenclature import do_nomenclatures_mapping + from geonature.core.imports.utils import ( get_mapping_data, load_transient_data_in_dataframe, @@ -42,11 +52,18 @@ class SiteImportActions: GEOMETRY_LOCAL_FIELD = "s__geom_local" ALTITUDE_MIN_FIELD = "s__altitude_min" ALTITUDE_MAX_FIELD = "s__altitude_max" + PARENT_ID_FIELD = "id_sites_group" + PARENT_UUID_FIELD = "uuid_sites_group" + PARENT_LINE_NO = "sites_group_line_no" LINE_NO = "site_line_no" ID_INVENTOR_FIELD = "s__id_inventor" @staticmethod - def check_sql(imprt: TImports): + def check_sql(imprt: TImports, isSitesGroups, isSitesGroupMandatory): + from gn_module_monitoring.monitoring.import_actions.sites_group_actions import ( + SitesGroupImportActions, + ) + entity = EntityImportActionsUtils.get_entity(imprt, SiteImportActions.ENTITY_CODE) entity_fields, fieldmapped_fields, _ = get_mapping_data(imprt, entity) @@ -101,6 +118,32 @@ def check_sql(imprt: TImports): do_nomenclatures_mapping(imprt, entity, fieldmapped_fields, fill_with_defaults=False) + ## process parent uuid and id only if the module accepts sites groups + + if isSitesGroups: + set_id_parent_from_destination( + imprt, + parent_entity=entity.parent, + entity=entity, + id_field=entity_fields.get(SitesGroupImportActions.ID_FIELD), + fields=[ + entity_fields.get(SitesGroupImportActions.UUID_FIELD), + ], + ) + + # Wire parent child + set_parent_line_no( + imprt, + parent_entity=entity.parent, + entity=entity, + parent_line_no=SitesGroupImportActions.LINE_NO, + fields=[ + entity_fields.get(SitesGroupImportActions.ID_ORIGIN_FIELD), + entity_fields.get(SitesGroupImportActions.UUID_FIELD), + ], + ) + SiteImportActions.check_parent_validity(imprt, isSitesGroupMandatory) + if SiteImportActions.ID_INVENTOR_FIELD in fieldmapped_fields: map_observer_matching(imprt, entity, fieldmapped_fields["s__id_inventor"]) @@ -166,6 +209,20 @@ def generate_id(imprt: TImports): "id_base_site", ) + @staticmethod + def set_parent_id_from_line_no(imprt: TImports): + from gn_module_monitoring.monitoring.import_actions.sites_group_actions import ( + SitesGroupImportActions, + ) + + entity = EntityImportActionsUtils.get_entity(imprt, SiteImportActions.ENTITY_CODE) + set_parent_id_from_line_no( + imprt, + entity=entity, + parent_line_no_field_name=SitesGroupImportActions.LINE_NO, + parent_id_field_name=SitesGroupImportActions.ID_FIELD, + ) + @staticmethod def check_entity_data_consistency(imprt: TImports): entity = EntityImportActionsUtils.get_entity(imprt, SiteImportActions.ENTITY_CODE) @@ -240,3 +297,61 @@ def check_altitudes(imprt: TImports): fields[SiteImportActions.ALTITUDE_MIN_FIELD], fields[SiteImportActions.ALTITUDE_MAX_FIELD], ) + + @staticmethod + def check_parent_validity(imprt: TImports, isSitesGroupMandatory: bool): + from gn_module_monitoring.monitoring.import_actions.sites_group_actions import ( + SitesGroupImportActions, + ) + + entity_site = EntityImportActionsUtils.get_entity(imprt, SiteImportActions.ENTITY_CODE) + entity_sites_group = EntityImportActionsUtils.get_entity( + imprt, SitesGroupImportActions.ENTITY_CODE + ) + + # A site referencing a sites group (by UUID or origin identifier) that could + # be resolved neither in the destination nor on another line of the file is + # erroneous, even when the sites group is optional: the provided reference + # would otherwise be silently ignored. + transient_table = imprt.destination.get_transient_table() + entity_fields, _, _ = get_mapping_data(imprt, entity_site) + for ref_field_name in ( + SitesGroupImportActions.UUID_FIELD, + SitesGroupImportActions.ID_ORIGIN_FIELD, + ): + ref_field = entity_fields.get(ref_field_name) + if ref_field is None: + continue + report_erroneous_rows( + imprt, + entity_site, + error_type=ImportCodeError.NO_PARENT_ENTITY, + error_column=ref_field_name, + whereclause=sa.and_( + transient_table.c[entity_site.validity_column].is_(True), + # no sites group defined on the same line... + transient_table.c[entity_sites_group.validity_column].is_(None), + # ...the reference was not found in the destination... + transient_table.c[SitesGroupImportActions.ID_FIELD].is_(None), + # ...nor on another line of the file... + transient_table.c[SitesGroupImportActions.LINE_NO].is_(None), + # ...although a reference was provided + transient_table.c[ref_field.source_field].isnot(None), + ), + ) + + if isSitesGroupMandatory: + check_no_parent_entity( + imprt, + parent_entity=entity_sites_group, + entity=entity_site, + id_parent=SitesGroupImportActions.ID_FIELD, + parent_line_no=SitesGroupImportActions.LINE_NO, + ) + + check_erroneous_parent_entities( + imprt, + parent_entity=entity_sites_group, + entity=entity_site, + parent_line_no=SitesGroupImportActions.LINE_NO, + ) diff --git a/backend/gn_module_monitoring/monitoring/import_actions/sites_group_actions.py b/backend/gn_module_monitoring/monitoring/import_actions/sites_group_actions.py new file mode 100644 index 000000000..c4a82b237 --- /dev/null +++ b/backend/gn_module_monitoring/monitoring/import_actions/sites_group_actions.py @@ -0,0 +1,253 @@ +from .entity_import_actions_utils import EntityImportActionsUtils + +from geonature.core.imports.models import TImports + +from geonature.core.imports.checks.sql.extra import ( + check_entity_data_consistency, + disable_duplicated_rows, + generate_entity_id, + generate_missing_uuid, + generate_missing_uuid_for_id_origin, + check_duplicate_uuid, + check_existing_uuid, + check_altitudes, +) + +from geonature.core.imports.checks.sql.geo import convert_geom_columns +from geonature.core.imports.checks.sql.nomenclature import do_nomenclatures_mapping + +from geonature.core.imports.utils import ( + get_mapping_data, + load_transient_data_in_dataframe, + update_transient_data_from_dataframe, + compute_bounding_box, +) + +from geonature.core.imports.checks.dataframe.geometry import check_geometry + +from geonature.core.imports.checks.sql.user import map_observer_matching + + +class SitesGroupImportActions: + ENTITY_CODE = "sites_group" + TABLE_NAME = "t_sites_groups" + ID_FIELD = "id_sites_group" + ID_ORIGIN_FIELD = "id_sites_group_origin" + LINE_NO = "sites_group_line_no" + UUID_FIELD = "uuid_sites_group" + GEOMETRY_FIELD = "g__geom" + GEOMETRY_4326_FIELD = "g__geom_4326" + GEOMETRY_LOCAL_FIELD = "g__geom_local" + ALTITUDE_MIN_FIELD = "g__altitude_min" + ALTITUDE_MAX_FIELD = "g__altitude_max" + + @staticmethod + def check_sql(imprt: TImports): + entity = EntityImportActionsUtils.get_entity(imprt, SitesGroupImportActions.ENTITY_CODE) + entity_fields, fieldmapped_fields, _ = get_mapping_data(imprt, entity) + + if SitesGroupImportActions.UUID_FIELD in fieldmapped_fields: + uuid_field = fieldmapped_fields.get(SitesGroupImportActions.UUID_FIELD) + # Check existing uuid + check_existing_uuid( + imprt, + entity, + uuid_field, + skip=True, # TODO config + ) + + # Disable duplicated definition row + disable_duplicated_rows( + imprt, + entity, + fieldmapped_fields, + uuid_field, + ) + + # Check duplicate uuid + check_duplicate_uuid(imprt, entity, uuid_field) + + if SitesGroupImportActions.ID_ORIGIN_FIELD in fieldmapped_fields: + generate_missing_uuid_for_id_origin( + imprt, + entity_fields.get(SitesGroupImportActions.UUID_FIELD), + entity_fields.get(SitesGroupImportActions.ID_ORIGIN_FIELD), + ) + generate_missing_uuid( + imprt, + entity, + entity_fields.get(SitesGroupImportActions.UUID_FIELD), + whereclause=None, + ) + + if SitesGroupImportActions.ID_ORIGIN_FIELD in fieldmapped_fields: + disable_duplicated_rows( + imprt, + entity, + fieldmapped_fields, + entity_fields.get(SitesGroupImportActions.ID_ORIGIN_FIELD), + ) + + SitesGroupImportActions.check_and_compute_geometries(imprt) + + SitesGroupImportActions.check_altitudes(imprt) + + do_nomenclatures_mapping(imprt, entity, fieldmapped_fields, fill_with_defaults=False) + + @staticmethod + def check_dataframe(imprt: TImports, config): + """ + Check the site group data before importing. + + List of checks and data operations (in order of execution): + - check types + - check required values + - convert geom columns + - check geography + - check if given geometries are valid (see ST_VALID in PostGIS) + + Parameters + ---------- + imprt : TImports + The import to check. + + """ + entity = EntityImportActionsUtils.get_entity(imprt, SitesGroupImportActions.ENTITY_CODE) + + entity_fields, _, source_cols = get_mapping_data(imprt, entity) + + # Save column names where the data was changed in the dataframe + updated_cols = set() + + ### Dataframe checks + df = load_transient_data_in_dataframe(imprt, entity, source_cols) + + updated_cols |= EntityImportActionsUtils.dataframe_checks(imprt, df, entity, entity_fields) + + geom_field_name = config.get(SitesGroupImportActions.ENTITY_CODE, {}).get( + "geom_field_name" + ) + if geom_field_name: + geom_field_name__local = f"g__{geom_field_name}_local" + geom_field_name__4326 = f"g__{geom_field_name}_4326" + geom_field_name__wkt = f"g__{geom_field_name}" + updated_cols |= check_geometry( + imprt, + entity, + df, + file_srid=imprt.srid, + geom_4326_field=entity_fields[geom_field_name__4326], + geom_local_field=entity_fields[geom_field_name__local], + wkt_field=entity_fields[geom_field_name__wkt], + ) + + update_transient_data_from_dataframe(imprt, entity, updated_cols, df) + + @staticmethod + def generate_id(imprt: TImports): + entity = EntityImportActionsUtils.get_entity(imprt, SitesGroupImportActions.ENTITY_CODE) + generate_entity_id( + imprt, + entity, + "gn_monitoring", + "t_sites_groups", + "uuid_sites_group", + "id_sites_group", + ) + + @staticmethod + def check_entity_data_consistency(imprt: TImports): + entity = EntityImportActionsUtils.get_entity(imprt, SitesGroupImportActions.ENTITY_CODE) + + _, fieldmapped_fields, _ = get_mapping_data(imprt, entity) + + if SitesGroupImportActions.ID_FIELD in fieldmapped_fields: + check_entity_data_consistency( + imprt, + entity, + fieldmapped_fields, + fieldmapped_fields.get(SitesGroupImportActions.ID_FIELD), + ) + if SitesGroupImportActions.UUID_FIELD in fieldmapped_fields: + check_entity_data_consistency( + imprt, + entity, + fieldmapped_fields, + fieldmapped_fields.get(SitesGroupImportActions.UUID_FIELD), + ) + + @staticmethod + def compute_bounding_box(imprt: TImports): + import geojson + from gn_module_monitoring.monitoring.import_actions.site_actions import SiteImportActions + + def get_bounding_box(points): + x_coordinates, y_coordinates = zip(*points) + + return [ + [ + (min(x_coordinates), max(y_coordinates)), + (min(x_coordinates), min(y_coordinates)), + (max(x_coordinates), min(y_coordinates)), + (max(x_coordinates), max(y_coordinates)), + (min(x_coordinates), max(y_coordinates)), + ] + ] + + # Problem with bounding box: the field doesn't have the same name between the transient table and the destination table + # It might be the problem + + # TMonitoringSites which has a foreign key with TMonitoringSitesGroups isn't part of GeoNature core + # thus can't be called from core function compute_bounding_box(). We aggregate two boxes instead. + sites_group_bounding_box = compute_bounding_box( + imprt=imprt, + geom_entity_code=SitesGroupImportActions.ENTITY_CODE, + geom_4326_field_name__transient=SitesGroupImportActions.GEOMETRY_FIELD, + geom_4326_field_name__destination=EntityImportActionsUtils.get_destination_column_name( + SitesGroupImportActions.GEOMETRY_FIELD + ), + ) + children_bounding_box = SiteImportActions.compute_bounding_box(imprt) + + if sites_group_bounding_box and children_bounding_box: + return geojson.Polygon( + get_bounding_box( + list( + geojson.utils.coords( + geojson.GeometryCollection( + [sites_group_bounding_box, children_bounding_box] + ) + ) + ) + ) + ) + + return sites_group_bounding_box or children_bounding_box + + @staticmethod + def check_and_compute_geometries(imprt: TImports): + entity_site = EntityImportActionsUtils.get_entity( + imprt, SitesGroupImportActions.ENTITY_CODE + ) + fields, _, _ = get_mapping_data(imprt, entity_site) + + convert_geom_columns( + imprt, + entity_site, + geom_4326_field=fields[SitesGroupImportActions.GEOMETRY_4326_FIELD], + geom_local_field=fields[SitesGroupImportActions.GEOMETRY_LOCAL_FIELD], + ) + + @staticmethod + def check_altitudes(imprt: TImports): + entity_sites_group = EntityImportActionsUtils.get_entity( + imprt, SitesGroupImportActions.ENTITY_CODE + ) + fields, _, _ = get_mapping_data(imprt, entity_sites_group) + + check_altitudes( + imprt, + entity_sites_group, + fields[SitesGroupImportActions.ALTITUDE_MIN_FIELD], + fields[SitesGroupImportActions.ALTITUDE_MAX_FIELD], + ) diff --git a/backend/gn_module_monitoring/monitoring/import_actions/visit_actions.py b/backend/gn_module_monitoring/monitoring/import_actions/visit_actions.py index 92407cc1f..25a924fda 100644 --- a/backend/gn_module_monitoring/monitoring/import_actions/visit_actions.py +++ b/backend/gn_module_monitoring/monitoring/import_actions/visit_actions.py @@ -27,7 +27,7 @@ from geonature.core.imports.models import Entity, TImports -from geonature.core.imports.checks.dataframe.core import check_datasets +from geonature.core.imports.checks.dataframe.core import check_datasets, check_required_values from geonature.utils.env import db diff --git a/backend/gn_module_monitoring/monitoring/models.py b/backend/gn_module_monitoring/monitoring/models.py index 5272ad095..99c031c8b 100644 --- a/backend/gn_module_monitoring/monitoring/models.py +++ b/backend/gn_module_monitoring/monitoring/models.py @@ -477,6 +477,8 @@ class TMonitoringSitesGroups(DB.Model, PermissionModel, SitesGroupsQuery): .scalar_subquery() ) + id_import = DB.Column(DB.Integer, nullable=True) + @hybrid_property def nb_visits(self): query = select(func.count(TMonitoringVisits.id_base_site)).where( diff --git a/backend/gn_module_monitoring/monitoring/serializer.py b/backend/gn_module_monitoring/monitoring/serializer.py index daa78d415..53d09e116 100644 --- a/backend/gn_module_monitoring/monitoring/serializer.py +++ b/backend/gn_module_monitoring/monitoring/serializer.py @@ -80,8 +80,8 @@ def flatten_specific_properties(self, properties, only=None): for attribut_name in self.config_schema(type_schema="specific"): if attribut_name in only or not only: properties[attribut_name] = data.get(attribut_name) - # Nécessaire pour récupérer tous les données des champs additionnels mêmes ceux non présents dans la config specific - # Nécessaire pour les récupérer coté frontend lors de l'envoie de l'objet (patch et post) + # Nécessaire pour récupérer toutes les données des champs additionnels mêmes celles non présentes dans la config specific + # Nécessaire pour les récupérer coté frontend lors de l'envoi de l'objet (patch et post) properties["additional_data_keys"] = [] for prop in data: if prop not in properties.keys(): @@ -149,7 +149,7 @@ def serialize_children(self, depth): children = {} for children_type in children_types: - # attention a bien nommer les relation en children_type + 's' !!! + # attention a bien nommer les relations en children_type + 's' !!! relation_name = children_type + "s" if not hasattr(self._model, relation_name): @@ -219,7 +219,7 @@ def serialize(self, depth=1, is_child=False, scope=None): # Liste des propriétés de l'objet qui doivent être récupérées display_properties = [] - # Liste des propriétés spécifique de l'objet qui doivent être récupérées + # Liste des propriétés spécifiques de l'objet qui doivent être récupérées display_specific = [] if is_child: module_config = self.config() @@ -248,13 +248,13 @@ def serialize(self, depth=1, is_child=False, scope=None): else: # Si l'objet n'est pas un enfant on récupére toutes les informations - # Pour pourvoir afficher le détails + # Pour pouvoir afficher les détails dump_object = MonitoringSerializer_dict[self._object_type](unknown=EXCLUDE).dump( self._model ) properties = dump_object - # Extraction des proprités spécifiques au même niveau que les génériques + # Extraction des propriétés spécifiques au même niveau que les génériques self.flatten_specific_properties(properties, only=display_specific) # Sérialisation des enfants diff --git a/backend/gn_module_monitoring/tests/test_import/files/existing_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/existing_sites_groups.csv new file mode 100644 index 000000000..1a295e2d3 --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/existing_sites_groups.csv @@ -0,0 +1,2 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom;uuid_base_site;s__base_site_code;s__base_site_name;s__base_site_description;s__meteo;s__profondeur_grotte;s__contact_name;s__first_use_date;s__roost_type;y;x;uuid_base_visit;v__visit_date_min;v__visit_date_max;v__meteo;v__observers;uuid_observation;o__cd_nom;o__comments;id_dataset +;@EXISTING_GROUP_UUID@;GEX;Groupe existant;spec EX;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));77777777-7777-7777-7777-777777777777;site_ex_01;Site EX 01;Site rattache a un groupe existant;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;88888888-8888-8888-8888-888888888888;01/01/2025;01/01/2025;beau;test_user1;99999999-8888-7777-6666-555555555555;798243;Observation groupe existant;@VALID_DATASET_UUID@ diff --git a/backend/gn_module_monitoring/tests/test_import/files/incoherent_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/incoherent_sites_groups.csv new file mode 100644 index 000000000..f17c45665 --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/incoherent_sites_groups.csv @@ -0,0 +1,3 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom;uuid_base_site;s__base_site_code;s__base_site_name;s__base_site_description;s__meteo;s__profondeur_grotte;s__contact_name;s__first_use_date;s__roost_type;y;x;uuid_base_visit;v__visit_date_min;v__visit_date_max;v__meteo;v__observers;uuid_observation;o__cd_nom;o__comments;id_dataset +;22222222-2222-2222-2222-222222222222;GDUP;Groupe dup un;;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));10000000-0000-0000-0000-000000000011;site_dup_11;Site Dup 11;Site du groupe dup ligne 2;METEO_N;100;George;01/01/2025;barrage;44.6;6.1;20000000-0000-0000-0000-000000000011;01/01/2025;01/01/2025;beau;test_user1;30000000-0000-0000-0000-000000000011;798243;Observation dup 11;@VALID_DATASET_UUID@ +;22222222-2222-2222-2222-222222222222;GDUP;Groupe dup deux;;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));10000000-0000-0000-0000-000000000012;site_dup_12;Site Dup 12;Site du groupe dup ligne 3;METEO_N;100;George;02/01/2025;barrage;44.7;6.2;20000000-0000-0000-0000-000000000012;02/01/2025;02/01/2025;beau;test_user1;30000000-0000-0000-0000-000000000012;438587;Observation dup 12;@VALID_DATASET_UUID@ diff --git a/backend/gn_module_monitoring/tests/test_import/files/invalid_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/invalid_sites_groups.csv new file mode 100644 index 000000000..8dc71f235 --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/invalid_sites_groups.csv @@ -0,0 +1,4 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom;uuid_base_site;s__base_site_code;s__base_site_name;s__base_site_description;s__meteo;s__profondeur_grotte;s__contact_name;s__first_use_date;s__roost_type;y;x;uuid_base_visit;v__visit_date_min;v__visit_date_max;v__meteo;v__observers;uuid_observation;o__cd_nom;o__comments;id_dataset +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));550e8400-e29b-41d4-a716-446655440002;test_site_01;Test Site 01;Site description 1;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;24ff384e-4af7-482c-bf6a-d10c2eb0eb94;01/01/2025;01/01/2025;pas ouf;test_user1;33208c1c-20bf-4bd7-82a2-2d4e7e83f382;798243;Observation 1 comments;@VALID_DATASET_UUID@ +;;GERR_ALT;Groupe altitudes inversees;;500;100;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));10000000-0000-0000-0000-000000000002;site_err_02;Site Err 02;Site du groupe altitudes inversees;METEO_N;100;George;02/01/2025;barrage;44.6;6.1;20000000-0000-0000-0000-000000000002;02/01/2025;02/01/2025;beau;test_user1;30000000-0000-0000-0000-000000000002;438587;Observation groupe altitudes;@VALID_DATASET_UUID@ +;;GERR_GEOM;Groupe geometrie invalide;;100;500;POLYGON((invalid));10000000-0000-0000-0000-000000000003;site_err_03;Site Err 03;Site du groupe geometrie invalide;METEO_N;100;George;03/01/2025;barrage;44.7;6.2;20000000-0000-0000-0000-000000000003;03/01/2025;03/01/2025;beau;test_user1;30000000-0000-0000-0000-000000000003;693711;Observation groupe geometrie;@VALID_DATASET_UUID@ diff --git a/backend/gn_module_monitoring/tests/test_import/files/no_parent_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/no_parent_sites_groups.csv new file mode 100644 index 000000000..311a63927 --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/no_parent_sites_groups.csv @@ -0,0 +1,4 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom;uuid_base_site;s__base_site_code;s__base_site_name;s__base_site_description;s__meteo;s__profondeur_grotte;s__contact_name;s__first_use_date;s__roost_type;y;x;uuid_base_visit;v__visit_date_min;v__visit_date_max;v__meteo;v__observers;uuid_observation;o__cd_nom;o__comments;id_dataset +;33333333-3333-3333-3333-333333333333;GNP;Groupe parent;;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));44444444-4444-4444-4444-444444444444;site_np_01;Site NP 01;Site avec groupe;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;20000000-0000-0000-0000-000000000021;01/01/2025;01/01/2025;beau;test_user1;30000000-0000-0000-0000-000000000021;798243;Observation NP 01;@VALID_DATASET_UUID@ +;99999999-9999-9999-9999-999999999999;;;;;;;55555555-5555-5555-5555-555555555555;site_np_02;Site NP 02;Site reference un groupe inexistant;METEO_N;100;George;01/01/2025;barrage;44.6;6.1;;;;;;;;; +;;;;;;;;66666666-6666-6666-6666-666666666666;site_np_03;Site NP 03;Site orphelin sans reference de groupe;METEO_N;100;George;01/01/2025;barrage;44.7;6.2;;;;;;;;; diff --git a/backend/gn_module_monitoring/tests/test_import/files/only_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/only_sites_groups.csv new file mode 100644 index 000000000..20899093e --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/only_sites_groups.csv @@ -0,0 +1,3 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom +;12121212-1212-1212-1212-121212121212;GC1;Groupe seul un;spec C1;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5)) +GROUP_C2;;GC2;Groupe seul deux;spec C2;200;600;POLYGON((6.8 44.8,6.9 44.8,6.9 44.85,6.8 44.85,6.8 44.8)) diff --git a/backend/gn_module_monitoring/tests/test_import/files/valid_sites_groups.csv b/backend/gn_module_monitoring/tests/test_import/files/valid_sites_groups.csv new file mode 100644 index 000000000..011da472f --- /dev/null +++ b/backend/gn_module_monitoring/tests/test_import/files/valid_sites_groups.csv @@ -0,0 +1,7 @@ +id_sites_group_origin;uuid_sites_group;g__sites_group_code;g__sites_group_name;g__group_specific;g__altitude_min;g__altitude_max;g__geom;uuid_base_site;s__base_site_code;s__base_site_name;s__base_site_description;s__meteo;s__profondeur_grotte;s__contact_name;s__first_use_date;s__roost_type;y;x;uuid_base_visit;v__visit_date_min;v__visit_date_max;v__meteo;v__observers;uuid_observation;o__cd_nom;o__comments;id_dataset +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));550e8400-e29b-41d4-a716-446655440002;test_site_01;Test Site 01;Site description 1;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;24ff384e-4af7-482c-bf6a-d10c2eb0eb94;01/01/2025;01/01/2025;pas ouf;test_user1;33208c1c-20bf-4bd7-82a2-2d4e7e83f382;798243;Observation 1 comments;@VALID_DATASET_UUID@ +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));550e8400-e29b-41d4-a716-446655440002;test_site_01;Test Site 01;Site description 1;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;24ff384e-4af7-482c-bf6a-d10c2eb0eb94;01/01/2025;01/01/2025;pas ouf;test_user1;d9949e54-86ce-40e5-b516-c020296337f8;438587;Observation 2 comments;@VALID_DATASET_UUID@ +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));550e8400-e29b-41d4-a716-446655440002;test_site_01;Test Site 01;Site description 1;METEO_N;100;George;01/01/2025;barrage;44.5764805;6.0531825;af9c37ff-e891-43c2-b98a-cc25744a34e9;19/01/2025;19/01/2025;beau;test_user1;50996584-2cf7-46f1-99ad-842d0c0d669c;693711;Observation 3 comments;@VALID_DATASET_UUID@ +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));01b93717-ab9a-482c-a852-4e6f61ae2ac0;test_site_02;Test Site 02;Site description 2;METEO_B;200;Anne;02/01/2025;barrage;44.88250842566327;6.762981626694984;4daed125-5c32-41be-9c7e-1cfb310c52de;16/01/2025;16/01/2025;pluie;test_user2,test_user3;9310c2a7-766b-4871-a219-8289a9b9f2c9;655844;Je suis un commentaire;@VALID_DATASET_UUID@ +;11111111-1111-1111-1111-111111111111;GA;Groupe A;spec A;100;500;POLYGON((6.0 44.5,6.8 44.5,6.8 44.9,6.0 44.9,6.0 44.5));01b93717-ab9a-482c-a852-4e6f61ae2ac0;test_site_02;Test Site 02;Site description 2;METEO_B;200;Anne;02/01/2025;barrage;44.88250842566327;6.762981626694984;4daed125-5c32-41be-9c7e-1cfb310c52de;16/01/2025;16/01/2025;pluie;test_user2,test_user3;e045c5d1-4ec6-40f0-b8c0-070e3307d193;283794;Wow;@VALID_DATASET_UUID@ +GROUP_B;;GB;Groupe B;spec B;200;600;POLYGON((6.8 44.8,6.9 44.8,6.9 44.85,6.8 44.85,6.8 44.8));4d514a15-6e9b-446a-aafe-1d22fad066e5;test_site_03;Test Site 03;Site description 3;METEO_M;300;Joris;03/01/2025;barrage;44.82957727096802;6.834298469329738;f65aef93-2ee1-42d2-9a3d-067c52381888;17/01/2025;17/01/2025;beau;test_user3;baeec6f0-bac5-4eb2-80d6-e39b5037381f;510052;Wow 2;@VALID_DATASET_UUID@ diff --git a/backend/gn_module_monitoring/tests/test_import/test_imports_monitoring.py b/backend/gn_module_monitoring/tests/test_import/test_imports_monitoring.py index f2c97610b..8326a6ed7 100644 --- a/backend/gn_module_monitoring/tests/test_import/test_imports_monitoring.py +++ b/backend/gn_module_monitoring/tests/test_import/test_imports_monitoring.py @@ -1,18 +1,32 @@ +import copy +import json from pathlib import Path import pytest import sqlalchemy as sa from apptax.taxonomie.models import BibListes -from flask import current_app, g +from flask import current_app, g, url_for +from geoalchemy2.shape import from_shape +from shapely.geometry import Point +from werkzeug.datastructures import Headers +from werkzeug.exceptions import Conflict + from geonature.core.gn_commons.models import TModules from geonature.core.gn_monitoring.models import TBaseSites, TBaseVisits, TObservations -from geonature.core.imports.models import BibFields, Destination, Entity +from geonature.core.imports.checks.errors import ImportCodeError +from geonature.core.imports.models import BibFields, Destination, Entity, TImports +from geonature.core.imports.utils import insert_import_data_in_transient_table from geonature.tests.imports.utils import assert_import_errors +from geonature.tests.utils import logged_user, set_logged_user, unset_logged_user from geonature.utils.env import db from pypnusershub.db.models import UserList from gn_module_monitoring.command.cmd import cmd_add_update_import_on_protocole -from gn_module_monitoring.monitoring.models import TMonitoringModules +from gn_module_monitoring.monitoring.models import ( + TMonitoringModules, + TMonitoringSites, + TMonitoringSitesGroups, +) from gn_module_monitoring.command.imports.protocol import update_protocol occhab = pytest.importorskip("gn_module_occhab") @@ -111,7 +125,23 @@ def fieldmapping( preset_fieldmapping, types_site, ): - return { + # Colonnes propres aux groupes de sites. + # Pas de mapping "modules" : cor_sites_group_module est rempli automatiquement + # avec le module de la destination d'import. + sites_group_mapping = { + "uuid_sites_group": {"column_src": "uuid_sites_group"}, + "id_sites_group_origin": {"column_src": "id_sites_group_origin"}, + "g__sites_group_code": {"column_src": "g__sites_group_code"}, + "g__sites_group_name": {"column_src": "g__sites_group_name"}, + "g__group_specific": {"column_src": "g__group_specific"}, + "g__altitude_min": {"column_src": "g__altitude_min"}, + "g__altitude_max": {"column_src": "g__altitude_max"}, + "g__geom": {"column_src": "g__geom"}, + } + # Import de groupes de sites seuls : aucun champ site/visite/observation mappé + if import_file_name == "only_sites_groups.csv": + return sites_group_mapping + mapping = { "uuid_base_site": {"column_src": "uuid_base_site"}, "s__base_site": {"column_src": "s__base_site"}, "s__id_inventor": {"constant_value": {"id_role": 3}}, @@ -138,6 +168,9 @@ def fieldmapping( "o__cd_nom": {"column_src": "o__cd_nom"}, "o__comments": {"column_src": "o__comments"}, } + if "sites_group" in import_file_name: + mapping.update(sites_group_mapping) + return mapping @pytest.fixture() @@ -148,12 +181,14 @@ def autogenerate(): @pytest.fixture(scope="function") def override_in_importfile( import_datasets, + site_group_without_sites, ): return { "@FORBIDDEN_DATASET_UUID@": str(import_datasets["admin"].unique_dataset_id), "@INACTIVE_DATASET_UUID@": str(import_datasets["user--inactive"].unique_dataset_id), "@DATASET_NOT_FOUND@": "03905a03-c7fa-4642-b143-5005fa805377", "@VALID_DATASET_UUID@": str(import_datasets["user"].unique_dataset_id), + "@EXISTING_GROUP_UUID@": str(site_group_without_sites.uuid_sites_group), } @@ -184,6 +219,72 @@ def no_default_uuid(monkeypatch): monkeypatch.setitem(current_app.config["IMPORT"], "DEFAULT_GENERATE_MISSING_UUID", False) +@pytest.fixture() +def sites_group_mandatory_config(monkeypatch): + """Simule un protocole où le groupe de sites est obligatoire : "site" n'est pas + au premier niveau du tree (cf. isSitesGroupMandatory dans check_transient_data).""" + from gn_module_monitoring.config import repositories + + original_get_config = repositories.get_config + + def get_config_without_first_level_site(module_code=None, force=False): + # deepcopy pour ne pas muter la config mise en cache dans current_app.config + config = copy.deepcopy(original_get_config(module_code, force)) + if module_code == "test": + config["tree"]["module"].pop("site", None) + return config + + monkeypatch.setattr(repositories, "get_config", get_config_without_first_level_site) + + +def run_import( + client, + user, + tests_path, + import_file_name, + override_in_importfile, + fieldmapping, + contentmapping, + observers_mapping, +): + """Rejoue la chaîne complète d'import (mêmes étapes que les fixtures + uploaded_import -> ... -> imported_import) pour un second import dans un même test.""" + set_logged_user(client, user) + with open(tests_path / "files" / import_file_name, "rb") as f: + r = client.post( + url_for("import.upload_file"), + data={"file": (f, import_file_name)}, + headers=Headers({"Content-Type": "multipart/form-data"}), + ) + assert r.status_code == 200, r.data + imprt = db.session.get(TImports, r.get_json()["id_import"]) + for before, after in override_in_importfile.items(): + imprt.source_file = imprt.source_file.replace( + before.encode("ascii"), + after.encode("ascii"), + ) + db.session.flush() + r = client.post( + url_for("import.decode_file", import_id=imprt.id_import), + data={"encoding": "utf-8", "format": "csv", "srid": 4326, "separator": ";"}, + ) + assert r.status_code == 200, r.data + db.session.refresh(imprt) + with db.session.begin_nested(): + imprt.fieldmapping = fieldmapping + imprt.source_count = insert_import_data_in_transient_table(imprt) + imprt.loaded = True + imprt.contentmapping = contentmapping + imprt.observermapping = observers_mapping + r = client.post(url_for("import.prepare_import", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + r = client.post(url_for("import.import_valid_data", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + unset_logged_user(client) + db.session.refresh(imprt) + return imprt + + @pytest.mark.usefixtures( "client_class", "temporary_transaction", @@ -237,6 +338,481 @@ def test_import_valid_file(self, datasets, imported_import): == imported_import.statistics["observation_count"] ) + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "valid_sites_groups.csv", None)], + ) + def test_import_valid_sites_groups(self, datasets, imported_import): + assert_import_errors( + imported_import, + set([]), + ) + + assert imported_import.statistics == { + "sites_group_count": 2, + "site_count": 3, + "visit_count": 4, + "observation_count": 6, + "taxa_count": 6, + "import_count": 15, # 2 groupes + 3 sites + 4 visites + 6 observations + "nb_line_valid": 6, + } + + groups = db.session.scalars( + sa.select(TMonitoringSitesGroups).where( + TMonitoringSitesGroups.id_import == imported_import.id_import + ) + ).all() + # Deux groupes de sites importés + assert len(groups) == 2 + assert {group.sites_group_code for group in groups} == {"GA", "GB"} + + groups_by_code = {group.sites_group_code: group for group in groups} + group_a = groups_by_code["GA"] + group_b = groups_by_code["GB"] + + # Les sites sont rattachés à leur groupe parent + assert len(group_a.sites) == 2 + assert len(group_b.sites) == 1 + + # GA : rattachement par UUID fourni -> l'UUID du CSV est conservé + assert str(group_a.uuid_sites_group) == "11111111-1111-1111-1111-111111111111" + # GB : rattachement par id_sites_group_origin (aucun UUID fourni) -> UUID généré + assert group_b.uuid_sites_group is not None + + for group in groups: + # Rattachement au module (cor_sites_group_module) + assert "test" in [module.module_code for module in group.modules] + # Géométrie du groupe importée telle quelle depuis le fichier + assert group.geom is not None + + # Altitudes importées telles quelles (check_altitudes ne valide que min <= max) + assert (group_a.altitude_min, group_a.altitude_max) == (100, 500) + assert (group_b.altitude_min, group_b.altitude_max) == (200, 600) + + # Champ spécifique du groupe stocké dans data (clé sans le préfixe g__), + # à parité avec les champs spécifiques de site/visite/observation + assert group_a.data["group_specific"] == "spec A" + assert group_b.data["group_specific"] == "spec B" + + # Le polygone importé de chaque groupe englobe la géométrie de ses sites enfants + # (cohérence des données du fichier, group.geom n'est pas une bbox calculée) + for group in groups: + for site in group.sites: + assert db.session.scalar(sa.select(sa.func.ST_Covers(group.geom, site.geom))) + + # La bounding box de l'import (SitesGroupImportActions.compute_bounding_box) + # fusionne l'emprise des groupes (GA+GB : x [6.0, 6.9], y [44.5, 44.9]) + # et celle des sites enfants (incluse ici dans celle des groupes) + bbox = imported_import.destination.actions.compute_bounding_box(imported_import) + assert json.loads(json.dumps(bbox)) == { + "type": "Polygon", + "coordinates": [ + [ + [6.0, 44.9], + [6.0, 44.5], + [6.9, 44.5], + [6.9, 44.9], + [6.0, 44.9], + ] + ], + } + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "only_sites_groups.csv", None)], + ) + def test_import_only_sites_groups(self, datasets, imported_import): + """Import d'un fichier ne contenant que des groupes de sites, sans aucun champ + site/visite/observation mappé.""" + assert_import_errors( + imported_import, + set([]), + ) + assert imported_import.statistics == { + "sites_group_count": 2, + "import_count": 2, + "nb_line_valid": 2, + } + + groups = db.session.scalars( + sa.select(TMonitoringSitesGroups).where( + TMonitoringSitesGroups.id_import == imported_import.id_import + ) + ).all() + assert len(groups) == 2 + groups_by_code = {group.sites_group_code: group for group in groups} + assert set(groups_by_code) == {"GC1", "GC2"} + # GC1 : UUID du fichier conservé ; GC2 : UUID généré via id_sites_group_origin + assert ( + str(groups_by_code["GC1"].uuid_sites_group) == "12121212-1212-1212-1212-121212121212" + ) + assert groups_by_code["GC2"].uuid_sites_group is not None + for group in groups: + assert "test" in [module.module_code for module in group.modules] + assert group.geom is not None + assert len(group.sites) == 0 + assert groups_by_code["GC1"].data["group_specific"] == "spec C1" + assert groups_by_code["GC2"].data["group_specific"] == "spec C2" + + # Sans site enfant, la bounding box de l'import est celle des groupes seuls + bbox = imported_import.destination.actions.compute_bounding_box(imported_import) + assert bbox is not None and bbox["type"] == "Polygon" + xs = [x for x, y in bbox["coordinates"][0]] + ys = [y for x, y in bbox["coordinates"][0]] + assert (min(xs), max(xs), min(ys), max(ys)) == (6.0, 6.9, 44.5, 44.9) + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "invalid_sites_groups.csv", None)], + ) + def test_import_sites_groups_errors(self, datasets, prepared_import): + # Ligne 3 : altitude min > altitude max ; ligne 4 : WKT non parsable. + # Le groupe en erreur invalide en cascade le site de la ligne, puis la + # visite et l'observation (ERRONEOUS_PARENT_ENTITY). + expected_errors = { + ( + ImportCodeError.ALTI_MIN_SUP_ALTI_MAX, + "sites_group", + "g__altitude_min", + frozenset({3}), + ), + ( + ImportCodeError.INVALID_WKT, + "sites_group", + "WKT", + frozenset({4}), + ), + ( + ImportCodeError.ERRONEOUS_PARENT_ENTITY, + "site", + "", + frozenset({3, 4}), + ), + ( + ImportCodeError.ERRONEOUS_PARENT_ENTITY, + "visit", + "", + frozenset({3, 4}), + ), + ( + ImportCodeError.ERRONEOUS_PARENT_ENTITY, + "observation", + "", + frozenset({3, 4}), + ), + } + assert_import_errors(prepared_import, expected_errors) + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "invalid_sites_groups.csv", None)], + ) + def test_preview_sites_groups_data(self, client, datasets, prepared_import): + """La prévisualisation avant validation expose l'entité groupe de sites, + ses colonnes mappées et ses comptages de lignes valides/invalides.""" + with logged_user(client, prepared_import.authors[0]): + r = client.get( + url_for("import.preview_valid_data", import_id=prepared_import.id_import) + ) + assert r.status_code == 200, r.data + entities = {e["entity"]["code"]: e for e in r.json["entities"]} + assert set(entities) == {"sites_group", "site", "visit", "observation"} + # Ligne 2 valide, lignes 3 et 4 invalides, pour chacune des quatre entités + # (groupes en erreur puis cascade site -> visite -> observation) + for code, entity_data in entities.items(): + assert entity_data["n_valid_data"] == 1, code + assert entity_data["n_invalid_data"] == 2, code + sites_group_columns = {c["prop"] for c in entities["sites_group"]["columns"]} + assert {"g__sites_group_code", "g__sites_group_name"} <= sites_group_columns + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "incoherent_sites_groups.csv", None)], + ) + def test_import_sites_groups_incoherent_uuid(self, datasets, prepared_import): + """Même UUID de groupe sur deux lignes avec un contenu différent -> INCOHERENT_DATA. + + Le groupe devient non identifiable (validité None) : la référence de groupe des + sites de ces lignes ne peut plus être résolue -> NO_PARENT_ENTITY sur les sites, + puis cascade ERRONEOUS_PARENT_ENTITY sur leurs visites et observations. + """ + expected_errors = { + ( + ImportCodeError.INCOHERENT_DATA, + "sites_group", + "uuid_sites_group", + frozenset({2, 3}), + ), + ( + ImportCodeError.NO_PARENT_ENTITY, + "site", + "uuid_sites_group", + frozenset({2, 3}), + ), + ( + ImportCodeError.ERRONEOUS_PARENT_ENTITY, + "visit", + "", + frozenset({2, 3}), + ), + ( + ImportCodeError.ERRONEOUS_PARENT_ENTITY, + "observation", + "", + frozenset({2, 3}), + ), + } + assert_import_errors(prepared_import, expected_errors) + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "no_parent_sites_groups.csv", None)], + ) + def test_import_sites_groups_mandatory_no_parent( + self, sites_group_mandatory_config, datasets, prepared_import + ): + """Groupe obligatoire (pas de "site" au 1er niveau du tree) : un site + référençant un groupe inexistant (ligne 3, erreur portée par la référence + fournie) ou sans aucun groupe (ligne 4) est rejeté.""" + expected_errors = { + ( + ImportCodeError.NO_PARENT_ENTITY, + "site", + "uuid_sites_group", + frozenset({3}), + ), + ( + ImportCodeError.NO_PARENT_ENTITY, + "site", + "id_sites_group", + frozenset({4}), + ), + } + assert_import_errors(prepared_import, expected_errors) + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "no_parent_sites_groups.csv", None)], + ) + def test_import_sites_without_sites_group(self, datasets, imported_import): + """Même fichier que test_import_sites_groups_mandatory_no_parent mais avec la + config par défaut du module test ("site" au 1er niveau du tree -> groupe + optionnel) : le site sans aucun groupe (ligne 4) est importé sans rattachement, + mais le site référençant un groupe inexistant (ligne 3) reste rejeté — la + référence fournie ne doit pas être silencieusement ignorée.""" + expected_errors = { + ( + ImportCodeError.NO_PARENT_ENTITY, + "site", + "uuid_sites_group", + frozenset({3}), + ), + } + assert_import_errors(imported_import, expected_errors) + assert imported_import.statistics == { + "sites_group_count": 1, + "site_count": 2, + "visit_count": 1, + "observation_count": 1, + "taxa_count": 1, + "import_count": 5, + "nb_line_valid": 2, + } + sites = db.session.scalars( + sa.select(TMonitoringSites).where( + TMonitoringSites.id_import == imported_import.id_import + ) + ).all() + sites_by_code = {site.base_site_code: site for site in sites} + assert set(sites_by_code) == {"site_np_01", "site_np_03"} + group = db.session.execute( + sa.select(TMonitoringSitesGroups).where( + TMonitoringSitesGroups.id_import == imported_import.id_import + ) + ).scalar_one() + # Le site de la ligne 2 est rattaché au groupe défini sur sa ligne + assert sites_by_code["site_np_01"].id_sites_group == group.id_sites_group + # Le site sans aucune référence de groupe est importé sans rattachement + assert sites_by_code["site_np_03"].id_sites_group is None + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "existing_sites_groups.csv", None)], + ) + def test_import_site_attached_to_existing_sites_group( + self, datasets, site_group_without_sites, imported_import + ): + """UUID de groupe déjà présent en base : le groupe du fichier est ignoré + (SKIP_EXISTING_UUID, chemin skip=True) et le site est rattaché au groupe existant.""" + assert_import_errors( + imported_import, + { + ( + ImportCodeError.SKIP_EXISTING_UUID, + "sites_group", + "uuid_sites_group", + frozenset({2}), + ), + }, + ) + # Aucun groupe importé, le reste de la ligne est importé + assert imported_import.statistics == { + "site_count": 1, + "visit_count": 1, + "observation_count": 1, + "taxa_count": 1, + "import_count": 3, + "nb_line_valid": 1, + } + site = db.session.execute( + sa.select(TMonitoringSites).where( + TMonitoringSites.id_import == imported_import.id_import + ) + ).scalar_one() + assert site.id_sites_group == site_group_without_sites.id_sites_group + # Le groupe existant n'est pas modifié par l'import + assert site_group_without_sites.id_import is None + assert site_group_without_sites.sites_group_name == "Site_eolien" + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "valid_sites_groups.csv", None)], + ) + def test_reimport_sites_groups_same_file( + self, + client, + datasets, + imported_import, + tests_path, + import_file_name, + override_in_importfile, + fieldmapping, + contentmapping, + observers_mapping, + ): + """Ré-import du même fichier : les entités identifiées par UUID sont ignorées + (SKIP_EXISTING_UUID) et ne sont pas dupliquées.""" + second_import = run_import( + client, + imported_import.authors[0], + tests_path, + import_file_name, + override_in_importfile, + fieldmapping, + contentmapping, + observers_mapping, + ) + expected_errors = { + ( + ImportCodeError.SKIP_EXISTING_UUID, + "sites_group", + "uuid_sites_group", + frozenset({2, 3, 4, 5, 6}), + ), + ( + ImportCodeError.SKIP_EXISTING_UUID, + "site", + "uuid_base_site", + frozenset({2, 3, 4, 5, 6, 7}), + ), + ( + ImportCodeError.SKIP_EXISTING_UUID, + "visit", + "uuid_base_visit", + frozenset({2, 3, 4, 5, 6, 7}), + ), + ( + ImportCodeError.SKIP_EXISTING_UUID, + "observation", + "uuid_observation", + frozenset({2, 3, 4, 5, 6, 7}), + ), + } + assert_import_errors(second_import, expected_errors) + # GA (UUID fourni dans le fichier) n'est pas dupliqué, ni ses sites + assert ( + db.session.scalar( + sa.select(sa.func.count()).where( + TMonitoringSitesGroups.uuid_sites_group + == "11111111-1111-1111-1111-111111111111" + ) + ) + == 1 + ) + assert ( + db.session.scalar( + sa.select(sa.func.count()).where( + TBaseSites.uuid_base_site == "550e8400-e29b-41d4-a716-446655440002" + ) + ) + == 1 + ) + # Comportement actuel : GB (identifié par id_sites_group_origin, sans UUID dans + # le fichier) reçoit un UUID aléatoire à chaque import et est donc recréé, + # sans site rattaché (son site, identifié par UUID, a été ignoré) + gb_groups = db.session.scalars( + sa.select(TMonitoringSitesGroups).where( + TMonitoringSitesGroups.sites_group_code == "GB" + ) + ).all() + assert len(gb_groups) == 2 + second_gb = next(g for g in gb_groups if g.id_import == second_import.id_import) + assert len(second_gb.sites) == 0 + assert second_import.statistics == { + "sites_group_count": 1, + "import_count": 1, + "nb_line_valid": 1, + } + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "valid_sites_groups.csv", None)], + ) + def test_remove_imported_sites_groups(self, client, datasets, imported_import): + id_import = imported_import.id_import + with logged_user(client, imported_import.authors[0]): + r = client.delete(url_for("import.delete_import", import_id=id_import)) + assert r.status_code == 200, r.data + # Toutes les données de l'import sont supprimées, groupes de sites compris + for model in (TMonitoringSitesGroups, TBaseSites, TBaseVisits, TObservations): + assert ( + db.session.scalar(sa.select(sa.func.count()).where(model.id_import == id_import)) + == 0 + ) + + @pytest.mark.parametrize( + "autogenerate, import_file_name,fieldmapping_preset_name", + [(False, "valid_sites_groups.csv", None)], + ) + def test_remove_import_sites_group_with_manual_site( + self, client, users, datasets, imported_import + ): + """La suppression de l'import est refusée (Conflict) si un site hors import + est rattaché à un groupe de sites importé.""" + group = db.session.scalars( + sa.select(TMonitoringSitesGroups) + .where(TMonitoringSitesGroups.id_import == imported_import.id_import) + .limit(1) + ).first() + site = TMonitoringSites( + id_inventor=users["user"].id_role, + id_digitiser=users["user"].id_role, + base_site_name="Site manuel", + base_site_code="SM1", + base_site_description="Site créé hors import", + geom=from_shape(Point(6.1, 44.6), srid=4326), + types_site=[], + id_sites_group=group.id_sites_group, + ) + with db.session.begin_nested(): + db.session.add(site) + with logged_user(client, imported_import.authors[0]): + r = client.delete(url_for("import.delete_import", import_id=imported_import.id_import)) + assert r.status_code == Conflict.code, r.data + assert str(group.id_sites_group) in r.json["description"] + assert str(site.id_base_site) in r.json["description"] + def test_update_module_label(self, import_destination, module_code): new_label = "test_change" module_data = {"module": {"module_label": new_label}} diff --git a/conftest.py b/conftest.py index 694a7c75a..25ae57008 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,7 @@ +import pytest + +pytest.register_assert_rewrite("geonature.tests.imports.utils") + from geonature.tests.fixtures import * from geonature.tests.fixtures import _session, app, _app, users