Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
16341cc
feat: add sites groups to monitoring import
IdrissaD Mar 19, 2026
a1cc865
fix: bounding boxes group sites
IdrissaD Mar 19, 2026
8d02967
fix: insert in cor_sites_group_module and t_site_complements
IdrissaD Mar 19, 2026
4c82547
style: delete useless commented lines
IdrissaD Mar 19, 2026
a7c0411
add migration
IdrissaD Mar 19, 2026
0c463fb
fix: method call erroneously deleted
IdrissaD Mar 19, 2026
69599d8
fix: sites without parents in a module with sites groups
IdrissaD Mar 19, 2026
f7b3e21
fix: entities insert order when sites and sites groups are at the sam…
IdrissaD Mar 19, 2026
064b6e4
fix: useless modifs
IdrissaD Mar 19, 2026
690cd22
fix: entities not yet handled
IdrissaD Mar 19, 2026
76c4076
refacto: bounding box computing
IdrissaD Mar 19, 2026
6d54429
complete ENTITIES_NOT_AVAILABLE
IdrissaD Mar 19, 2026
16731f6
fix merge problem
IdrissaD Mar 19, 2026
127fa09
fix: reparent sites_groups id_import migration onto develop head to l…
IdrissaD Jul 20, 2026
35cb20a
fix: use valid number widget for sites_group altitude fields to pass …
IdrissaD Jul 21, 2026
02df8a2
test: add sites_groups import test
IdrissaD Jul 21, 2026
6905d83
test: map mandatory modules field for sites_groups import
IdrissaD Jul 21, 2026
273446f
test: drop unsupported sites_groups specific-field assertion
IdrissaD Jul 21, 2026
23214e2
style: apply black formatting
IdrissaD Jul 21, 2026
db11bd3
fix: import sites_groups specific fields into data column for parity …
IdrissaD Jul 21, 2026
60d01d6
fix: remove silent exception handling that swallowed insert errors du…
IdrissaD Jul 21, 2026
f13ea84
fix: remove leftover debug query and print in site set_parent_id_from…
IdrissaD Jul 21, 2026
8d13c39
fix: ignore sites_group modules field in import since cor_sites_group…
IdrissaD Jul 21, 2026
e18f266
style: deduplicate sites_group uuid checks, LINE_NO attribute and ent…
IdrissaD Jul 21, 2026
40e94f0
test: cover sites_group import error paths, existing uuid attachment,…
IdrissaD Jul 22, 2026
7747460
test: fix sites_group import error tests after CI run (full rows to a…
IdrissaD Jul 22, 2026
1284b6b
test: import a file containing only sites groups without any site/vis…
IdrissaD Jul 22, 2026
0146225
test: import sites without sites group when the group entity is optio…
IdrissaD Jul 22, 2026
9a32887
fix: report NO_PARENT_ENTITY when a site references an unresolvable s…
IdrissaD Jul 22, 2026
aab040a
test: preview of an import exposes the sites_group entity with its ma…
IdrissaD Jul 22, 2026
b7e7264
style: rely on core assert rewriting for assert_import_errors diffs i…
IdrissaD Jul 22, 2026
9cd4880
style: drop explanatory comment above register_assert_rewrite
IdrissaD Jul 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion backend/gn_module_monitoring/command/imports/constant.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@
TOOLTIPS = {
"id_base_site_origin": "Identifiant alphanumérique permettant de faire le lien entre les sites et leurs visites si aucun UUID est fourni",
"id_base_visit_origin": "Identifiant alphanumérique permettant de faire le lien entre les visites et leurs observations si aucun UUID est fourni",
"id_sites_group_origin": "Identifiant alphanumérique permettant de faire le lien entre les groupes de sites et leurs sites si aucun UUID est fourni",
}

ENTITIES_NOT_AVAILABLE = ["sites_group", "observation_detail"]
ENTITIES_NOT_AVAILABLE = ["observation_detail", "individual", "marking"]

# Valeur pouvant être retourné suite à l'analyse de la modification du protocole
# cas de la mise à jour d'un module
Expand Down
86 changes: 50 additions & 36 deletions backend/gn_module_monitoring/command/imports/entity.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,17 @@ def get_entities_protocol(module_code: str) -> list:
list
Liste des entités du module.
"""

entities_order = [
"sites_group",
"site",
"visit",
"observation",
"observation_detail", # Pas encore géré par l'import
"individual", # Pas encore géré par l'import
"marking", # Pas encore géré par l'import
]

module_path = monitoring_module_config_path(module_code)

if not (module_path / "config.json").is_file():
Expand All @@ -40,7 +51,9 @@ def get_entities_protocol(module_code: str) -> list:
data_config = json_from_file(module_path / "config.json")
tree = data_config.get("tree", {}).get("module", {})
keys = extract_keys(tree)
return [key for key in list(dict.fromkeys(keys)) if key not in ENTITIES_NOT_AVAILABLE]
unique_keys = [key for key in list(dict.fromkeys(keys)) if key not in ENTITIES_NOT_AVAILABLE]

return [entity for entity in entities_order if entity in unique_keys]


def get_entity_parent(tree: dict, entity_code: str):
Expand All @@ -58,6 +71,10 @@ def find_parent(node, target, parent=None):
return found
return None

# Handle protocol which have sites and sites groups at the same level in config
if entity_code == "site" and "site" in tree and "sites_group" in tree:
return "sites_group"

parent_entity = find_parent(tree, entity_code)
return parent_entity

Expand Down Expand Up @@ -114,6 +131,7 @@ def insert_entities(
"obs_detail" if entity_code == "observation_detail" else entity_code
)
mapping_entity_object_code = {
"sites_group": "MONITORINGS_GRP_SITES",
"site": "MONITORINGS_SITES",
"visit": "MONITORINGS_VISITES",
"observation": "MONITORINGS_OBSERVATIONS",
Expand Down Expand Up @@ -248,47 +266,43 @@ def insert_entity_field_relations(
bib_themes=bib_themes,
is_parent_link=True,
)
get_cor_entity_field(
entity_id=entity_id,
field_name=f"uuid_base_{parent_code}",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)
get_cor_entity_field(
entity_id=entity_id,
field_name=f"id_base_{parent_code}_origin",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)
if parent_code == "sites_group":
get_cor_entity_field(
entity_id=entity_id,
field_name=f"uuid_{parent_code}",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)
get_cor_entity_field(
entity_id=entity_id,
field_name=f"id_{parent_code}_origin",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)
else:
get_cor_entity_field(
entity_id=entity_id,
field_name=f"uuid_base_{parent_code}",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)
get_cor_entity_field(
entity_id=entity_id,
field_name=f"id_base_{parent_code}_origin",
id_destination=id_destination,
bib_themes=bib_themes,
is_parent_link=True,
)


def get_cor_entity_field(
entity_id, field_name, id_destination, bib_themes, order=None, is_parent_link=False
):
"""
Crée une relation entre une entité et un champ dans cor_entity_field.

Parameters
----------
entity_id : int
ID de l'entité
field_name : str
Nom du champ
id_destination : int
ID de la destination (table) où chercher les IDs des entités
bib_themes : dict
Dictionnaire contenant les thèmes
order : int
Ordre du champ dans la hiérarchie des entités
is_parent_link : bool
Indique si le champ est une relation parent

Returns
-------
bool
True si la relation a été créée, False sinon
Crée une relation entre une entité et un champ dans cor_entity_field
"""

id_field = DB.session.execute(
Expand Down
5 changes: 4 additions & 1 deletion backend/gn_module_monitoring/command/imports/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,24 @@ def prepare_fields(

ignored_fields = [
"id_module", # handled manually
"modules", # handled manually (cor_sites_group_module rempli avec le module de la destination)
"id_dataset", # handled manually
"id_digitiser", # handled manually
"uuid_sites_group", # handled manually
"uuid_base_visit", # handled manually
"uuid_base_site", # handled manually
"uuid_observation", # handled manually
"id_sites_group", # generated
"id_base_site", # generated
"id_base_visit", # generated
"id_observation", # generated
"id_nomenclature_type_site", # handled manually
"last_visit", # dynamic
"nb_sites", # dynamic
"nb_visits", # dynamic
"nb_observations", # dynamic
"nb_individuals", # dynamic
"medias", # not importable
"id_sites_group", # ignored for now
]

field_set_manually = []
Expand Down
77 changes: 44 additions & 33 deletions backend/gn_module_monitoring/command/imports/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def get_protocol_data(module_code: str, id_destination: int):
"generic_data": generic_data,
}

# Now we can iterate safetly over confs
# Now we can iterate safely over confs
for entity_code in entity_confs:
entity_conf = entity_confs[entity_code]
parent_entity = get_entity_parent(tree, entity_code)
Expand Down Expand Up @@ -129,16 +129,15 @@ def get_protocol_data(module_code: str, id_destination: int):
[
{
"name_field": name_field,
"fr_label": "Géometrie (WKT)",
"fr_label": "Géométrie (WKT)",
"type_field": "textarea",
"type_column": "text",
"mandatory": True,
"mandatory": False,
"autogenerated": False,
"display": True,
"source_field": f"src_{name_field}",
"multi": False,
"id_destination": id_destination,
"optional_conditions": ["x", "y"],
},
{
"name_field": f"{name_field}_4326",
Expand All @@ -164,39 +163,51 @@ def get_protocol_data(module_code: str, id_destination: int):
"multi": False,
"id_destination": id_destination,
},
{
"name_field": "x",
"fr_label": "Longitude (x)",
"type_field": "number",
"type_column": "number",
"mandatory": True,
"autogenerated": False,
"display": True,
"source_field": "src_x",
"multi": False,
"id_destination": id_destination,
"optional_conditions": [name_field],
},
{
"name_field": "y",
"fr_label": "Latitude (y)",
"type_field": "number",
"type_column": "number",
"mandatory": True,
"autogenerated": False,
"display": True,
"source_field": "src_y",
"multi": False,
"id_destination": id_destination,
"optional_conditions": [name_field],
},
]
)
for name in ["site", "visit", "observation"]:

if entity_code == "site":
for dict in protocol_data["site"]["generic"]:
if dict["name_field"] == name_field:
dict["mandatory"] = True
dict["optional_conditions"] = ["x", "y"]

protocol_data["site"]["generic"].extend(
[
{
"name_field": "x",
"fr_label": "Longitude (x)",
"type_field": "number",
"type_column": "number",
"mandatory": True,
"autogenerated": False,
"display": True,
"source_field": "src_x",
"multi": False,
"id_destination": id_destination,
"optional_conditions": [name_field],
},
{
"name_field": "y",
"fr_label": "Latitude (y)",
"type_field": "number",
"type_column": "number",
"mandatory": True,
"autogenerated": False,
"display": True,
"source_field": "src_y",
"multi": False,
"id_destination": id_destination,
"optional_conditions": [name_field],
},
]
)

for name in ["sites_group", "site", "visit", "observation"]:
label_entity = f"{name.capitalize()}e" if name == "visit" else f"{name.capitalize()}"
if name not in entities:
continue
suffix = name if name == "observation" else f"base_{name}"
suffix = name if name in ["sites_group", "observation"] else f"base_{name}"
protocol_data[name]["generic"].extend(
[
{
Expand Down Expand Up @@ -271,7 +282,7 @@ def get_protocol_data(module_code: str, id_destination: int):
]
)

# Add observation_detail the file exists
# Add observation_detail if the file exists
if "observation_detail" in entities:
observation_detail_specific_path = module_config_dir_path / "observation_detail.json"
observation_detail_generic_path = os.path.join(
Expand Down
17 changes: 8 additions & 9 deletions backend/gn_module_monitoring/command/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,14 @@ def validate_json_file_protocol(module_code: str):
try:
entities = get_entities_protocol(module_code)
for entity_code in entities:
if not entity_code == "sites_group":
# Valid specific file
specific_path = module_config_dir / f"{entity_code}.json"
errors.extend(validate_json_file(specific_path, valid_type_widgets))

# Valid generic file
project_root = Path(__file__).parent.parent
generic_path = project_root / "config" / "generic" / f"{entity_code}.json"
errors.extend(validate_json_file(generic_path, valid_type_widgets))
# Valid specific file
specific_path = module_config_dir / f"{entity_code}.json"
errors.extend(validate_json_file(specific_path, valid_type_widgets))

# Valid generic file
project_root = Path(__file__).parent.parent
generic_path = project_root / "config" / "generic" / f"{entity_code}.json"
errors.extend(validate_json_file(generic_path, valid_type_widgets))
except Exception as e:
errors.append(f"Erreur lors de la lecture des entités: {str(e)}")

Expand Down
4 changes: 2 additions & 2 deletions backend/gn_module_monitoring/config/generic/sites_group.json
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@
"schema_dot_table": "gn_monitoring.t_sites_groups"
},
"altitude_min": {
"type_widget": "integer",
"type_widget": "number",
"attribut_label": "Altitude (min)"
},
"altitude_max": {
"type_widget": "integer",
"type_widget": "number",
"attribut_label": "Altitude (max)"
},
"id_digitiser": {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""[import] add id_import column for sites groups

Revision ID: 974hpr70u7hd
Revises: 461b82ee737a
Create Date: 2026-02-11 16:00:05.425708

"""

from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = "974hpr70u7hd"
down_revision = "3d39820c9ab7"
branch_labels = None
depends_on = None

import_column_name = "id_import"
schema = "gn_monitoring"
table = "t_sites_groups"


def upgrade():
op.add_column(
schema=schema,
table_name=table,
column=sa.Column(import_column_name, sa.Integer, nullable=True),
)


def downgrade():
op.drop_column(
schema=schema,
table_name=table,
column_name=import_column_name,
)
Loading
Loading