diff --git a/CHANGELOG.md b/CHANGELOG.md index e6bf0bc..6cd660b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # CHANGELOG - +- Possibilité d'utiliser des points de montage pour rediriger les chemins donnés dans le shapefile vers un autre dossier +- [Breaking change] Utilisation d'un shapefile pour définir les fichiers donneurs à utiliser pour chaque zone - génération de la carte d'indice même quand il n'y a pas de points à ajouter ## 1.1.1 diff --git a/README.md b/README.md index dd9fde1..2537191 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # patchwork -Patchwork est un outil permettant d'enrichir un fichier lidar à haute densité avec des points d'un fichier à basse densité dans les secteurs où le premier fichier n'a pas de point mais où le second en possède. +Patchwork est un outil permettant d'enrichir un fichier lidar à haute densité avec des points d'un ou plusieurs fichiers à basse densité dans les secteurs où le premier fichier n'a pas de point mais où le second en possède. ## Fonctionnement Les données en entrée sont: - un fichier lidar que l'ont souhaite enrichir -- un fichier lidar contenant des points supplémentaires +- un fichier shapefile, décrivant les fichiers qui serviront à enrichir le fichier lidar et les zones d'application potentielles (détails dans [Définition du fichier shapefile](#définition-du-fichier-shapefile)) En sortie il y a : -- Un fichier, copie du premier en entrée, enrichi des points voulus +- Un fichier, copie du premier en entrée, enrichi des points des fichiers basse densité dans les zones identifiées. -Les deux fichiers d'entrée sont découpés en tuiles carrées, généralement d'1m². Si une tuile du fichier à enrichir ne contient aucun point ayant le classement qui nous intéresse, on prend les points de la tuile de même emplacement du fichier de points supplémentaire. +Les deux fichiers d'entrée sont découpés en mailles carrées, part défaut d'1m². Si une tuile du fichier à enrichir ne contient aucun point ayant le classement qui nous intéresse, on prend les points de la tuile de même emplacement du fichier de points supplémentaire. -L'appartenance à une tuile est décidée par un arrondi par défaut, c'est-à-dire que tous les éléments de [n, n+1[ (ouvert en n+1) font parti de la même tuile. +L'appartenance à une tuile est décidée par un arrondi par défaut, c'est-à-dire que tous les éléments de [n, n+1[ (ouvert en n+1) font partie de la même tuile. ## Installation pré-requis: installer anaconda @@ -24,24 +24,23 @@ git clone https://github.com/IGNF/patchwork.git conda env create -f environment.yml conda activate patchwork ``` -## utilisation +## Utilisation Le script d'ajout de points peut être lancé via : +```bash +python main.py \ + filepath.RECIPIENT_DIRECTORY=[dossier parent du fichier receveur] \ + filepath.RECIPIENT_NAME=[nom du fichier receveur] \ + filepath.SHP_DIRECTORY=[dossier parent du shapefile] \ + filepath.SHP_NAME=[nom du fichier shapefile] \ + filepath.OUTPUT_DIR=[dossier de sortie] \ + filepath.OUTPUT_NAME=[nom du fichier de sortie] \ + [autres options] ``` -python main.py filepath.DONOR_FILE=[chemin fichier donneur] filepath.RECIPIENT_FILE=[chemin fichier receveur] filepath.OUTPUT_FILE=[chemin fichier de sortie] [autres options] -``` -Les différentes options, modifiables soit dans le fichier `configs/configs_patchwork.yaml`, soit en ligne de commande comme indiqué juste au-dessus : - -filepath.DONOR_DIRECTORY : Le répertoire du fichier qui peut donner des points à ajouter -filepath.DONOR_NAME : Le nom du fichier qui peut donner des points à ajouter -filepath.RECIPIENT_DIRECTORY : Le répertoire du fichier qui va obtenir des points en plus -filepath.RECIPIENT_NAME : Le nom du fichier qui va obtenir des points en plus -filepath.OUTPUT_DIR : Le répertoire du fichier en sortie -filepath.OUTPUT_NAME : Le nom du fichier en sortie -filepath.OUTPUT_INDICES_MAP_DIR : Le répertoire de sortie du fichier d'indice -filepath.OUTPUT_INDICES_MAP_NAME : Le nom de sortie du fichier d'indice - -DONOR_CLASS_LIST : Défaut [2, 22]. La liste des classes des points du fichier donneur qui peuvent être ajoutés. -RECIPIENT_CLASS_LIST : Défaut [2, 3, 9, 17]. La liste des classes des points du fichier receveur qui, s'ils sont absents dans une cellule, justifirons de prendre les points du fichier donneur de la même cellule -TILE_SIZE : Défaut 1000. Taille du côté de l'emprise carrée représentée par les fichiers lidar d'entrée -PATCH_SIZE : Défaut 1. taille en mètre du côté d'une cellule (doit être un diviseur de TILE_SIZE, soit pour 1000 : 0.25, 0.5, 2, 4, 5, 10, 25...) +Les différentes options sont modifiables soit dans le fichier `configs/configs_patchwork.yaml`, soit en ligne de commande comme indiqué juste au-dessus. +Voir le fichier [config_patchwork.yaml](configs/configs_patchwork.yaml) pour le détail des options + + +## Définition du fichier shapefile + +TODO \ No newline at end of file diff --git a/configs/configs_patchwork.yaml b/configs/configs_patchwork.yaml index 7c4a359..688f7b0 100644 --- a/configs/configs_patchwork.yaml +++ b/configs/configs_patchwork.yaml @@ -40,6 +40,11 @@ filepath: # path to this subdirectory can be configured using "DONOR_SUBDIRECTORY" DONOR_SUBDIRECTORY: "data" +mount_points: + - ORIGINAL_PATH: \\store\my-store # WARNING: do NOT use quotes around the path if it contains \\ + MOUNTED_PATH: /my_mounted_store/ + ORIGINAL_PLATFORM_IS_WINDOWS: true + CRS: 2154 DONOR_CLASS_LIST: [2, 22] diff --git a/patchwork/patchwork.py b/patchwork/patchwork.py index a645469..bc850d4 100644 --- a/patchwork/patchwork.py +++ b/patchwork/patchwork.py @@ -211,10 +211,7 @@ def patchwork(config: DictConfig): shapefile_path = os.path.join(config.filepath.SHP_DIRECTORY, config.filepath.SHP_NAME) donor_info_df = get_donor_info_from_shapefile( - shapefile_path, - x_shapefile, - y_shapefile, - config.filepath.DONOR_SUBDIRECTORY, + shapefile_path, x_shapefile, y_shapefile, config.filepath.DONOR_SUBDIRECTORY, config.mount_points ) complementary_bd_points = get_complementary_points( diff --git a/patchwork/path_manipulation.py b/patchwork/path_manipulation.py new file mode 100644 index 0000000..0bb7d70 --- /dev/null +++ b/patchwork/path_manipulation.py @@ -0,0 +1,38 @@ +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Dict, List + + +def get_mounted_path_from_raw_path(raw_path: str, mount_points: List[Dict]): + """Get mounted path from a raw path and a list of mount points. + In case the raw path does not correspond to any mount point, the input raw_path is returned. + + Each mount point is described in a dictionary with keys: + - ORIGINAL_PATH (str): Original path of the mounted directory (root of the raw path to replace) + - MOUNTED_PATH (str): Mounted path of the directory (root path by which to replace the root of the raw path + in order to access to the directory on the current computer) + - ORIGINAL_PLATFORM_IS_WINDOWS (bool): true if the raw path should be interpreted as a windows path + when using this mount point + + Args: + raw_path (str): Original path to convert to a mounted path + mount_points (List[Dict]): List of mount points (as described above) + """ + mounted_path = None + for mount_point in mount_points: + mounted_path = get_mounted_path_from_mount_point(raw_path, mount_point) + if mounted_path is not None: + break + if mounted_path is None: + mounted_path = Path(raw_path) + + return mounted_path + + +def get_mounted_path_from_mount_point(raw_path, mount_point): + out_path = None + PureInputPath = PureWindowsPath if mount_point["ORIGINAL_PLATFORM_IS_WINDOWS"] else PurePosixPath + if PureInputPath(raw_path).is_relative_to(PureInputPath(mount_point["ORIGINAL_PATH"])): + relative_path = PureInputPath(raw_path).relative_to(PureInputPath(mount_point["ORIGINAL_PATH"])) + out_path = mount_point["MOUNTED_PATH"] / Path(relative_path) + + return out_path diff --git a/patchwork/shapefile_data_extraction.py b/patchwork/shapefile_data_extraction.py index 02bba63..82de460 100644 --- a/patchwork/shapefile_data_extraction.py +++ b/patchwork/shapefile_data_extraction.py @@ -1,10 +1,16 @@ import fnmatch import os +from typing import Dict, List import geopandas as gpd +from omegaconf import DictConfig +from patchwork.path_manipulation import get_mounted_path_from_raw_path -def get_donor_info_from_shapefile(input_shapefile: str, x: int, y: int, tile_subdirectory: str) -> gpd.GeoDataFrame: + +def get_donor_info_from_shapefile( + input_shapefile: str, x: int, y: int, tile_subdirectory: str, mount_points: List[Dict] | DictConfig +) -> gpd.GeoDataFrame: """Retrieve paths to all the donor files associated with a given tile (with origin x, y) from a shapefile. The shapefile should contain one geometry per donor file, with attributes: @@ -20,6 +26,13 @@ def get_donor_info_from_shapefile(input_shapefile: str, x: int, y: int, tile_sub It is stored in the "full_path" column of the output geodataframe + The mount_point dictionaries should contains these keys: + - ORIGINAL_PATH (str): Original path of the mounted directory (root of the raw path to replace) + - MOUNTED_PATH (str): Mounted path of the directory (root path by which to replace the root of the raw path + in order to access to the directory on the current computer) + - ORIGINAL_PLATFORM_IS_WINDOWS (bool): true if the raw path should be interpreted as a windows path + when using this mount point + Args: input_shapefile (str): Shapefile describing donor files x (int): x coordinate of the tile for which to get the donors @@ -27,6 +40,9 @@ def get_donor_info_from_shapefile(input_shapefile: str, x: int, y: int, tile_sub y (int): y coordinate of the tile for which to get the donors (in the same unit as in the shapefile, usually km) tile_subdirectory (str): subdirectory of "nuage_mixa" in which the donor files are stored + mount_points (List[Dict]): dictionaries describing the mount points to use to interpret paths from "nuage_mixa" + in case the path is related to a distant folder that can be mounted in different ways *(cf. dictionary + structure above) Raises: NotImplementedError: if nom_coord is false (case not handled) @@ -51,8 +67,9 @@ def get_donor_info_from_shapefile(input_shapefile: str, x: int, y: int, tile_sub if len(gdf.index): - def find_las_path_from_geometry_attributes(x: int, y: int, path_root: str): - tile_directory = os.path.join(path_root, tile_subdirectory) + def find_las_path_from_geometry_attributes(x: int, y: int, path_root: str, mount_points: List[Dict]): + mounted_path_root = get_mounted_path_from_raw_path(path_root, mount_points) + tile_directory = os.path.join(mounted_path_root, tile_subdirectory) if not os.path.isdir(tile_directory): raise FileNotFoundError(f"Directory {tile_directory} not found") potential_filenames = fnmatch.filter(os.listdir(tile_directory), f"*{x}_{y}*.la[sz]") @@ -68,10 +85,9 @@ def find_las_path_from_geometry_attributes(x: int, y: int, path_root: str): return os.path.join(tile_directory, potential_filenames[0]) gdf["full_path"] = gdf.apply( - lambda row: find_las_path_from_geometry_attributes(row["x"], row["y"], row["nuage_mixa"]), + lambda row: find_las_path_from_geometry_attributes(row["x"], row["y"], row["nuage_mixa"], mount_points), axis="columns", ) - else: gdf = gpd.GeoDataFrame(columns=["x", "y", "full_path", "geometry"]) diff --git a/test/configs/config_test_mount_points.yaml b/test/configs/config_test_mount_points.yaml new file mode 100644 index 0000000..a2581e2 --- /dev/null +++ b/test/configs/config_test_mount_points.yaml @@ -0,0 +1,63 @@ +# @package _global_ + +# path to original working directory +# hydra hijacks working directory by changing it to the current log directory, +# so it's useful to have this path as a special variable +# learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory +work_dir: ${hydra:runtime.cwd} + +# disable ouput directory from being created +hydra: + output_subdir: null + run: + dir: . + +# disable main.log from being created +defaults: + - override hydra/hydra_logging: disabled + - override hydra/job_logging: disabled + - _self_ + +filepath: + SHP_NAME: null # name of the shapefile used to match tiles to patch + SHP_DIRECTORY: null # path to the directory containing the shapefile + + OUTPUT_DIR: null # directory of the file with added points, from patchwork. + OUTPUT_NAME: null # name of the file with added points, from patchwork. + + INPUT_INDICES_MAP_DIR: null + INPUT_INDICES_MAP_NAME: null + + OUTPUT_INDICES_MAP_DIR: null # path to the directory for the indices map reflecting the changes to the recipient, from patchwork + OUTPUT_INDICES_MAP_NAME: null # name of the indices map reflecting the changes to the recipient, from patchwork + + RECIPIENT_DIRECTORY: null # directory containing the recipient file for patchwork + RECIPIENT_NAME: null # name of the recipient file for patchwork + + # The input shapefile should contain a "nuage_mixa" attrubute for each geometry + # "nuage_mixa" contains the path to the folder containing the files related to a specific donor source. + # Laz/las files from this source are usually contained in a subdirectory of "nuage_mixa" + # path to this subdirectory can be configured using "DONOR_SUBDIRECTORY" + DONOR_SUBDIRECTORY: "data" + +mount_points: + - ORIGINAL_PATH: \\store\my-store # WARNING: do NOT use quotes around the path if it contains \\ + MOUNTED_PATH: . + ORIGINAL_PLATFORM_IS_WINDOWS: true + - ORIGINAL_PATH: /store/my-store # WARNING: do NOT use quotes around the path if it contains \\ + MOUNTED_PATH: . + ORIGINAL_PLATFORM_IS_WINDOWS: false + +CRS: 2154 + +DONOR_CLASS_LIST: [2, 22] +RECIPIENT_CLASS_LIST: [2, 6, 9, 17] + +TILE_SIZE: 1000 +SHP_X_Y_TO_METER_FACTOR: 1000 # multiplication factor to convert shapefile x, y attributes values to meters +PATCH_SIZE: 1 # size of a patch of the grid. Must be a divisor of TILE_SIZE, so for 1000: 0.25, 0.5, 2, 4, 5, 10, 25... +NEW_COLUMN: null # If not null, contains the name of the new column +NEW_COLUMN_SIZE: 8 # must be 8, 16, 32 or 64 +VALUE_ADDED_POINTS: 1 # in case of a new column, value of the new point (the other are set to 0) +VIRTUAL_CLASS_TRANSLATION: {2: 69, 22: 70} # if there is no new column, translate the class of DONOR_CLASS_LIST into those values +# each value of DONOR_CLASS_LIST must be a key in VIRTUAL_CLASS_TRANSLATION. Not used if NEW_COLUMN is not None (or "") diff --git a/test/data/shapefile_mounted_unix_path/patchwork_geometries.dbf b/test/data/shapefile_mounted_unix_path/patchwork_geometries.dbf new file mode 100644 index 0000000..f9a559f Binary files /dev/null and b/test/data/shapefile_mounted_unix_path/patchwork_geometries.dbf differ diff --git a/test/data/shapefile_mounted_unix_path/patchwork_geometries.shp b/test/data/shapefile_mounted_unix_path/patchwork_geometries.shp new file mode 100644 index 0000000..b2f95df Binary files /dev/null and b/test/data/shapefile_mounted_unix_path/patchwork_geometries.shp differ diff --git a/test/data/shapefile_mounted_unix_path/patchwork_geometries.shx b/test/data/shapefile_mounted_unix_path/patchwork_geometries.shx new file mode 100644 index 0000000..59d87a4 Binary files /dev/null and b/test/data/shapefile_mounted_unix_path/patchwork_geometries.shx differ diff --git a/test/data/shapefile_mounted_windows_path/patchwork_geometries.dbf b/test/data/shapefile_mounted_windows_path/patchwork_geometries.dbf new file mode 100644 index 0000000..eda8952 Binary files /dev/null and b/test/data/shapefile_mounted_windows_path/patchwork_geometries.dbf differ diff --git a/test/data/shapefile_mounted_windows_path/patchwork_geometries.shp b/test/data/shapefile_mounted_windows_path/patchwork_geometries.shp new file mode 100644 index 0000000..b2f95df Binary files /dev/null and b/test/data/shapefile_mounted_windows_path/patchwork_geometries.shp differ diff --git a/test/data/shapefile_mounted_windows_path/patchwork_geometries.shx b/test/data/shapefile_mounted_windows_path/patchwork_geometries.shx new file mode 100644 index 0000000..59d87a4 Binary files /dev/null and b/test/data/shapefile_mounted_windows_path/patchwork_geometries.shx differ diff --git a/test/test_indices_map.py b/test/test_indices_map.py index 9d9f97d..9280ae1 100644 --- a/test/test_indices_map.py +++ b/test/test_indices_map.py @@ -61,7 +61,6 @@ def test_create_indices_map(tmp_path_factory): grid = raster.read() grid = grid.transpose() # indices aren't read the way we want otherwise - print(grid) for point in POINTS_IN_GRID: assert grid[point] == 1 diff --git a/test/test_patchwork.py b/test/test_patchwork.py index 5623af6..4c5d66e 100644 --- a/test/test_patchwork.py +++ b/test/test_patchwork.py @@ -406,3 +406,80 @@ def test_patchwork_with_origin(tmp_path_factory, recipient_path, expected_nb_add assert len(output_points) == len(recipient_points) + expected_nb_added_points assert np.sum(output_points.Origin == 0) == len(recipient_points) assert np.sum(output_points.Origin == 1) == expected_nb_added_points + + +@pytest.mark.parametrize( + "input_shp_path, recipient_path, expected_nb_added_points", + # Same tests as "test_patchwork_default", but with shapefiles that refer to paths in mounted stores + [ + ( + "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0673_6362_LA93_IGN69_decimated.laz", + 128675, + ), # One donor / unix paths + ( + "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0673_6363_LA93_IGN69_decimated.laz", + 149490, + ), # Two donors / unix paths + ( + "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0674_6363_LA93_IGN69_decimated.laz", + 0, + ), # No donor / unix paths + ( + "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0673_6362_LA93_IGN69_decimated.laz", + 128675, + ), # One donor / windows paths + ( + "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0673_6363_LA93_IGN69_decimated.laz", + 149490, + ), # Two donors / windows paths + ( + "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp", + "test/data/lidar_HD_decimated/Semis_2022_0674_6363_LA93_IGN69_decimated.laz", + 0, + ), # No donor / windows paths + ], +) +def test_patchwork_with_mount_points(tmp_path_factory, input_shp_path, recipient_path, expected_nb_added_points): + tmp_file_dir = tmp_path_factory.mktemp("data") + tmp_output_las_name = "result_patchwork.laz" + tmp_output_indices_map_name = "result_patchwork_indices.tif" + + with initialize(version_base="1.2", config_path="configs"): # Use configs dir from test directory + config = compose( + config_name="config_test_mount_points.yaml", + overrides=[ + f"filepath.RECIPIENT_DIRECTORY={os.path.dirname(recipient_path)}", + f"filepath.RECIPIENT_NAME={os.path.basename(recipient_path)}", + f"filepath.SHP_DIRECTORY={os.path.dirname(input_shp_path)}", + f"filepath.SHP_NAME={os.path.basename(input_shp_path)}", + f"filepath.OUTPUT_DIR={tmp_file_dir}", + f"filepath.OUTPUT_NAME={tmp_output_las_name}", + f"filepath.OUTPUT_INDICES_MAP_DIR={tmp_file_dir}", + f"filepath.OUTPUT_INDICES_MAP_NAME={tmp_output_indices_map_name}", + f"DONOR_CLASS_LIST={DONOR_CLASS_LIST}", + f"RECIPIENT_CLASS_LIST={RECIPIENT_CLASS_LIST}", + "NEW_COLUMN='Origin'", + ], + ) + patchwork(config) + output_path = os.path.join(tmp_file_dir, tmp_output_las_name) + indices_map_path = os.path.join(tmp_file_dir, tmp_output_indices_map_name) + assert os.path.isfile(output_path) + assert os.path.isfile(indices_map_path) + + with laspy.open(recipient_path) as las_file: + recipient_points = las_file.read().points + with laspy.open(output_path) as las_file: + output_points = las_file.read().points + assert {n for n in las_file.header.point_format.dimension_names} == { + n for n in las_file.header.point_format.standard_dimension_names + } | {"Origin"} + + assert len(output_points) == len(recipient_points) + expected_nb_added_points + assert np.sum(output_points.Origin == 0) == len(recipient_points) + assert np.sum(output_points.Origin == 1) == expected_nb_added_points diff --git a/test/test_path_manipulation.py b/test/test_path_manipulation.py new file mode 100644 index 0000000..20d6a48 --- /dev/null +++ b/test/test_path_manipulation.py @@ -0,0 +1,119 @@ +from pathlib import Path, PurePosixPath, PureWindowsPath + +import geopandas as gpd +import pytest +from hydra import compose, initialize + +import patchwork.path_manipulation as path_manipulation + + +def test_read_from_shapefile_with_windows_path(): + shp_path = "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp" + gdf = gpd.GeoDataFrame.from_file(shp_path, encoding="utf-8") + assert PureWindowsPath(gdf["nuage_mixa"][0]) == PureWindowsPath( + "\\\\store\\my-store\\test\\data\\aveyron_aval_lidarBD" + ) + + +def test_read_from_shapefile_with_unix_path(): + shp_path = "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp" + gdf = gpd.GeoDataFrame.from_file(shp_path, encoding="utf-8") + assert PurePosixPath(gdf["nuage_mixa"][0]) == PurePosixPath("/store/my-store/test/data/aveyron_aval_lidarBD") + + +def test_read_from_config_file(): + with initialize(version_base="1.2", config_path="configs"): + config = compose( + config_name="config_test_mount_points.yaml", + overrides=[], + ) + assert PureWindowsPath(config.mount_points[0].ORIGINAL_PATH) == PureWindowsPath("\\\\store\\my-store") + assert PurePosixPath(config.mount_points[1].ORIGINAL_PATH) == PurePosixPath("/store/my-store") + + +@pytest.mark.parametrize( + "raw_path, mount_point, expected_output", + [ + ( # Windows path with \\ + "\\\\store\\my-store\\my\\path", + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-store\\", + "MOUNTED_PATH": "/my/mounted/store", + }, + Path("/my/mounted/store/my/path"), + ), + ( # Windows path with // + "//store/my-store/my/path", + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-store\\", + "MOUNTED_PATH": "/my/mounted/store", + }, + Path("/my/mounted/store/my/path"), + ), + ( # Unix path with + "/store/my-store/my/path", + { + "ORIGINAL_PLATFORM_IS_WINDOWS": False, + "ORIGINAL_PATH": "/store/my-store/", + "MOUNTED_PATH": "/my/mounted/store", + }, + Path("/my/mounted/store/my/path"), + ), + ( # Windows path that does not correspond to the store + "\\\\store\\my-other-store\\my\\path", + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-store\\", + "MOUNTED_PATH": "/my/mounted/store", + }, + None, + ), + ], +) +def test_get_mounted_path_from_mount_point(raw_path, mount_point, expected_output): + mounted_path = path_manipulation.get_mounted_path_from_mount_point(raw_path, mount_point) + assert mounted_path == expected_output + + +@pytest.mark.parametrize( + "raw_path, expected_output", + [ + # Path in first store + ("//store/my-windows-store-1/my/path", Path("/my/mounted/store-1/my/path")), + # Path in second store's root + ("//store/my-windows-store-2", Path("/my/mounted/store-2")), + # Path in third store + ("/store/my-unix-store/", Path("/my/mounted/unix-store")), + # Path in no store + ("/my/path/from/no/store/", Path("/my/path/from/no/store/")), + ], +) +def test_get_mounted_path_from_raw_path(raw_path, expected_output): + mount_points = [ + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-windows-store-1\\", + "MOUNTED_PATH": "/my/mounted/store-1", + }, + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-windows-store-2\\", + "MOUNTED_PATH": "/my/mounted/store-2", + }, + { + "ORIGINAL_PLATFORM_IS_WINDOWS": False, + "ORIGINAL_PATH": "/store/my-unix-store/", + "MOUNTED_PATH": "/my/mounted/unix-store", + }, + ] + mounted_path = path_manipulation.get_mounted_path_from_raw_path(raw_path, mount_points) + assert mounted_path == expected_output + + +def test_get_mounted_path_from_raw_path_no_mount_point(): + mount_points = [] + raw_path = "/my/path" + mounted_path = path_manipulation.get_mounted_path_from_raw_path(raw_path, mount_points) + assert mounted_path == Path(raw_path) diff --git a/test/test_shapefile_data_extraction.py b/test/test_shapefile_data_extraction.py index 977e4c6..c3d4485 100644 --- a/test/test_shapefile_data_extraction.py +++ b/test/test_shapefile_data_extraction.py @@ -4,6 +4,18 @@ INPUT_SHP_PATH = "test/data/shapefile_local/patchwork_geometries.shp" DONOR_SUBDIRECTORY = "data" +MOUNT_POINTS = [ + { + "ORIGINAL_PLATFORM_IS_WINDOWS": True, + "ORIGINAL_PATH": "\\\\store\\my-store\\", + "MOUNTED_PATH": "/my/mounted/windows/store", + }, + { + "ORIGINAL_PLATFORM_IS_WINDOWS": False, + "ORIGINAL_PATH": "/store/my-store/", + "MOUNTED_PATH": "/my/mounted/unix/store", + }, +] @pytest.mark.parametrize( @@ -23,10 +35,15 @@ }, ), # Expect 2 output files (673, 6365, set()), # Expect no output file + ( + 673, + 6362, + {"test/data/aveyron_aval_lidarBD/data/NUALID_1-0_VLIDAVEYRONAVAL_PTS_0673_6362_LAMB93_IGN69_20210319.laz"}, + ), # Expect only one output file ], ) def test_get_donor_info_from_shapefile(x, y, expected_full_path): - gdf = get_donor_info_from_shapefile(INPUT_SHP_PATH, x, y, DONOR_SUBDIRECTORY) + gdf = get_donor_info_from_shapefile(INPUT_SHP_PATH, x, y, DONOR_SUBDIRECTORY, []) # Check that all paths are filled assert set(gdf.columns) == {"x", "y", "full_path", "geometry"} assert len(gdf.index) == len(expected_full_path) @@ -64,4 +81,47 @@ def test_get_donor_info_from_shapefile(x, y, expected_full_path): ) def test_get_donor_info_from_shapefile_raise_error(input_shp, x, y, error_type): with pytest.raises(error_type): - get_donor_info_from_shapefile(input_shp, x, y, DONOR_SUBDIRECTORY) + get_donor_info_from_shapefile(input_shp, x, y, DONOR_SUBDIRECTORY, []) + + +@pytest.mark.parametrize( + "input_shp_path, x,y, expected_lookup_directory", + [ + ( + "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp", + 673, + 6362, + "/my/mounted/unix/store/test/data/aveyron_aval_lidarBD/data", + ), # Expect only one output file with unix path + ( + "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp", + 673, + 6363, + "/my/mounted/windows/store/test/data/aveyron_aval_lidarBD/data", # noqa: E501 }, + ), # Expect one output file with windows path + ], +) +def test_get_donor_info_from_shapefile_with_mount_points_check_lookup_directory( + input_shp_path, x, y, expected_lookup_directory +): + with pytest.raises(FileNotFoundError, match=f"Directory {expected_lookup_directory} not found"): + get_donor_info_from_shapefile(input_shp_path, x, y, DONOR_SUBDIRECTORY, MOUNT_POINTS) + + +@pytest.mark.parametrize( + "input_shp_path", + [ + "test/data/shapefile_mounted_windows_path/patchwork_geometries.shp", + # Expect no output file with windows paths + "test/data/shapefile_mounted_unix_path/patchwork_geometries.shp", # Expect no output file with unix paths + ], +) +def test_get_donor_info_from_shapefile_with_mount_points_no_path(input_shp_path): + x = 673 + y = 6365 + expected_full_path = set() + gdf = get_donor_info_from_shapefile(input_shp_path, x, y, DONOR_SUBDIRECTORY, MOUNT_POINTS) + # Check that all paths are filled + assert set(gdf.columns) == {"x", "y", "full_path", "geometry"} + assert len(gdf.index) == len(expected_full_path) + assert set(gdf["full_path"]) == expected_full_path