diff --git a/docs/source/_static/DLC_to_poseinterface_worklow.svg b/docs/source/_static/DLC_to_poseinterface_worklow.svg new file mode 100644 index 0000000..11d9af8 --- /dev/null +++ b/docs/source/_static/DLC_to_poseinterface_worklow.svg @@ -0,0 +1,5 @@ + + +DLC projectbenchmark project{DLCProjectName}/└── labeled-data/├── videos/├── {VideoName}.mp4└── {VideoName}/├── CollectedData_{Name}.csv├── img{index}.png├── img{index}.png├── {VideoName}{model}.csv└── {VideoName}{model}.h5poseinterface_benchmarks/└── Train/ └── {BenchmarkProjectName}/├── Frames/├── sub-{ID}_ses-{ID}_cam-{ID}_frame-{index}.png├── sub-{ID}_ses-{ID}_cam-{ID}_frame-{index}.png├── ...└── sub-{ID}_ses-{ID}_cam-{ID}_framelabels.json├── sub-{ID}_ses-{ID}_cam-{ID}.mp4└── sub-{ID}_ses-{ID}/├── sub-{ID}_ses-{ID}_cam-{ID}_start-{index}_dur-{N}.mp4├── sub-{ID}_ses-{ID}_cam-{ID}_start-{index}_dur-{N}_cliplabels.json└── ...└── ...annotations_to_poseinterfaceframes_to_poseinterfacevideo_to_poseinterfacepredictions_to_poseinterface├── sub-{ID}_ses-{ID}_cam-{ID}_videolabels.jsonextract_clipexctract_clip1. Convert projectrun once per session2. Extract clipscan be runmany times per session└── Clips/ diff --git a/docs/source/_static/project_icon.png b/docs/source/_static/project_icon.png new file mode 100644 index 0000000..0d48013 Binary files /dev/null and b/docs/source/_static/project_icon.png differ diff --git a/docs/source/api_index.rst b/docs/source/api_index.rst index 7e8de89..5ef924b 100644 --- a/docs/source/api_index.rst +++ b/docs/source/api_index.rst @@ -2,7 +2,7 @@ API === io ----- +-- .. currentmodule:: poseinterface.io @@ -11,11 +11,14 @@ io :template: function.rst annotations_to_poseinterface - video_to_poseinterface + frames_to_poseinterface predictions_to_poseinterface + video_to_poseinterface + clips ----- + .. currentmodule:: poseinterface.clips .. autosummary:: @@ -23,3 +26,15 @@ clips :template: function.rst extract_clip + + +utils +----- + +.. currentmodule:: poseinterface.utils + +.. autosummary:: + :toctree: api_generated + :template: function.rst + + tree diff --git a/docs/source/benchmark-dataset.md b/docs/source/benchmark-dataset.md index e21cebb..fc620d9 100644 --- a/docs/source/benchmark-dataset.md +++ b/docs/source/benchmark-dataset.md @@ -17,6 +17,7 @@ We mark requirements with italicised *keywords* that should be interpreted as de The current scope is limited to **single-animal pose estimation** from a **single camera view**. Support for multi-camera setups is planned for a future version. +(target-dataset-folder-structure)= ## Folder structure :::{note} diff --git a/docs/source/conf.py b/docs/source/conf.py index aaca80a..45a85be 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -76,6 +76,9 @@ autosummary_generate = True autodoc_default_flags = ["members", "inherited-members"] +# sphinx-autodoc-typehints configuration +always_use_bars_union = True + # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. @@ -86,6 +89,12 @@ "**/includes/**", # Auto-generated by sphinx-gallery; contains dangling refs to ignored examples "sg_execution_times.rst", + # exclude .py and .ipynb files in examples generated by sphinx-gallery + # this is to prevent sphinx from complaining about duplicate source files + "auto_examples/*.ipynb", + "auto_examples/*.py", + "auto_examples/**/*.ipynb", + "auto_examples/**/*.py", ] # -- Options for HTML output ------------------------------------------------- @@ -138,14 +147,12 @@ } # -- Sphinx-Gallery configuration ------------------------------------------- + sphinx_gallery_conf = { "examples_dirs": ["../../examples"], "gallery_dirs": ["auto_examples"], - # Patterns of example filenames to ignore (not built or shown in gallery). - # Use this for scripts that depend on data/resources not available in CI - # or on the developer's machine (e.g. files on a specific mount point). - # To re-enable an example, remove its pattern from this list. - "ignore_pattern": r"SWC-plusmaze_to_benchmark", + "reference_url": {"poseinterface": None}, + "filename_pattern": "/*.py", # which files to execute before inclusion } # -- linkcheck configuration ------------------------------------------------- diff --git a/examples/SWC-plusmaze_to_benchmark.py b/examples/SWC-plusmaze_to_benchmark.py deleted file mode 100644 index 7fa4600..0000000 --- a/examples/SWC-plusmaze_to_benchmark.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Convert SWC EPM dataset to pose benchmarks format -==================================================== -Convert keypoint annotations -from the Elevated Plus Maze (EPM) dataset from DLC to COCO format. - -Also copy a video and its labeled frames to a target directory, -organised in the pose benchmarks dataset structure. -""" - -# %% -# Imports -# ------- -import shutil -from pathlib import Path - -from poseinterface.io import annotations_to_poseinterface - -# %% -# Background -# ---------- -# We've identified potential datasets from SWC that could be used for the pilot -# version of the pose benchmark dataset. -# Among these is the Elevated Plus Maze (EPM) dataset produced by -# Loukia Katsouri, for John O'Keefe's lab. -# It contains single-animal top-down videos of mice exploring an elevated plus -# maze, with keypoint annotations and predictions from DeepLabCut (DLC). -# -# In this example, we convert the DLC annotations to COCO .json format. - -# %% -# Define source and target directories -# ------------------------------------ -# We specify the paths to the source DLC project directory -# as well as the target directory where converted files will be saved. -# The target will be organised in the pose benchmarks dataset structure. - -source_base_dir = Path( - "/media/ceph-niu/neuroinformatics/sirmpilatzen/behav_data" - "/Loukia/MASTER_DoNotModify" -) -source_project_dir = source_base_dir / "MouseTopDown-Loukia-2022-09-13" -assert source_project_dir.exists(), ( - f"DLC project directory not found: {source_project_dir}" -) - -target_base_dir = Path("/mnt/Data/pose_benchmarks") -target_dataset_dir = target_base_dir / "SWC-plusmaze" -target_dataset_dir.mkdir(parents=True, exist_ok=True) - -# %% -# Copy video to target location -# ----------------------------- -# We identify a specific video by name and copy it to the target directory -# with a standardised naming convention. - -source_video_name = "M708149_EPM_20200317_165049331-converted.mp4" -source_video_path = source_project_dir / "videos" / source_video_name - -# Define subject, session, and view identifiers -subject_id = "M708149" -session_id = "20200317" -view_id = "topdown" -video_id = f"sub-{subject_id}_ses-{session_id}_view-{view_id}" - -# Create target session directory -target_session_dir = target_dataset_dir / f"sub-{subject_id}_ses-{session_id}" -target_session_dir.mkdir(parents=True, exist_ok=True) - -# Copy video to target location -target_video_path = target_session_dir / f"{video_id}.mp4" -if not target_video_path.exists(): - shutil.copy2(source_video_path, target_video_path) - print(f"Copied video to: {target_video_path}") -else: - print(f"Video already exists at: {target_video_path}") - -# %% -# Define source annotations path -# ------------------------------ -# The first attempt failed because the paths in the DLC annotations -# csv file were given as -# ``labeled-data,,.`` -# instead of the required -# ``labeled-data//.``. -# We fixed this by replacing the commas with slashes in the csv file. - -source_labels_dir = ( - source_project_dir / "labeled-data" / source_video_name.replace(".mp4", "") -) -source_annotations_path = source_labels_dir / "CollectedData_Loukia.csv" - -# Create Frames directory inside the session directory -target_frames_dir = target_session_dir / "Frames" -target_frames_dir.mkdir(parents=True, exist_ok=True) - -# %% -# Convert DLC annotations to COCO format -# -------------------------------------- -# Here we use the :func:`annotations_to_poseinterface` -# function from `poseinterface.io` -# which wraps around `sleap_io` functionality to perform the conversion. - -output_annotations_path = annotations_to_poseinterface( - input_path=source_annotations_path, - output_dir=target_frames_dir, - sub_id=subject_id, - ses_id=session_id, - cam_id=view_id, -) -print(f"Saved COCO annotations to: {output_annotations_path}") - -# %% -# Copy labeled frames to target directory -# --------------------------------------- -# Copy the frames used for labeling and rename them to follow -# the naming convention: -# ``sub-{subjectID}_ses-{SessionID}_view-{ViewID}_frame-{FrameID}.png`` - -for source_frame_path in source_labels_dir.glob("*.png"): - # Extract frame number from original filename, e.g. "img0042.png" -> "0042" - frame_number = source_frame_path.stem.replace("img", "") - target_frame_path = ( - target_frames_dir / f"{video_id}_frame-{frame_number}.png" - ) - if not target_frame_path.exists(): - shutil.copy2(source_frame_path, target_frame_path) - -print(f"Copied labeled frames to: {target_frames_dir}") - -# %% diff --git a/examples/convert_dlc_to_benchmark.py b/examples/convert_dlc_to_benchmark.py new file mode 100644 index 0000000..92fd996 --- /dev/null +++ b/examples/convert_dlc_to_benchmark.py @@ -0,0 +1,356 @@ +"""Convert DeepLabCut project to benchmark dataset +================================================== + +Create a ``poseinterface`` benchmark dataset from a DeepLabCut (DLC) project. +""" + +# sphinx_gallery_thumbnail_path = '_static/project_icon.png' + +# %% +# Imports +# ------- +import json +import shutil +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +import poseinterface +from poseinterface.clips import extract_clip +from poseinterface.io import ( + annotations_to_poseinterface, + frames_to_poseinterface, + predictions_to_poseinterface, + video_to_poseinterface, +) +from poseinterface.utils import tree + +# %% +# Overview +# -------- +# We'll handle the conversion in two steps: +# +# 1. **Convert:** DLC project files (videos, frame annotations, and +# keypoint predictions) are restructured into the +# :ref:`poseinterface benchmark layout `. +# 2. **Extract clips:** Short video clips and their labels are extracted +# from the converted videos and their corresponding keypoint +# predictions, ready for expert review. +# +# .. figure:: /_static/DLC_to_poseinterface_worklow.svg +# :alt: Workflow diagram showing how a DLC project is converted +# to a poseinterface benchmark dataset +# :align: center +# +# High-level overview of the two-step conversion workflow. + +# %% +# Source DLC project +# ------------------ +# We work with a dataset from the +# `Sainsbury Wellcome Centre (SWC) `_, +# produced by Loukia Katsouri from John O'Keefe's lab. +# It contains single-animal top-down videos of mice exploring an +# Elevated Plus Maze (EPM), analysed using +# `DeepLabCut (DLC) `_. +# +# .. note:: +# +# This example runs against a lightweight fixture shipped with the +# repository (under ``tests/data/``). This fixture contains only a subset +# of the original DLC project, and is intended for testing and demonstration +# purposes. +# +# Replace ``source_project_dir`` and ``benchmark_base_dir`` with the paths +# to your DLC project and benchmark dataset directories, respectively. Keep +# in mind that your project will contain more files than are shown here. + + +source_project_dir = ( + Path(".").resolve().parent + / "tests" + / "data" + / "dlc" + / "MouseTopDown-Loukia-2022-09-13" +) +print(tree(source_project_dir, level=1, exclude_hidden=True)) + +# For this example we use a temporary directory, cleaned up at the end. +benchmark_base_dir = Path(tempfile.mkdtemp(prefix="poseinterface-benchmark-")) +print(f"\nBenchmark dataset will be saved to: {benchmark_base_dir}") + +# %% +# The two source project sub-directories we care about are: +# +# - ``videos/``: the session videos and their corresponding prediction files. +# - ``labeled-data/``: sampled frames and their keypoint annotations. +# +# Let's peek inside each. + +print(tree(source_project_dir / "videos", level=1, exclude_hidden=True)) + +# %% +# In ``videos/``, each video (ending in ``converted.mp4``) has a companion .csv +# prediction file. + +print(tree(source_project_dir / "labeled-data", level=2, exclude_hidden=True)) + +# %% +# In ``labeled-data/``, the sub-directories mirror the video names (without +# .mp4) and contain the sampled frame images (.png) and their annotations +# (.csv). In real projects you may also find predictions and annotations in +# .h5 format, as well as filtered prediction files. + +# %% +# Define sessions to convert +# --------------------------- +# We select two sessions from the DLC project and assign each to either +# the ``Train`` or ``Test`` split of the +# :ref:`benchmark dataset `. +# You may expand this list with more sessions, but ensure that each session +# belongs to exactly one split, and that the same subject doesn't appear in +# both splits (to avoid data leakage). +# All videos use the same top-down camera view (``cam-topdown``). + +sessions = [ + { + "split": "Train", + "source_video": "M727755_EPM_20200317_170544999-converted.mp4", + "sub_id": "M727755", + "ses_id": "20200317", + "cam_id": "topdown", + }, + { + "split": "Test", + "source_video": "M708154_EPM_20200317_185651629-converted.mp4", + "sub_id": "M708154", + "ses_id": "20200317", + "cam_id": "topdown", + }, +] + +project_name = "SWC-plusmaze" + +# %% +# Convert to benchmark format +# ---------------------------- +# For each session we: +# +# 1. copy (and re-encode, if necessary) the session video; +# 2. convert DLC keypoint annotations to COCO JSON, as well as copy and +# rename the corresponding frame images; +# 3. convert DLC keypoint predictions to COCO JSON. + +for session in sessions: + split = session["split"] + ids = {k: session[k] for k in ["sub_id", "ses_id", "cam_id"]} + sub_ses_prefix = f"sub-{ids['sub_id']}_ses-{ids['ses_id']}" + sub_ses_cam_prefix = f"{sub_ses_prefix}_cam-{ids['cam_id']}" + source_video_path = source_project_dir / "videos" / session["source_video"] + source_frames_dir = ( + source_project_dir / "labeled-data" / source_video_path.stem + ) + target_session_dir = ( + benchmark_base_dir / split / project_name / sub_ses_prefix + ) + target_frames_dir = target_session_dir / "Frames" + target_frames_dir.mkdir(parents=True, exist_ok=True) + + print(f"Converting session: {split}/{project_name}/{sub_ses_prefix}") + # Copy the session video, re-encoding to H.264/yuv420p if necessary + video_to_poseinterface( + input_video=source_video_path, + output_video_dir=target_session_dir, + **ids, + ) + print(f"\tvideo: {source_video_path.name} -> {sub_ses_cam_prefix}.mp4") + + # Convert DLC annotations to COCO frame labels JSON, then copy the + # corresponding frame images with standardised poseinterface filenames. + # In real projects there may be multiple annotation CSVs (e.g. for + # different labelers); adjust the glob pattern to select the right one. + source_annotations_path = next( + source_frames_dir.glob("CollectedData_*.csv"), + None, + ) + if source_annotations_path is None: + print( + f"\tNo CollectedData CSV found in {source_frames_dir}." + " Skipping annotations-to-poseinterface conversion." + ) + else: + framelabels_path = annotations_to_poseinterface( + input_path=source_annotations_path, + output_dir=target_frames_dir, + format="frame", + **ids, + ) + frames_to_poseinterface( + input_dir=source_frames_dir, + output_dir=target_frames_dir, + framelabels_path=framelabels_path, + ) + print( + f"\tannotations (+ frame images): {source_annotations_path.name} " + f"-> {framelabels_path.name}" + ) + + # Convert DLC predictions to COCO video labels JSON for clip extraction. + # In real projects there may be multiple prediction CSVs (e.g. filtered + # versions); adjust the glob pattern to select the right one. + source_predictions_path = next( + (source_project_dir / "videos").glob(f"{source_video_path.stem}*.csv"), + None, + ) + if source_predictions_path is None: + print( + f"\tNo prediction CSV found for {source_video_path.stem!r} in " + f"{source_project_dir / 'videos'}. Skipping predictions-to-" + "poseinterface conversion." + ) + else: + predictions_to_poseinterface( + input_path=source_predictions_path, + video_path=source_video_path, + output_dir=target_session_dir, + **ids, + ) + print( + f"\tpredictions: {source_predictions_path.name} -> " + f"{sub_ses_cam_prefix}_videolabels.json" + ) + print("Done.\n") + +# %% +# The resulting benchmark dataset: + +print(tree(benchmark_base_dir, level=5)) + +# %% +# .. note:: +# +# Frame labels (``framelabels.json``) are generated for both splits, but in +# the **published** dataset the ``Test`` split intentionally omits them for +# evaluation. See the +# :ref:`folder structure specification` for +# details. +# +# The ``videolabels.json`` files generated alongside each session video are +# intermediate artifacts used for clip extraction in the next section, and +# will not be included in the published dataset. + + +# %% +# Extract clips +# ------------- +# Clips (short video segments) can be extracted from the converted session +# videos. When the ``videolabels.json`` files are present, the corresponding +# clip label files (``cliplabels.json``) are generated automatically during +# clip extraction. +# These clip label files should then be proof-read and corrected by +# experts before being included in the benchmark dataset. +# +# First, we specify the clip-extraction parameters. This step can be repeated +# with different parameters to incrementally expand the clip set. + +duration = 5 # in frames +start_frames = [25, 50, 75] +print(f"Extracting {duration}-frame clips starting at frames: {start_frames}") + +# %% +# We loop over all sessions and extract clips at each start frame. +# The resulting video clips and their ``cliplabels.json`` files are saved +# in a ``Clips/`` subdirectory within each session folder. + +for session in sessions: + sub_ses_prefix = f"sub-{session['sub_id']}_ses-{session['ses_id']}" + sub_ses_cam_prefix = f"{sub_ses_prefix}_cam-{session['cam_id']}" + session_dir = ( + benchmark_base_dir / session["split"] / project_name / sub_ses_prefix + ) + + for start_frame in start_frames: + clip_path, _ = extract_clip( + video_path=session_dir / f"{sub_ses_cam_prefix}.mp4", + start_frame=start_frame, + duration=duration, + ) + print(f"Extracted clip: {clip_path.stem}") + + +# %% +# The resulting benchmark dataset, including the extracted clips and their +# corresponding labels: + +print(tree(benchmark_base_dir, level=5)) + + +# %% +# .. note:: +# +# In the published dataset, the ``Train`` split includes all +# ``cliplabels.json`` files. The ``Test`` split omits all +# ``cliplabels.json`` files and instead provides only clip start labels +# (``startlabels.json``), derived from each clip's first frame, +# to support point-tracker evaluation. +# The ``videolabels.json`` files generated in the previous section are +# intermediate artifacts used for clip extraction, and are never shared. +# See the :ref:`folder structure specification` for details. + + +# %% +# Record provenance (optional) +# ---------------------------- +# This step is optional and can be safely skipped, but it is highly recommended +# when converting real data, for book-keeping and reproducibility purposes. +# +# We save a copy of this script alongside a JSON sidecar with the +# ``poseinterface`` version (including git commit, via ``setuptools_scm``) +# and a UTC timestamp. Both files are written to a top-level +# ``.provenance/`` folder, named by project, so multiple projects under the +# same ``benchmark_base_dir`` stay distinct. + +# sphinx_gallery_capture_repr = () +provenance_dir = benchmark_base_dir / ".provenance" +provenance_dir.mkdir(parents=True, exist_ok=True) + +# ``__file__`` is set when running this script directly with Python, but not +# when sphinx-gallery executes it during the docs build, guard accordingly. +script_path_str = globals().get("__file__") +if script_path_str: + shutil.copy(Path(script_path_str), provenance_dir / f"{project_name}.py") + +(provenance_dir / f"{project_name}.json").write_text( + json.dumps( + { + "poseinterface_version": poseinterface.__version__, + "converted_at": datetime.now(timezone.utc).isoformat(), + "source_project_dir": str(source_project_dir), + }, + indent=2, + ) +) + +# %% +# Clean up +# -------- +# Since this example writes to a temporary directory, we remove it at the end. +# +# .. warning:: +# +# Only run this cell when ``benchmark_base_dir`` points to a temporary +# location. The guard below refuses to delete anything outside the system +# temp directory, so it is safe to leave in place when you adapt this +# example to a real benchmark dataset path. + +system_tempdir = Path(tempfile.gettempdir()).resolve() +target = benchmark_base_dir.resolve() +if target.is_relative_to(system_tempdir) and target != system_tempdir: + shutil.rmtree(target) + print(f"Removed temporary benchmark directory: {target}") +else: + print( + f"Refusing to remove {target}: not inside system temp dir " + f"({system_tempdir}). Delete manually if you really want to." + ) diff --git a/poseinterface/io.py b/poseinterface/io.py index 4a6b798..bbabfa8 100644 --- a/poseinterface/io.py +++ b/poseinterface/io.py @@ -5,6 +5,7 @@ import logging import re import shutil +import warnings from pathlib import Path from typing import Literal, TypeAlias @@ -466,6 +467,69 @@ def _reencode_video( return reencoded_video_path +def frames_to_poseinterface( + input_dir: Path, + output_dir: Path, + framelabels_path: Path, +) -> None: + """Copy and rename frame images to match filenames in COCO JSON. + + Source frames are matched to target names by frame number: the + first group of digits in each source filename is compared against + the ``frame-`` field in the COCO ``file_name`` entries. + + Parameters + ---------- + input_dir + Directory containing the source frame images (e.g. DLC + ``labeled-data/