diff --git a/it.py b/it.py new file mode 100644 index 0000000..81a647d --- /dev/null +++ b/it.py @@ -0,0 +1,8 @@ +import os + +import django +from django.urls import resolve + +os.environ.setdefault("UV_ENV_FILE", "./dev/.env.docker-compose") +django.setup() +resolve("/") diff --git a/mixtape/core/models/clustering_result.py b/mixtape/core/models/clustering_result.py index a1de92b..f47e5bc 100644 --- a/mixtape/core/models/clustering_result.py +++ b/mixtape/core/models/clustering_result.py @@ -5,7 +5,7 @@ from django.contrib.postgres.fields import ArrayField from django.db import models -from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS, FEATURE_TYPES +# from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS, FEATURE_TYPES from .episode import Episode diff --git a/mixtape/core/models/step.py b/mixtape/core/models/step.py index c7be872..4acf3cc 100644 --- a/mixtape/core/models/step.py +++ b/mixtape/core/models/step.py @@ -26,6 +26,7 @@ class Meta: number = models.PositiveIntegerField() image = models.ImageField(null=True, blank=True) + # TODO: move to algorithms? @contextmanager @staticmethod def rgb_array_to_file(rgb_array: npt.NDArray, file_base_name: str) -> Generator[File]: diff --git a/mixtape/core/tasks/clustering_tasks.py b/mixtape/core/tasks/clustering_tasks.py index 0c803ac..687e27e 100644 --- a/mixtape/core/tasks/clustering_tasks.py +++ b/mixtape/core/tasks/clustering_tasks.py @@ -13,8 +13,8 @@ from django.db.models.query import QuerySet from django.utils import timezone -from mixtape.core.analysis.clustering import cluster_episodes_all_features -from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS, DEFAULT_FEATURE_TYPES +# from mixtape.core.analysis.clustering import cluster_episodes_all_features +# from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS, DEFAULT_FEATURE_TYPES from mixtape.core.json_encoder import CustomJSONEncoder from mixtape.core.models import ClusteringResult, Episode @@ -134,7 +134,9 @@ def compute_single_episode_clustering(episode_id: int) -> dict[str, Any]: Returns: Dict with task status and clustering result ID """ + # TODO: why is this delaying another task? async_result = compute_episode_clustering.delay([episode_id]) + # TODO: why is this returning a result in a task? return { 'status': 'queued', 'parent_task_id': async_result.id, diff --git a/mixtape/core/views.py b/mixtape/core/views.py index 43432ee..f20ebfd 100644 --- a/mixtape/core/views.py +++ b/mixtape/core/views.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from collections import defaultdict from itertools import accumulate import json @@ -9,15 +11,15 @@ from django.http import Http404, HttpRequest, HttpResponse, JsonResponse from django.shortcuts import get_object_or_404, render import numpy as np -from sklearn import cluster, decomposition, pipeline, preprocessing -import umap.umap_ as umap - -from mixtape.core.analysis.clustering import ( - _cluster_episodes_by_feature, - _fetch_all_episode_features, - cluster_episodes_all_features, -) -from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS +# from sklearn import cluster, decomposition, pipeline, preprocessing +# import umap.umap_ as umap + +# from mixtape.core.analysis.clustering import ( +# _cluster_episodes_by_feature, +# _fetch_all_episode_features, +# cluster_episodes_all_features, +# ) +# from mixtape.core.analysis.constants import DEFAULT_CLUSTERING_PARAMS from mixtape.core.json_encoder import CustomJSONEncoder from mixtape.core.models import ActionMapping, ClusteringResult, Episode from mixtape.core.tasks.clustering_tasks import ( diff --git a/pyproject.toml b/pyproject.toml index fdd846c..f86d6a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,6 @@ license-files = ["LICENSE", "NOTICE"] classifiers = ["Private :: Do Not Upload"] dependencies = [ # Runtime dependencies, always needed - "ale-py", - "autorom[accept-rom-license]", "celery", "django[argon2]", "django-allauth", @@ -28,22 +26,9 @@ dependencies = [ "django-stubs-ext", "djangorestframework", "drf-yasg", - "einops", - "gymnasium[atari, other]", - # PettingZoo[butterfly] extra pulls in both pymunk and pygame, but the pypi version of PettingZoo - # is extremely outdated and depends on a version of pymunk that does not build and has no bdist - # wheel for our target platforms. So we manually specify those transitive dependency versions. - "pymunk==6.11.0", - "pygame>=2.3.0", - "PettingZoo>=1.24.3", - "Pillow", "psycopg[binary]", - "ray[default, rllib]==2.34.0", "rich", "scikit-learn", - "SuperSuit", - "torch", - "umap-learn", "whitenoise[brotli]", # Production-only "django-s3-file-field[s3]", @@ -64,6 +49,23 @@ development = [ "watchdog", "werkzeug", ] +analysis = [ + "ale-py", + "autorom[accept-rom-license]", + "einops", + "gymnasium[atari, other]", + # PettingZoo[butterfly] extra pulls in both pymunk and pygame, but the pypi version of PettingZoo + # is extremely outdated and depends on a version of pymunk that does not build and has no bdist + # wheel for our target platforms. So we manually specify those transitive dependency versions. + "pymunk==6.11.0", + "pygame>=2.3.0", + "PettingZoo>=1.24.3", + "Pillow", + "ray[default, rllib]==2.34.0", + "SuperSuit", + "torch", + "umap-learn", +] [dependency-groups] dev = [ diff --git a/tox.ini b/tox.ini index 1d75254..a669f48 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ runner = uv-venv-lock-runner pass_env = DJANGO_* extras = + analysis development [testenv:lint] diff --git a/uv.lock b/uv.lock index 6f7d725..020618c 100644 --- a/uv.lock +++ b/uv.lock @@ -2060,8 +2060,6 @@ name = "mixtape" version = "0.0.0" source = { editable = "." } dependencies = [ - { name = "ale-py" }, - { name = "autorom", extra = ["accept-rom-license"] }, { name = "celery" }, { name = "django", extra = ["argon2"] }, { name = "django-allauth" }, @@ -2079,25 +2077,29 @@ dependencies = [ { name = "django-stubs-ext" }, { name = "djangorestframework" }, { name = "drf-yasg" }, - { name = "einops" }, { name = "gunicorn" }, + { name = "psycopg", extra = ["binary"] }, + { name = "rich" }, + { name = "scikit-learn" }, + { name = "sentry-sdk", extra = ["celery", "django", "pure-eval"] }, + { name = "whitenoise", extra = ["brotli"] }, +] + +[package.optional-dependencies] +analysis = [ + { name = "ale-py" }, + { name = "autorom", extra = ["accept-rom-license"] }, + { name = "einops" }, { name = "gymnasium", extra = ["atari", "other"] }, { name = "pettingzoo" }, { name = "pillow" }, - { name = "psycopg", extra = ["binary"] }, { name = "pygame" }, { name = "pymunk" }, { name = "ray", extra = ["default", "rllib"] }, - { name = "rich" }, - { name = "scikit-learn" }, - { name = "sentry-sdk", extra = ["celery", "django", "pure-eval"] }, { name = "supersuit" }, { name = "torch" }, { name = "umap-learn" }, - { name = "whitenoise", extra = ["brotli"] }, ] - -[package.optional-dependencies] development = [ { name = "django-browser-reload" }, { name = "django-debug-toolbar" }, @@ -2148,8 +2150,8 @@ type = [ [package.metadata] requires-dist = [ - { name = "ale-py" }, - { name = "autorom", extras = ["accept-rom-license"] }, + { name = "ale-py", marker = "extra == 'analysis'" }, + { name = "autorom", extras = ["accept-rom-license"], marker = "extra == 'analysis'" }, { name = "celery" }, { name = "django", extras = ["argon2"] }, { name = "django-allauth" }, @@ -2172,27 +2174,27 @@ requires-dist = [ { name = "django-stubs-ext" }, { name = "djangorestframework" }, { name = "drf-yasg" }, - { name = "einops" }, + { name = "einops", marker = "extra == 'analysis'" }, { name = "gunicorn" }, - { name = "gymnasium", extras = ["atari", "other"] }, + { name = "gymnasium", extras = ["atari", "other"], marker = "extra == 'analysis'" }, { name = "ipython", marker = "extra == 'development'" }, - { name = "pettingzoo", specifier = ">=1.24.3" }, - { name = "pillow" }, + { name = "pettingzoo", marker = "extra == 'analysis'", specifier = ">=1.24.3" }, + { name = "pillow", marker = "extra == 'analysis'" }, { name = "psycopg", extras = ["binary"] }, - { name = "pygame", specifier = ">=2.3.0" }, - { name = "pymunk", specifier = "==6.11.0" }, - { name = "ray", extras = ["default", "rllib"], specifier = "==2.34.0" }, + { name = "pygame", marker = "extra == 'analysis'", specifier = ">=2.3.0" }, + { name = "pymunk", marker = "extra == 'analysis'", specifier = "==6.11.0" }, + { name = "ray", extras = ["default", "rllib"], marker = "extra == 'analysis'", specifier = "==2.34.0" }, { name = "rich" }, { name = "scikit-learn" }, { name = "sentry-sdk", extras = ["celery", "django", "pure-eval"] }, - { name = "supersuit" }, - { name = "torch" }, - { name = "umap-learn" }, + { name = "supersuit", marker = "extra == 'analysis'" }, + { name = "torch", marker = "extra == 'analysis'" }, + { name = "umap-learn", marker = "extra == 'analysis'" }, { name = "watchdog", marker = "extra == 'development'" }, { name = "werkzeug", marker = "extra == 'development'" }, { name = "whitenoise", extras = ["brotli"] }, ] -provides-extras = ["development"] +provides-extras = ["development", "analysis"] [package.metadata.requires-dev] dev = [