From fa84e040b0842c8c2bea737cf46f2e78b1ec2bb7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 31 Jul 2026 14:07:49 -0700 Subject: [PATCH 1/8] adding protection against calling worker with no work to do --- pcgl2cache/app/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pcgl2cache/app/common.py b/pcgl2cache/app/common.py index bb05e81..358d657 100644 --- a/pcgl2cache/app/common.py +++ b/pcgl2cache/app/common.py @@ -184,7 +184,7 @@ def handle_attributes(graph_id: str, is_binary=False): _add_offset_to_coords(graph_id, l2ids, result) _rescale_volume(graph_id, l2ids, result) update_cache = request.args.get("update_cache", default=True, type=toboolean) - if not update_cache or len(l2ids) == 0: + if not update_cache or len(missing_l2ids) == 0: return result try: _trigger_cache_update(missing_l2ids, graph_id, cache_client.table_id) @@ -236,6 +236,8 @@ def _add_offset_to_coords(graph_id: str, l2ids: Iterable, result: dict): def _trigger_cache_update(l2ids, graph_id: str, l2_cache_id: str) -> None: + if len(l2ids) == 0: + return payload = np.array(l2ids, dtype=np.uint64).tobytes() attributes = { "table_id": graph_id, From 7f6082f7fa4d37ab17edf92fec51262e5d7e5387 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 31 Jul 2026 15:00:24 -0700 Subject: [PATCH 2/8] try python3.13 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0a9db0a..21146c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -ARG PYTHON_VERSION=3.14 +ARG PYTHON_VERSION=3.13 FROM python:${PYTHON_VERSION}-slim ENV GIT_SSL_NO_VERIFY=1 From 2d5d2c5826e1ae459636ffdf8d4f7f645bdadf5d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 2 Aug 2026 18:56:17 -0700 Subject: [PATCH 3/8] flask upgrade fixes --- pcgl2cache/app/__init__.py | 11 ++++++++++- pcgl2cache/app/common.py | 4 ++-- pcgl2cache/app/utils.py | 5 +++-- uwsgi.ini | 9 +++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pcgl2cache/app/__init__.py b/pcgl2cache/app/__init__.py index 5fbf024..4886382 100644 --- a/pcgl2cache/app/__init__.py +++ b/pcgl2cache/app/__init__.py @@ -8,6 +8,7 @@ import numpy as np import redis from flask import Flask +from flask.json.provider import DefaultJSONProvider from flask.logging import default_handler from flask_cors import CORS from rq import Queue @@ -37,6 +38,12 @@ def default(self, obj): return obj.__str__() return json.JSONEncoder.default(self, obj) + +class CustomJSONProvider(DefaultJSONProvider): + def dumps(self, obj, **kwargs): + return super().dumps(obj, default=None, cls=CustomJsonEncoder, **kwargs) + + def get_app_base_path(): return os.path.dirname(os.path.realpath(__file__)) @@ -47,7 +54,9 @@ def get_instance_folder_path(): def create_app(test_config=None): app = Flask(__name__,instance_path=get_instance_folder_path(), instance_relative_config=True) - app.json_encoder = CustomJsonEncoder + # app.json_encoder was removed in Flask 2.3; without the provider numpy + # arrays raise "Object of type ndarray is not JSON serializable" + app.json = CustomJSONProvider(app) CORS(app, expose_headers="WWW-Authenticate") configure_app(app) diff --git a/pcgl2cache/app/common.py b/pcgl2cache/app/common.py index 358d657..4ff0438 100644 --- a/pcgl2cache/app/common.py +++ b/pcgl2cache/app/common.py @@ -87,7 +87,7 @@ def unhandled_exception(e): status_code = 500 response_time = (time.time() - current_app.request_start_time) * 1000 user_ip = str(request.remote_addr) - tb = traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__) + tb = traceback.format_exception(e) current_app.logger.error( { "message": str(e), @@ -114,7 +114,7 @@ def unhandled_exception(e): def api_exception(e): response_time = (time.time() - current_app.request_start_time) * 1000 user_ip = str(request.remote_addr) - tb = traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__) + tb = traceback.format_exception(e) current_app.logger.error( { "message": str(e), diff --git a/pcgl2cache/app/utils.py b/pcgl2cache/app/utils.py index 8988aa9..1d16da2 100644 --- a/pcgl2cache/app/utils.py +++ b/pcgl2cache/app/utils.py @@ -29,14 +29,15 @@ def get_instance_folder_path(): def jsonify_with_kwargs(data, as_response=True, **kwargs): kwargs.setdefault("separators", (",", ":")) - if current_app.config["JSONIFY_PRETTYPRINT_REGULAR"] or current_app.debug: + # JSONIFY_PRETTYPRINT_REGULAR and JSONIFY_MIMETYPE were removed in Flask 2.3 + if current_app.json.compact == False or current_app.debug: kwargs["indent"] = 2 kwargs["separators"] = (", ", ": ") resp = json.dumps(data, **kwargs) if as_response: return current_app.response_class( - resp + "\n", mimetype=current_app.config["JSONIFY_MIMETYPE"] + resp + "\n", mimetype=current_app.json.mimetype ) else: return resp diff --git a/uwsgi.ini b/uwsgi.ini index 1917bba..3ea934f 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -66,6 +66,15 @@ reload-mercy = 605 harakiri = 1200 +### Stats +# Worker state for the uwsgi-exporter sidecar, which turns it into the +# uwsgi_perc_busy_workers metric the API autoscaler reads. Pod-local only; +# the sidecar shares the pod's network namespace. +stats = 127.0.0.1:9191 +stats-http = 127.0.0.1:9192 +stats-interval = 5 + + ### Misc # Maintain Python thread support enable-threads = true From 9231732d961421516532051ab82524b769c5ed1b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 2 Aug 2026 19:20:08 -0700 Subject: [PATCH 4/8] fixing attribute calls --- pcgl2cache/app/common.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pcgl2cache/app/common.py b/pcgl2cache/app/common.py index 4ff0438..a553c7a 100644 --- a/pcgl2cache/app/common.py +++ b/pcgl2cache/app/common.py @@ -144,6 +144,12 @@ def handle_attr_metadata(): } +def _attribute_name(key) -> str: + """Column name, whether kvdbclient keys the row by Attribute or raw bytes.""" + key = getattr(key, "key", key) + return key.decode() if isinstance(key, bytes) else str(key) + + def handle_attributes(graph_id: str, is_binary=False): if is_binary: l2ids = np.frombuffer(request.data, np.uint64) @@ -171,13 +177,14 @@ def handle_attributes(graph_id: str, is_binary=False): result[int(l2id)] = {} for k, v in attrs.items(): val = v[0].value + name = _attribute_name(k) try: # if empty list skip from response if len(val) > 0: - result[int(l2id)][k.decode()] = val + result[int(l2id)][name] = val except TypeError: # add all scalar values to response - result[int(l2id)][k.decode()] = val + result[int(l2id)][name] = val except KeyError: result[int(l2id)] = {} missing_l2ids.append(l2id) From 2257996c8a2314610ceb16778cf0af95229801c9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 4 Aug 2026 13:38:14 -0700 Subject: [PATCH 5/8] update stats config --- uwsgi.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uwsgi.ini b/uwsgi.ini index 3ea934f..a2e8128 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -70,8 +70,12 @@ harakiri = 1200 # Worker state for the uwsgi-exporter sidecar, which turns it into the # uwsgi_perc_busy_workers metric the API autoscaler reads. Pod-local only; # the sidecar shares the pod's network namespace. +# On uWSGI 2.0.31 (this image) stats-http is a boolean that makes the `stats` +# socket speak HTTP -- it is not a second address. Older uWSGI (2.0.21, still +# used by pychunkedgraph) accepted an address there, which is why that repo's +# ini looks like it binds two ports. Keep the port on `stats`. stats = 127.0.0.1:9191 -stats-http = 127.0.0.1:9192 +stats-http = true stats-interval = 5 From 6cea0ccef978a5919bf632aab51552350632415b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 16 Aug 2026 06:54:36 -0700 Subject: [PATCH 6/8] import the ingest CLI lazily so the API does not carry sklearn/scipy app/__init__.py imported ..ingest.cli at module scope purely to register Flask CLI commands. That import chain reaches pcgl2cache.core.features, which pulls in sklearn (43 MB), sklearn.decomposition (10 MB) and scipy.ndimage (22 MB). The registration itself was already conditional on USE_REDIS_JOBS, and the API deployment runs DevelopmentConfig where that is False -- so every uwsgi worker paid for imports whose functions were never called. core.features computes L2 features at ingest time (PCA, EDT); the API only reads precomputed values out of Bigtable. Moving the two imports into the branch that uses them, measured in a deployment pod with a fresh interpreter: before: 8 MB -> 216 MB after create_app() sklearn+scipy.ndimage loaded after: 8 MB -> 140 MB after create_app() neither loaded 76 MB per process, 35% less. Route registration is unchanged (8 routes, including the attributes endpoint), and both deferred imports still resolve for the USE_REDIS_JOBS=True configurations that actually register the commands. Context: the API pods run 8 uwsgi workers minimum against a 900Mi memory request, putting their floor at ~1.05 GiB before serving anything, which was driving repeated node-pressure evictions on a shared node pool. Co-Authored-By: Claude Opus 5 (1M context) --- pcgl2cache/app/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pcgl2cache/app/__init__.py b/pcgl2cache/app/__init__.py index 4886382..f7399cd 100644 --- a/pcgl2cache/app/__init__.py +++ b/pcgl2cache/app/__init__.py @@ -16,8 +16,8 @@ from . import config from .common import bp as l2cache_bp from .v1.routes import bp as l2cache_api_v1 -from ..ingest.cli import init_ingest_cmds -from ..ingest.rq_cli import init_rq_cmds + +# ..ingest.cli and ..ingest.rq_cli are imported lazily in configure_app, see the note there. class CustomJsonEncoder(json.JSONEncoder): @@ -85,6 +85,15 @@ def configure_app(app): app.logger.propagate = False if app.config["USE_REDIS_JOBS"]: + # Imported here rather than at module scope: ..ingest.cli pulls in + # pcgl2cache.core.features, and with it sklearn and scipy.ndimage -- about 76 MB + # of interpreter memory, per uwsgi worker. Only the ingest/CLI configurations set + # USE_REDIS_JOBS, so the API deployment (DevelopmentConfig, USE_REDIS_JOBS=False) + # was paying that cost for commands it never registers. The API reads precomputed + # features out of Bigtable and never computes them, so it has no need of either. + from ..ingest.cli import init_ingest_cmds + from ..ingest.rq_cli import init_rq_cmds + app.redis = redis.Redis.from_url(app.config["REDIS_URL"]) app.test_q = Queue("test", connection=app.redis) with app.app_context(): From 755bcb7d0e6231d6079c0927f6eb3be31a49d1c3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 16 Aug 2026 07:34:41 -0700 Subject: [PATCH 7/8] cache the BigTable client and CloudVolume instead of rebuilding them per request get_l2cache_client built a fresh BigTableClient -- and with it a new gRPC channel -- on every request, and get_l2cache_cv built a fresh CloudVolume, re-parsing the info document each time. Both were called from the attributes endpoint, so a worker did this once per request. The channels were dropped immediately, but the allocation churn sets the worker's heap high-water and glibc does not return it. Measured on minniev7: workers sit at ~176 MB anonymous against the uwsgi master's 157 MB despite the master having *more* loaded, and a worker reaches roughly 68 MB above its post-import baseline within about 20 requests, then stays flat (25 req -> 208 MB, 71 req -> 212 MB). That flatness is why this looked like a fixed cost earlier: the high-water is reached almost at once, not that the churn is free. It also explains why making the ingest imports lazy freed 76 MB of imports without changing RSS -- the churn simply expanded into the space that freed up. Long-lived clients are the intended pattern: gRPC reconnects internally, and the CloudVolume is only read for metadata here (resolution, bounds, graph_chunk_size, meta), never voxel data, so one instance is safe to share. Caches are keyed on the backing l2cache_id / cv_path rather than graph_id, so two graphs pointing at the same table share one client and the cache stays correct if config is rebuilt. The per-request config lookup and its assert are unchanged. Also drops the module-level CACHE dict, which was declared and never referenced anywhere in the package; these caches are evidently what it was meant to be. Co-Authored-By: Claude Opus 5 (1M context) --- pcgl2cache/app/utils.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/pcgl2cache/app/utils.py b/pcgl2cache/app/utils.py index 1d16da2..17109a9 100644 --- a/pcgl2cache/app/utils.py +++ b/pcgl2cache/app/utils.py @@ -1,4 +1,5 @@ import os +from functools import lru_cache from typing import Iterable import numpy as np @@ -11,8 +12,6 @@ from ..core import attributes -CACHE = {} - class DoNothingCreds(Credentials): def refresh(self, request): @@ -43,13 +42,39 @@ def jsonify_with_kwargs(data, as_response=True, **kwargs): return resp +@lru_cache(maxsize=32) +def _l2cache_client(l2cache_id: str) -> BigTableClient: + """One BigTableClient per table, reused for the life of the worker. + + Constructing this per request builds a fresh gRPC channel each time. The channels + are dropped immediately but the allocation churn sets the worker's heap high-water, + which glibc never returns: a worker reaches ~68 MB above its post-import baseline + within roughly 20 requests and then stays there. Long-lived clients are also what + gRPC is designed for -- it reconnects internally, so there is nothing to refresh. + + Keyed on l2cache_id rather than graph_id so the cache stays correct if two graphs + resolve to the same table, or if config is rebuilt. + """ + info = get_default_client_info() + return BigTableClient(l2cache_id, config=info.CONFIG) + + +@lru_cache(maxsize=32) +def _l2cache_cv(cv_path: str) -> CloudVolume: + """One CloudVolume per path, reused for the life of the worker. + + Only metadata is read from it here (resolution, bounds, graph_chunk_size, meta), + never voxel data, so sharing one instance across requests is safe. Constructing it + per request re-parses the info document on every call. + """ + return CloudVolume(cv_path) + + def get_l2cache_client(graph_id: str) -> BigTableClient: l2cache_config = current_app.config["L2CACHE_CONFIG"] assert graph_id in l2cache_config, f"Dataset {graph_id} does not have an L2 Cache." - l2cache_id = l2cache_config[graph_id]["l2cache_id"] - info = get_default_client_info() - return BigTableClient(l2cache_id, config=info.CONFIG) + return _l2cache_client(l2cache_config[graph_id]["l2cache_id"]) def get_l2cache_cv(graph_id: str) -> CloudVolume: @@ -58,8 +83,7 @@ def get_l2cache_cv(graph_id: str) -> CloudVolume: graph_id in l2cache_config ), f"Dataset {graph_id} does not have CV graphene path." - cv_path = l2cache_config[graph_id]["cv_path"] - return CloudVolume(cv_path) + return _l2cache_cv(l2cache_config[graph_id]["cv_path"]) def toboolean(value): From 11b3d8534823191841ab5edac2deaed9ad4c3ed3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 16 Aug 2026 22:13:45 -0700 Subject: [PATCH 8/8] fixing bug in table_id reference --- pcgl2cache/app/common.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pcgl2cache/app/common.py b/pcgl2cache/app/common.py index a553c7a..109b845 100644 --- a/pcgl2cache/app/common.py +++ b/pcgl2cache/app/common.py @@ -194,9 +194,20 @@ def handle_attributes(graph_id: str, is_binary=False): if not update_cache or len(missing_l2ids) == 0: return result try: - _trigger_cache_update(missing_l2ids, graph_id, cache_client.table_id) + # kvdbclient's Client stores the table id privately and exposes no public property, + # so `.table_id` raised AttributeError here while evaluating the argument -- before + # _trigger_cache_update was ever entered. Silently, because of the handler below: the + # endpoint still returned 200 with the missing ids simply absent, so callers saw + # incomplete data and no recompute was ever queued. + _trigger_cache_update(missing_l2ids, graph_id, cache_client._table_id) except Exception as e: - current_app.logger.error(str(e)) + # exc_info so the next failure here shows a traceback. str(e) alone is what made an + # AttributeError read like an idle queue for days. + current_app.logger.error( + f"Failed to trigger l2cache update for {len(missing_l2ids)} l2 ids " + f"on {graph_id}: {e}", + exc_info=True, + ) return result