Implemented experiments for ICLR 2027

.gitignore

@@ -12,6 +12,7 @@ logs/
 .DS_Store
 tmp/
 elm-stuff/
+models--*/
 
 # Testing
 .coverage

contrib/mimics/exps/003-baselines/baseline_bioclip.py

@@ -0,0 +1,9 @@
+"""Baseline classifier sweep for BioCLIP2.5 ViT-H/14 on Cambridge butterflies (256p, v1.6)."""
+
+
+def make_cfgs() -> list[dict]:
+    return [
+        {
+            "shards_dpath": "/fs/scratch/PAS2136/jbeattie/saev/shards/2e66437c",
+        }
+    ]

contrib/mimics/exps/003-baselines/baseline_dinov3.py

@@ -0,0 +1,15 @@
+"""Baseline classifier sweep for DINOv3 ViT-L/16 on Cambridge butterflies (384p, v1.6).
+
+The DINOv3 activation shards for the Cambridge dataset were cleaned from
+/fs/scratch, so shards_dpath below currently has no recorded activations.
+Regenerate them first with `launch.py shards` (family=dinov3, layers=[-2],
+dataset cambridge-segfolder-v1.6), then point shards_dpath at the new hash.
+"""
+
+
+def make_cfgs() -> list[dict]:
+    return [
+        {
+            "shards_dpath": "/fs/scratch/PAS2136/jbeattie/saev/shards/a1da4c41",
+        }
+    ]

contrib/mimics/exps/003-baselines/baselines.csv

@@ -0,0 +1,49 @@
+model,shard_id,task_name,n_erato,n_melpomene,bal_acc_mean,bal_acc_std,auc_mean,auc_std
+bioclip2.5-vith14,2e66437c,lativitta_dorsal_vs_malleti_dorsal,1636,891,0.98302196015358,0.00737219849964053,0.9986968146672579,0.0003511235869817655
+bioclip2.5-vith14,2e66437c,lativitta_ventral_vs_malleti_ventral,1637,893,0.9827393933379357,0.009525820925983789,0.996903508139499,0.0022787005502441066
+bioclip2.5-vith14,2e66437c,cyrbia_dorsal_vs_cythera_dorsal,370,55,0.9232186732186731,0.05503630659607398,0.9864864864864863,0.0076916343206380935
+bioclip2.5-vith14,2e66437c,cyrbia_ventral_vs_cythera_ventral,370,55,0.8777641277641278,0.08918614326636368,0.9751842751842752,0.033295783029945826
+bioclip2.5-vith14,2e66437c,notabilis_dorsal_vs_plesseni_dorsal,309,241,0.9979166666666666,0.004166666666666697,1.0,0.0
+bioclip2.5-vith14,2e66437c,notabilis_ventral_vs_plesseni_ventral,307,239,0.9885774722369117,0.008380015055501564,0.9998641615191032,0.0001663828438319494
+bioclip2.5-vith14,2e66437c,hydara_dorsal_vs_melpomene_dorsal,109,229,0.93861158165506,0.037442895749045385,0.9837618420227117,0.022318468187199975
+bioclip2.5-vith14,2e66437c,hydara_ventral_vs_melpomene_ventral,109,227,0.9637185519794217,0.018130152873928586,0.9933414894284459,0.007190838769793013
+bioclip2.5-vith14,2e66437c,venus_dorsal_vs_vulcanus_dorsal,207,102,0.9227409988385598,0.057558479724736634,0.9845238095238095,0.012762319947820351
+bioclip2.5-vith14,2e66437c,venus_ventral_vs_vulcanus_ventral,206,103,0.9173228803716608,0.03407028712074117,0.9563298490127758,0.02277101157538431
+bioclip2.5-vith14,2e66437c,ssp.nov.P_dorsal_vs_malleti_dorsal,168,891,0.929491414120901,0.03730958878190825,0.9896223083706236,0.0036714959436587574
+bioclip2.5-vith14,2e66437c,ssp.nov.P_ventral_vs_malleti_ventral,158,893,0.9241313471806845,0.018797049459278355,0.9869168399610416,0.008224553298665255
+dinov3-vitl16,3756d5b9,lativitta_dorsal_vs_malleti_dorsal,1636,891,0.9991355303845694,0.0017289392308612152,0.9999589974885961,0.00008200502280764077
+dinov3-vitl16,3756d5b9,lativitta_ventral_vs_malleti_ventral,1637,893,0.9965897383944167,0.0032030888606340096,0.9998493871349808,0.00020534225891704125
+dinov3-vitl16,3756d5b9,cyrbia_dorsal_vs_cythera_dorsal,370,55,0.97002457002457,0.02196029705795918,0.9992628992628994,0.0009828009828009633
+dinov3-vitl16,3756d5b9,cyrbia_ventral_vs_cythera_ventral,370,55,0.9764127764127764,0.023886689272108814,0.9992628992628994,0.0009828009828009633
+dinov3-vitl16,3756d5b9,notabilis_dorsal_vs_plesseni_dorsal,309,241,0.9917091836734693,0.007784212579146788,0.9996639784946236,0.0005205622777160503
+dinov3-vitl16,3756d5b9,notabilis_ventral_vs_plesseni_ventral,307,239,0.9946379781420764,0.006833674172100678,1.0,0.0
+dinov3-vitl16,3756d5b9,hydara_dorsal_vs_melpomene_dorsal,109,229,0.9772055963360311,0.016045493102670942,0.9992007026789634,0.0007536160223566593
+dinov3-vitl16,3756d5b9,hydara_ventral_vs_melpomene_ventral,109,227,0.9636219336219337,0.019430827541366824,0.9955768868812347,0.003954363302864833
+dinov3-vitl16,3756d5b9,venus_dorsal_vs_vulcanus_dorsal,207,102,0.9560162601626017,0.005716345011033408,0.9945063879210221,0.004070777435308824
+dinov3-vitl16,3756d5b9,venus_ventral_vs_vulcanus_ventral,206,103,0.8956678281068525,0.03649535597185299,0.958931475029036,0.02279378961790591
+dinov3-vitl16,3756d5b9,ssp.nov.P_dorsal_vs_malleti_dorsal,168,891,0.9398349678890392,0.01644217808546759,0.9934171495590778,0.002530140740667018
+dinov3-vitl16,3756d5b9,ssp.nov.P_ventral_vs_malleti_ventral,158,893,0.9505699161808685,0.020351160246502353,0.9847742583439469,0.011082548387451065
+bioclip2.5-vith14,46bcd3db,lativitta_dorsal_vs_malleti_dorsal,1636,891,0.9974012464326174,0.0023298014380150355,,
+bioclip2.5-vith14,46bcd3db,lativitta_ventral_vs_malleti_ventral,1637,893,0.9914546084416772,0.004146557071709601,,
+bioclip2.5-vith14,46bcd3db,cyrbia_dorsal_vs_cythera_dorsal,370,55,0.9595823095823096,0.03344909037772437,,
+bioclip2.5-vith14,46bcd3db,cyrbia_ventral_vs_cythera_ventral,370,55,0.97002457002457,0.03617597169207131,,
+bioclip2.5-vith14,46bcd3db,notabilis_dorsal_vs_plesseni_dorsal,309,241,0.9906092275619924,0.006544308644715273,,
+bioclip2.5-vith14,46bcd3db,notabilis_ventral_vs_plesseni_ventral,307,239,0.9860851467008711,0.014261852431014492,,
+bioclip2.5-vith14,46bcd3db,hydara_dorsal_vs_melpomene_dorsal,109,229,0.9633339607252651,0.02736056650225681,,
+bioclip2.5-vith14,46bcd3db,hydara_ventral_vs_melpomene_ventral,109,227,0.9524537298450342,0.033661532193360044,,
+bioclip2.5-vith14,46bcd3db,venus_dorsal_vs_vulcanus_dorsal,207,102,0.9735133565621371,0.024227851067502797,,
+bioclip2.5-vith14,46bcd3db,venus_ventral_vs_vulcanus_ventral,206,103,0.9028048780487804,0.027212208950103726,,
+bioclip2.5-vith14,46bcd3db,ssp.nov.P_dorsal_vs_malleti_dorsal,168,891,0.9011971021196468,0.016108159771161937,,
+bioclip2.5-vith14,46bcd3db,ssp.nov.P_ventral_vs_malleti_ventral,158,893,0.9195288438953471,0.03817699564487849,,
+dinov3-vitl16,a1da4c41,lativitta_dorsal_vs_malleti_dorsal,1636,891,0.9907195071199049,0.005006835619635315,0.9989628839251422,0.001081629014746479
+dinov3-vitl16,a1da4c41,lativitta_ventral_vs_malleti_ventral,1637,893,0.9953633582692707,0.0028327215815540396,0.9997260569569437,0.00042195852979583025
+dinov3-vitl16,a1da4c41,cyrbia_dorsal_vs_cythera_dorsal,370,55,0.97002457002457,0.024598877407799202,0.9948402948402949,0.006879606879606887
+dinov3-vitl16,a1da4c41,cyrbia_ventral_vs_cythera_ventral,370,55,0.9673218673218674,0.03603803694839543,0.9732186732186733,0.0314784112788394
+dinov3-vitl16,a1da4c41,notabilis_dorsal_vs_plesseni_dorsal,309,241,0.9983870967741936,0.0032258064516129,1.0,0.0
+dinov3-vitl16,a1da4c41,notabilis_ventral_vs_plesseni_ventral,307,239,1.0,0.0,1.0,0.0
+dinov3-vitl16,a1da4c41,hydara_dorsal_vs_melpomene_dorsal,109,229,0.9795771378380074,0.013133002394586164,0.9980105401844532,0.0018819838034961117
+dinov3-vitl16,a1da4c41,hydara_ventral_vs_melpomene_ventral,109,227,0.9797935880544577,0.013167875270855053,0.9961697722567289,0.005763239496210127
+dinov3-vitl16,a1da4c41,venus_dorsal_vs_vulcanus_dorsal,207,102,0.9632752613240418,0.026335879665141195,0.993472706155633,0.008467576036107372
+dinov3-vitl16,a1da4c41,venus_ventral_vs_vulcanus_ventral,206,103,0.8909639953542394,0.030562997811800288,0.9526074332171893,0.006649191809858322
+dinov3-vitl16,a1da4c41,ssp.nov.P_dorsal_vs_malleti_dorsal,168,891,0.9603370808894999,0.016896932610991146,0.9950845563829128,0.003783597180852484
+dinov3-vitl16,a1da4c41,ssp.nov.P_ventral_vs_malleti_ventral,158,893,0.9666360385310846,0.018140630344994885,0.9904828737235782,0.010554644666379915

contrib/mimics/exps/003-baselines/export_csv.py

@@ -0,0 +1,58 @@
+"""Export baseline classifier stats to a readable CSV.
+
+Reads every baseline_cls_*.parquet from DEFAULT_OUT_DPATH and writes a single
+CSV with one row per (shard, task) and a short model label.
+"""
+
+import pathlib
+
+import beartype
+import polars as pl
+from mimics.baselines import DEFAULT_OUT_DPATH
+
+MODEL_LABELS = {
+    "clip": "bioclip2.5-vith14",
+    "dinov3": "dinov3-vitl16",
+}
+
+
+@beartype.beartype
+def get_model_label(family: str) -> str:
+    """Short human-readable model name for a family."""
+    return MODEL_LABELS.get(family, family)
+
+
+@beartype.beartype
+def main(out_fpath: pathlib.Path) -> None:
+    dfs = []
+    for pq_fpath in sorted(DEFAULT_OUT_DPATH.glob("baseline_cls_*.parquet")):
+        df = pl.read_parquet(pq_fpath).with_columns(
+            pl.col("family").map_elements(get_model_label).alias("model")
+        )
+        dfs.append(df)
+    assert dfs, f"No baseline parquets found in '{DEFAULT_OUT_DPATH}'."
+    cols = [
+        "model",
+        "shard_id",
+        "task_name",
+        "n_erato",
+        "n_melpomene",
+        "bal_acc_mean",
+        "bal_acc_std",
+        "auc_mean",
+        "auc_std",
+    ]
+    padded = []
+    for df in dfs:
+        df = df.select(c for c in cols if c in df.columns)
+        for c in cols:
+            if c not in df.columns:
+                df = df.with_columns(pl.lit(None, dtype=pl.Float64).alias(c))
+        padded.append(df)
+    result = pl.concat(padded).select(cols)
+    result.write_csv(out_fpath)
+    print(f"Wrote {result.height} rows to '{out_fpath}'.")
+
+
+if __name__ == "__main__":
+    main(pathlib.Path("baselines.csv"))