diff --git a/Implementation/Metacentrum/Laion/1000_5000/Metacentrum.py b/Implementation/Metacentrum/Laion/1000_5000/Metacentrum.py index 893289a..4ff87d2 100644 --- a/Implementation/Metacentrum/Laion/1000_5000/Metacentrum.py +++ b/Implementation/Metacentrum/Laion/1000_5000/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -300,15 +300,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -327,7 +327,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -371,7 +371,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -390,7 +390,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -468,7 +468,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Laion/1000_5000/run_experiment.sh b/Implementation/Metacentrum/Laion/1000_5000/run_experiment.sh index 582b61a..c27f516 100755 --- a/Implementation/Metacentrum/Laion/1000_5000/run_experiment.sh +++ b/Implementation/Metacentrum/Laion/1000_5000/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/Laion/100_500/Metacentrum.py b/Implementation/Metacentrum/Laion/100_500/Metacentrum.py index 81573ed..e34ad8e 100644 --- a/Implementation/Metacentrum/Laion/100_500/Metacentrum.py +++ b/Implementation/Metacentrum/Laion/100_500/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -300,15 +300,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -327,7 +327,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -371,7 +371,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -390,7 +390,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -468,7 +468,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Laion/100_500/run_experiment.sh b/Implementation/Metacentrum/Laion/100_500/run_experiment.sh index b3ae530..469a052 100755 --- a/Implementation/Metacentrum/Laion/100_500/run_experiment.sh +++ b/Implementation/Metacentrum/Laion/100_500/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/Laion/10_50/Metacentrum.py b/Implementation/Metacentrum/Laion/10_50/Metacentrum.py index 7a57f8f..956c7a8 100644 --- a/Implementation/Metacentrum/Laion/10_50/Metacentrum.py +++ b/Implementation/Metacentrum/Laion/10_50/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -300,15 +300,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -327,7 +327,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -371,7 +371,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -390,7 +390,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -468,7 +468,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Laion/10_50/run_experiment.sh b/Implementation/Metacentrum/Laion/10_50/run_experiment.sh index f646947..08f044e 100755 --- a/Implementation/Metacentrum/Laion/10_50/run_experiment.sh +++ b/Implementation/Metacentrum/Laion/10_50/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/Metacentrum.py b/Implementation/Metacentrum/Metacentrum.py index d4c5e55..d973f0a 100644 --- a/Implementation/Metacentrum/Metacentrum.py +++ b/Implementation/Metacentrum/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,36 +43,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -143,8 +143,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/selected_indices.npy') k_list = [10, 50, 100, 500, 1000, 5000] growth_factors = [1.0] @@ -152,15 +152,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -211,7 +211,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -230,7 +230,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -308,7 +308,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Profimedia/1000_5000/Metacentrum.py b/Implementation/Metacentrum/Profimedia/1000_5000/Metacentrum.py index 2bbeda6..cc2a52e 100644 --- a/Implementation/Metacentrum/Profimedia/1000_5000/Metacentrum.py +++ b/Implementation/Metacentrum/Profimedia/1000_5000/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,36 +43,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -143,8 +143,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/selected_indices.npy') k_list = [1000, 5000] growth_factors = [1.0] @@ -152,15 +152,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -211,7 +211,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -230,7 +230,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -308,7 +308,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/1000_5000/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/1000_5000/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Profimedia/1000_5000/run_experiment.sh b/Implementation/Metacentrum/Profimedia/1000_5000/run_experiment.sh index 582b61a..c27f516 100755 --- a/Implementation/Metacentrum/Profimedia/1000_5000/run_experiment.sh +++ b/Implementation/Metacentrum/Profimedia/1000_5000/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/1000_5000/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/Profimedia/100_500/Metacentrum.py b/Implementation/Metacentrum/Profimedia/100_500/Metacentrum.py index 065baf2..90fb90f 100644 --- a/Implementation/Metacentrum/Profimedia/100_500/Metacentrum.py +++ b/Implementation/Metacentrum/Profimedia/100_500/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,36 +43,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -143,8 +143,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/selected_indices.npy') k_list = [100, 500] growth_factors = [1.0] @@ -152,15 +152,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -211,7 +211,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -230,7 +230,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -308,7 +308,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/100_500/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/100_500/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Profimedia/100_500/run_experiment.sh b/Implementation/Metacentrum/Profimedia/100_500/run_experiment.sh index b3ae530..469a052 100755 --- a/Implementation/Metacentrum/Profimedia/100_500/run_experiment.sh +++ b/Implementation/Metacentrum/Profimedia/100_500/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/100_500/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/Profimedia/10_50/Metacentrum.py b/Implementation/Metacentrum/Profimedia/10_50/Metacentrum.py index 6a0e101..3c82be4 100644 --- a/Implementation/Metacentrum/Profimedia/10_50/Metacentrum.py +++ b/Implementation/Metacentrum/Profimedia/10_50/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,36 +43,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -143,8 +143,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/selected_indices.npy') k_list = [10, 50] growth_factors = [1.0] @@ -152,15 +152,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -211,7 +211,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -230,7 +230,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -308,7 +308,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/10_50/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/10_50/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum/Profimedia/10_50/run_experiment.sh b/Implementation/Metacentrum/Profimedia/10_50/run_experiment.sh index f646947..08f044e 100755 --- a/Implementation/Metacentrum/Profimedia/10_50/run_experiment.sh +++ b/Implementation/Metacentrum/Profimedia/10_50/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/10_50/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum/run_experiment.sh b/Implementation/Metacentrum/run_experiment.sh index 5142f29..8599cc3 100755 --- a/Implementation/Metacentrum/run_experiment.sh +++ b/Implementation/Metacentrum/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=48:mem=700gb:scratch_local=400mb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/ export PYTHONPATH=$PYTHONPATH:.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum/Metacentrum.py \ No newline at end of file diff --git "a/Implementation/Metacentrum_Cholesky/Laion/Metacentrum - k\303\263pia.py" "b/Implementation/Metacentrum_Cholesky/Laion/Metacentrum - k\303\263pia.py" index a84a831..87f2b6f 100644 --- "a/Implementation/Metacentrum_Cholesky/Laion/Metacentrum - k\303\263pia.py" +++ "b/Implementation/Metacentrum_Cholesky/Laion/Metacentrum - k\303\263pia.py" @@ -39,7 +39,7 @@ from scipy import stats # for describe, spearmanr, kendalltau # ------------------------- Logging like before ------------------------- -RESULTS_ROOT = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/laion" +RESULTS_ROOT = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/laion" os.makedirs(RESULTS_ROOT, exist_ok=True) log_file_path = os.path.join(RESULTS_ROOT, "output_log.txt") log_file = open(log_file_path, "w", buffering=1) @@ -53,7 +53,7 @@ print("=== Logging started ===") USE_PREVIOUS_QUERIES = True # set False to keep the old random-sampling behavior -QUERY_ROOT_DIRECTORY = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/Semantic ax" +QUERY_ROOT_DIRECTORY = "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/Semantic ax" # QUERY_AXES = ["Activities"] # same idea as your semantic_axes_list QUERY_AXES = [ "Activities", @@ -155,16 +155,16 @@ def load_previous_queries(root_directory: str, # Two datasets (edit paths). Same shard layout as before. DATASETS = [ # LAION (unchanged) - ("LAION2B-en", "/storage/plzen1/home/xsikyna/PhD/laion/laionDir") + ("LAION2B-en", "/storage/plzen1/home/anonymous/PhD/laion/laionDir") # Profimedia (NEW) # ("Profimedia", { # "type": "profimedia", # "filenames": [ - # "/Users/matussikyna/PhD local/Paper2/dataset/369.txt", + # "/Users/anonymous/PhD local/Paper2/dataset/369.txt", # ], # # optional: where to reuse/store the 1k random query indices - # "selected_indices_path": "/Users/matussikyna/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy", + # "selected_indices_path": "/Users/anonymous/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy", # "max_items": 20000 # }) ] @@ -183,15 +183,15 @@ def load_previous_queries(root_directory: str, # Matrices: reuse your previous ones (edit paths if needed) methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] # ------------------------- Shard helpers (same spirit as before) ------------------------- diff --git a/Implementation/Metacentrum_Cholesky/Laion/Metacentrum.py b/Implementation/Metacentrum_Cholesky/Laion/Metacentrum.py index b6a2dcb..6afb5dc 100644 --- a/Implementation/Metacentrum_Cholesky/Laion/Metacentrum.py +++ b/Implementation/Metacentrum_Cholesky/Laion/Metacentrum.py @@ -39,7 +39,7 @@ from scipy import stats # for describe, spearmanr, kendalltau # ------------------------- Logging like before ------------------------- -RESULTS_ROOT = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/laion" +RESULTS_ROOT = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/laion" os.makedirs(RESULTS_ROOT, exist_ok=True) log_file_path = os.path.join(RESULTS_ROOT, "output_log.txt") log_file = open(log_file_path, "w", buffering=1) @@ -53,7 +53,7 @@ print("=== Logging started ===") USE_PREVIOUS_QUERIES = True # set False to keep the old random-sampling behavior -QUERY_ROOT_DIRECTORY = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/Semantic ax" +QUERY_ROOT_DIRECTORY = "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/Semantic ax" # QUERY_AXES = ["Activities"] # same idea as your semantic_axes_list QUERY_AXES = [ "Activities", @@ -164,16 +164,16 @@ def load_previous_queries(root_directory: str, # Two datasets (edit paths). Same shard layout as before. DATASETS = [ # LAION (unchanged) - ("LAION2B-en", "/storage/plzen1/home/xsikyna/PhD/laion/laionDir") + ("LAION2B-en", "/storage/plzen1/home/anonymous/PhD/laion/laionDir") # Profimedia (NEW) # ("Profimedia", { # "type": "profimedia", # "filenames": [ - # "/Users/matussikyna/PhD local/Paper2/dataset/369.txt", + # "/Users/anonymous/PhD local/Paper2/dataset/369.txt", # ], # # optional: where to reuse/store the 1k random query indices - # "selected_indices_path": "/Users/matussikyna/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy", + # "selected_indices_path": "/Users/anonymous/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy", # "max_items": 20000 # }) ] @@ -192,15 +192,15 @@ def load_previous_queries(root_directory: str, # Matrices: reuse your previous ones (edit paths if needed) methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] # ------------------------- Shard helpers (same spirit as before) ------------------------- diff --git a/Implementation/Metacentrum_Cholesky/Laion/run_experiment.sh b/Implementation/Metacentrum_Cholesky/Laion/run_experiment.sh index d4899c4..8b934bc 100755 --- a/Implementation/Metacentrum_Cholesky/Laion/run_experiment.sh +++ b/Implementation/Metacentrum_Cholesky/Laion/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_ssd=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Laion/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Laion/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Laion/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Laion/Metacentrum.py \ No newline at end of file diff --git "a/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum - k\303\263pia.py" "b/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum - k\303\263pia.py" index 193859d..a70f737 100644 --- "a/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum - k\303\263pia.py" +++ "b/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum - k\303\263pia.py" @@ -39,7 +39,7 @@ from scipy import stats # for describe, spearmanr, kendalltau # ------------------------- Logging like before ------------------------- -RESULTS_ROOT = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/profimedia" +RESULTS_ROOT = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/profimedia" os.makedirs(RESULTS_ROOT, exist_ok=True) log_file_path = os.path.join(RESULTS_ROOT, "output_log.txt") log_file = open(log_file_path, "w", buffering=1) @@ -53,7 +53,7 @@ print("=== Logging started ===") USE_PREVIOUS_QUERIES = True # set False to keep the old random-sampling behavior -QUERY_ROOT_DIRECTORY = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/Semantic ax" +QUERY_ROOT_DIRECTORY = "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/Semantic ax" # QUERY_AXES = ["Activities"] # same idea as your semantic_axes_list QUERY_AXES = [ "Activities", @@ -155,45 +155,45 @@ def load_previous_queries(root_directory: str, # Two datasets (edit paths). Same shard layout as before. DATASETS = [ # LAION (unchanged) - # ("LAION2B-en", "/storage/plzen1/home/xsikyna/PhD/laion/laionDir") + # ("LAION2B-en", "/storage/plzen1/home/anonymous/PhD/laion/laionDir") # Profimedia (NEW) ("Profimedia", { "type": "profimedia", "filenames": [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ], # optional: where to reuse/store the 1k random query indices - "selected_indices_path": "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/profimedia/elected_indices.npy", + "selected_indices_path": "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/profimedia/elected_indices.npy", "max_items": 500000 }) ] @@ -212,15 +212,15 @@ def load_previous_queries(root_directory: str, # Matrices: reuse your previous ones (edit paths if needed) methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] # ------------------------- Shard helpers (same spirit as before) ------------------------- diff --git a/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py b/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py index 700639c..82eaf9d 100644 --- a/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py +++ b/Implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py @@ -39,7 +39,7 @@ from scipy import stats # for describe, spearmanr, kendalltau # ------------------------- Logging like before ------------------------- -RESULTS_ROOT = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/profimedia" +RESULTS_ROOT = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/profimedia" os.makedirs(RESULTS_ROOT, exist_ok=True) log_file_path = os.path.join(RESULTS_ROOT, "output_log.txt") log_file = open(log_file_path, "w", buffering=1) @@ -53,7 +53,7 @@ print("=== Logging started ===") USE_PREVIOUS_QUERIES = True # set False to keep the old random-sampling behavior -QUERY_ROOT_DIRECTORY = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/Semantic ax" +QUERY_ROOT_DIRECTORY = "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/Semantic ax" # QUERY_AXES = ["Activities"] # same idea as your semantic_axes_list QUERY_AXES = [ "Activities", @@ -166,45 +166,45 @@ def load_previous_queries(root_directory: str, # Two datasets (edit paths). Same shard layout as before. DATASETS = [ # LAION (unchanged) - # ("LAION2B-en", "/storage/plzen1/home/xsikyna/PhD/laion/laionDir") + # ("LAION2B-en", "/storage/plzen1/home/anonymous/PhD/laion/laionDir") # Profimedia (NEW) ("Profimedia", { "type": "profimedia", "filenames": [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ], # optional: where to reuse/store the 1k random query indices - "selected_indices_path": "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/cholesky/profimedia/elected_indices.npy", + "selected_indices_path": "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/cholesky/profimedia/elected_indices.npy", "max_items": 500000 }) ] @@ -223,15 +223,15 @@ def load_previous_queries(root_directory: str, # Matrices: reuse your previous ones (edit paths if needed) methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), - ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))), + ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] # ------------------------- Shard helpers (same spirit as before) ------------------------- diff --git a/Implementation/Metacentrum_Cholesky/Profimedia/run_experiment.sh b/Implementation/Metacentrum_Cholesky/Profimedia/run_experiment.sh index 1237e98..519bfcf 100755 --- a/Implementation/Metacentrum_Cholesky/Profimedia/run_experiment.sh +++ b/Implementation/Metacentrum_Cholesky/Profimedia/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=96:0:0 #PBS -l select=1:ncpus=48:mem=600gb:scratch_ssd=400gb -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Profimedia/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Profimedia/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_Cholesky/Profimedia/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py b/Implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py index 47047d2..e2860df 100644 --- a/Implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -301,15 +301,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["AutofaissIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -328,7 +328,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -460,7 +460,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -481,7 +481,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -621,7 +621,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissIP/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Laion/AutofaissIP/run_experiment.sh b/Implementation/Metacentrum_approximate/Laion/AutofaissIP/run_experiment.sh index cb06e92..55ba151 100755 --- a/Implementation/Metacentrum_approximate/Laion/AutofaissIP/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Laion/AutofaissIP/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=32:mem=700gb:scratch_local=400gb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissIP/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissIP/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissIP/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py b/Implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py index 60fc40a..a99b14d 100644 --- a/Implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -301,15 +301,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["AutofaissL2"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -328,7 +328,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -460,7 +460,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -481,7 +481,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -621,7 +621,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/AutofaissL2/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Laion/AutofaissL2/run_experiment.sh b/Implementation/Metacentrum_approximate/Laion/AutofaissL2/run_experiment.sh index b481a2c..17b9a17 100755 --- a/Implementation/Metacentrum_approximate/Laion/AutofaissL2/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Laion/AutofaissL2/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=32:mem=700gb:scratch_local=400gb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissL2/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissL2/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/AutofaissL2/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py b/Implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py index 29363b4..5d838f2 100644 --- a/Implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -258,7 +258,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -301,15 +301,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatIP", "FlatL2"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -328,7 +328,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -460,7 +460,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/Flat" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/Flat" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -481,7 +481,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -621,7 +621,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Laion/Flat/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Laion/Flat/run_experiment.sh b/Implementation/Metacentrum_approximate/Laion/Flat/run_experiment.sh index aceaaff..8ba6ee7 100755 --- a/Implementation/Metacentrum_approximate/Laion/Flat/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Laion/Flat/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/Flat/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/Flat/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Laion/Flat/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py b/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py index 6d59197..0565326 100644 --- a/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -44,36 +44,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -109,8 +109,8 @@ N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/selected_indices.npy') import os, re, glob, random from typing import List, Tuple, Optional, Sequence @@ -327,7 +327,7 @@ def peek_laion( print(f" {s}") print() -# peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +# peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -370,15 +370,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["AutofaissIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -397,7 +397,7 @@ def calculate_scaling_factor(A): # dataset_name = "LAION2B-en" # will appear in CSV and index filenames # # Point this to the folder that contains both file types -# LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +# LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" # # feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( # dirpath=LAION_DIR, @@ -531,7 +531,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -552,7 +552,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -692,7 +692,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissIP/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/run_experiment.sh b/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/run_experiment.sh index bcbc62c..ec847c0 100755 --- a/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Profimedia/AutofaissIP/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=32:mem=700gb:scratch_local=400gb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissIP/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissIP/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissIP/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py b/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py index 05eeea4..029afc6 100644 --- a/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -44,36 +44,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -109,8 +109,8 @@ N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/selected_indices.npy') import os, re, glob, random from typing import List, Tuple, Optional, Sequence @@ -327,7 +327,7 @@ def peek_laion( print(f" {s}") print() -# peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +# peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -370,15 +370,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["AutofaissL2"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -397,7 +397,7 @@ def calculate_scaling_factor(A): # dataset_name = "LAION2B-en" # will appear in CSV and index filenames # # Point this to the folder that contains both file types -# LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +# LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" # # feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( # dirpath=LAION_DIR, @@ -531,7 +531,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -552,7 +552,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -692,7 +692,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/AutofaissL2/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/run_experiment.sh b/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/run_experiment.sh index b5b81e3..f3bf969 100755 --- a/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Profimedia/AutofaissL2/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=32:mem=700gb:scratch_local=400gb:spec=11.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissL2/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissL2/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/AutofaissL2/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py b/Implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py index b39097e..71201bb 100644 --- a/Implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py +++ b/Implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py @@ -23,7 +23,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -44,36 +44,36 @@ warnings.filterwarnings("ignore", category=FutureWarning) filenames = [ - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/369.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/368.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/367.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/366.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/365.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/364.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/363.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/362.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/361.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/360.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/359.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/358.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/357.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/356.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/355.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/354.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/353.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/352.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/351.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/350.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/349.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/348.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/347.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/346.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/345.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/344.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/343.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/342.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/341.txt", - "/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors/340.txt" + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/369.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/368.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/367.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/366.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/365.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/364.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/363.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/362.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/361.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/360.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/359.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/358.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/357.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/356.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/355.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/354.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/353.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/352.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/351.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/350.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/349.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/348.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/347.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/346.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/345.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/344.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/343.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/342.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/341.txt", + "/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors/340.txt" ] # Create an empty dictionary to store the feature vectors @@ -109,8 +109,8 @@ N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/selected_indices.npy') import os, re, glob, random from typing import List, Tuple, Optional, Sequence @@ -327,7 +327,7 @@ def peek_laion( print(f" {s}") print() -# peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +# peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -370,15 +370,15 @@ def calculate_scaling_factor(A): # indices = ["AutofaissL2", "AutofaissIP", "FlatL2", "FlatIP"] indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -397,7 +397,7 @@ def calculate_scaling_factor(A): # dataset_name = "LAION2B-en" # will appear in CSV and index filenames # # Point this to the folder that contains both file types -# LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +# LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" # # feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( # dirpath=LAION_DIR, @@ -531,7 +531,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -552,7 +552,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -692,7 +692,7 @@ def describe_array_nan(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/approximate/Profimedia/Flat/results.csv' FIELDNAMES = [ # NOTEEE 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params', diff --git a/Implementation/Metacentrum_approximate/Profimedia/Flat/run_experiment.sh b/Implementation/Metacentrum_approximate/Profimedia/Flat/run_experiment.sh index cf9e409..acc1a29 100755 --- a/Implementation/Metacentrum_approximate/Profimedia/Flat/run_experiment.sh +++ b/Implementation/Metacentrum_approximate/Profimedia/Flat/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=16:mem=400gb:scratch_local=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/Flat/ +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/Flat/ export PYTHONPATH=$PYTHONPATH:../../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_approximate/Profimedia/Flat/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py b/Implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py index c68a6b3..2d49492 100644 --- a/Implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py +++ b/Implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/selected_indices.npy') k_list = [10, 100] growth_factors = [1.0] @@ -124,15 +124,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -277,7 +277,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -299,7 +299,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -413,7 +413,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissIP/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_full_profimedia/AutofaissIP/run_experiment.sh b/Implementation/Metacentrum_full_profimedia/AutofaissIP/run_experiment.sh index 8394a6a..2830a0d 100755 --- a/Implementation/Metacentrum_full_profimedia/AutofaissIP/run_experiment.sh +++ b/Implementation/Metacentrum_full_profimedia/AutofaissIP/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=1:mem=1200gb:scratch_ssd=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissIP/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissIP/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissIP/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py b/Implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py index 0a8d740..27b758c 100644 --- a/Implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py +++ b/Implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/selected_indices.npy') k_list = [10, 100] growth_factors = [1.0] @@ -125,15 +125,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -271,7 +271,7 @@ def index_identity_key(dataset_name, index_name, size, params): def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -292,7 +292,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -406,7 +406,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/AutofaissL2/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_full_profimedia/AutofaissL2/run_experiment.sh b/Implementation/Metacentrum_full_profimedia/AutofaissL2/run_experiment.sh index d4d9cc2..426de7b 100755 --- a/Implementation/Metacentrum_full_profimedia/AutofaissL2/run_experiment.sh +++ b/Implementation/Metacentrum_full_profimedia/AutofaissL2/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=72:0:0 #PBS -l select=1:ncpus=1:mem=1100gb:scratch_local=400gb -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissL2/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissL2/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/AutofaissL2/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py b/Implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py index 2ccd4b1..271f932 100644 --- a/Implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py +++ b/Implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat/selected_indices.npy') k_list = [10, 100] growth_factors = [1.0] @@ -124,15 +124,15 @@ def calculate_scaling_factor(A): indices = ["FlatL2", "FlatIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -183,7 +183,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -202,7 +202,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -294,7 +294,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/Flat/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/Flat/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_full_profimedia/Flat/run_experiment.sh b/Implementation/Metacentrum_full_profimedia/Flat/run_experiment.sh index 0488b2d..48e154f 100755 --- a/Implementation/Metacentrum_full_profimedia/Flat/run_experiment.sh +++ b/Implementation/Metacentrum_full_profimedia/Flat/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=1:mem=1200gb:scratch_local=400mb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Flat/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Flat/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Flat/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_full_profimedia/Metacentrum.py b/Implementation/Metacentrum_full_profimedia/Metacentrum.py index a6f3c52..1ee36ac 100644 --- a/Implementation/Metacentrum_full_profimedia/Metacentrum.py +++ b/Implementation/Metacentrum_full_profimedia/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') k_list = [100] growth_factors = [1.0] @@ -124,15 +124,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -183,7 +183,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -202,7 +202,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -280,7 +280,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_full_profimedia/run_experiment.sh b/Implementation/Metacentrum_full_profimedia/run_experiment.sh index 71a53a6..4912809 100755 --- a/Implementation/Metacentrum_full_profimedia/run_experiment.sh +++ b/Implementation/Metacentrum_full_profimedia/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=4:mem=1600gb:scratch_local=400mb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ export PYTHONPATH=$PYTHONPATH:.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py b/Implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py index c18f5b2..cf61018 100644 --- a/Implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py +++ b/Implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/AutofaissIP/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion/AutofaissIP/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "8")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissIP/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissIP/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -316,7 +316,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -361,15 +361,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -395,7 +395,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -574,7 +574,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissIP" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissIP" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -723,7 +723,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissIP/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissIP/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion/AutofaissIP/run_experiment.sh b/Implementation/Metacentrum_laion/AutofaissIP/run_experiment.sh index e14f531..1fa2958 100755 --- a/Implementation/Metacentrum_laion/AutofaissIP/run_experiment.sh +++ b/Implementation/Metacentrum_laion/AutofaissIP/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=8:mem=1200gb:scratch_ssd=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissIP/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissIP/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissIP/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py b/Implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py index 133be6c..a37b2a0 100644 --- a/Implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py +++ b/Implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/AutofaissL2/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion/AutofaissL2/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "8")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissL2/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissL2/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -316,7 +316,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -361,15 +361,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -395,7 +395,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -574,7 +574,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissL2" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissL2" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -723,7 +723,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/AutofaissL2/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/AutofaissL2/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion/AutofaissL2/run_experiment.sh b/Implementation/Metacentrum_laion/AutofaissL2/run_experiment.sh index 864a31b..83db472 100755 --- a/Implementation/Metacentrum_laion/AutofaissL2/run_experiment.sh +++ b/Implementation/Metacentrum_laion/AutofaissL2/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=8:mem=1200gb:scratch_ssd=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissL2/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissL2/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/AutofaissL2/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion/Flat/Metacentrum.py b/Implementation/Metacentrum_laion/Flat/Metacentrum.py index 7edde3c..5ec12b6 100644 --- a/Implementation/Metacentrum_laion/Flat/Metacentrum.py +++ b/Implementation/Metacentrum_laion/Flat/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/Flat/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion/Flat/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "16")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/Flat/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/Flat/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -304,7 +304,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -348,15 +348,15 @@ def calculate_scaling_factor(A): indices = ["FlatL2", "FlatIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -375,7 +375,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -505,7 +505,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/Flat" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/Flat" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -634,7 +634,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion/Flat/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion/Flat/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion/Flat/run_experiment.sh b/Implementation/Metacentrum_laion/Flat/run_experiment.sh index 5e38d9b..9678244 100755 --- a/Implementation/Metacentrum_laion/Flat/run_experiment.sh +++ b/Implementation/Metacentrum_laion/Flat/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=1:mem=1200gb:scratch_local=400mb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/Flat/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/Flat/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/Flat/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion/Flat/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion/Metacentrum.py b/Implementation/Metacentrum_laion/Metacentrum.py index a6f3c52..1ee36ac 100644 --- a/Implementation/Metacentrum_laion/Metacentrum.py +++ b/Implementation/Metacentrum_laion/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') k_list = [100] growth_factors = [1.0] @@ -124,15 +124,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -183,7 +183,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -202,7 +202,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -280,7 +280,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion/run_experiment.sh b/Implementation/Metacentrum_laion/run_experiment.sh index 71a53a6..4912809 100755 --- a/Implementation/Metacentrum_laion/run_experiment.sh +++ b/Implementation/Metacentrum_laion/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=4:mem=1600gb:scratch_local=400mb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ export PYTHONPATH=$PYTHONPATH:.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py b/Implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py index 70eb4f9..af4ee96 100644 --- a/Implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py +++ b/Implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion_large/AutofaissIP/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion_large/AutofaissIP/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "8")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -316,7 +316,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -361,15 +361,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -395,7 +395,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -574,7 +574,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -723,7 +723,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissIP/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion_large/AutofaissIP/run_experiment.sh b/Implementation/Metacentrum_laion_large/AutofaissIP/run_experiment.sh index 7c10d54..9ad91e7 100755 --- a/Implementation/Metacentrum_laion_large/AutofaissIP/run_experiment.sh +++ b/Implementation/Metacentrum_laion_large/AutofaissIP/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=72:0:0 #PBS -l select=1:ncpus=8:mem=1000gb:scratch_local=400gb -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissIP/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissIP/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissIP/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py b/Implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py index 0176cdb..63add0c 100644 --- a/Implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py +++ b/Implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion_large/AutofaissL2/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion_large/AutofaissL2/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "32")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -316,7 +316,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -361,15 +361,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -395,7 +395,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -574,7 +574,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -723,7 +723,7 @@ def describe_array(data, which): return out -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/AutofaissL2/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion_large/AutofaissL2/run_experiment.sh b/Implementation/Metacentrum_laion_large/AutofaissL2/run_experiment.sh index 63308d3..bc44ed9 100755 --- a/Implementation/Metacentrum_laion_large/AutofaissL2/run_experiment.sh +++ b/Implementation/Metacentrum_laion_large/AutofaissL2/run_experiment.sh @@ -3,13 +3,13 @@ #PBS -l walltime=96:0:0 #PBS -l select=1:ncpus=32:mem=1000gb:scratch_local=400gb -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissL2/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissL2/ export PYTHONPATH=$PYTHONPATH:../.. export TMPDIR=$SCRATCHDIR -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/AutofaissL2/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion_large/Flat/Metacentrum.py b/Implementation/Metacentrum_laion_large/Flat/Metacentrum.py index dd5ec3b..7580635 100644 --- a/Implementation/Metacentrum_laion_large/Flat/Metacentrum.py +++ b/Implementation/Metacentrum_laion_large/Flat/Metacentrum.py @@ -26,7 +26,7 @@ # --- NEW: knobs for big runs (place near imports) --- CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap) -EMB_MEMMAP_DIR = "/storage/plzen1/home/xsikyna/PhD/laion_large/Flat/memmap" +EMB_MEMMAP_DIR = "/storage/plzen1/home/anonymous/PhD/laion_large/Flat/memmap" EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB) FAISS_THREADS = int(os.environ.get("FAISS_THREADS", "16")) @@ -75,7 +75,7 @@ def build_or_open_memmap_from_shards(dirpath: str, target_size: int, dtype=np.fl emb_mm = np.memmap(mmap_path, mode="r", dtype=dtype, shape=shape) return emb_mm, mmap_path -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/Flat/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/Flat/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -304,7 +304,7 @@ def peek_laion( print(f" {s}") print() -peek_laion("/storage/plzen1/home/xsikyna/PhD/laion/laionDir", shards=2, rows=5) +peek_laion("/storage/plzen1/home/anonymous/PhD/laion/laionDir", shards=2, rows=5) def euclidean_distances(target_vector, feature_vectors_keys, feature_vectors_values): # Compute differences all at once @@ -349,15 +349,15 @@ def calculate_scaling_factor(A): indices = ["FlatL2", "FlatIP"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) - # ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))) + # ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -376,7 +376,7 @@ def calculate_scaling_factor(A): dataset_name = "LAION2B-en" # will appear in CSV and index filenames # Point this to the folder that contains both file types -LAION_DIR = "/storage/plzen1/home/xsikyna/PhD/laion/laionDir" +LAION_DIR = "/storage/plzen1/home/anonymous/PhD/laion/laionDir" feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline( dirpath=LAION_DIR, @@ -506,7 +506,7 @@ def index_identity_key(dataset_name, index_name, size, params): _emb_file_for_ident = {} def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/Flat" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/Flat" os.makedirs(base_dir, exist_ok=True) tag_needed = (index_name in ("IVFPQ", "AutofaissL2", "AutofaissIP")) @@ -635,7 +635,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/laion_large/Flat/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/laion_large/Flat/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion_large/Flat/run_experiment.sh b/Implementation/Metacentrum_laion_large/Flat/run_experiment.sh index fd5686f..8130360 100755 --- a/Implementation/Metacentrum_laion_large/Flat/run_experiment.sh +++ b/Implementation/Metacentrum_laion_large/Flat/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=72:0:0 #PBS -l select=1:ncpus=16:mem=1100gb:scratch_local=400gb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/Flat/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/Flat/ export PYTHONPATH=$PYTHONPATH:../.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/Flat/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_laion_large/Flat/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Metacentrum_laion_large/Metacentrum.py b/Implementation/Metacentrum_laion_large/Metacentrum.py index a6f3c52..1ee36ac 100644 --- a/Implementation/Metacentrum_laion_large/Metacentrum.py +++ b/Implementation/Metacentrum_laion_large/Metacentrum.py @@ -22,7 +22,7 @@ import threading from concurrent.futures import ThreadPoolExecutor, as_completed -log_file_path = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" +log_file_path = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/output_log.txt" log_file = open(log_file_path, "w", buffering=1) sys.stdout = log_file sys.stderr = log_file @@ -43,7 +43,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) from pathlib import Path -base = Path("/storage/brno2/home/xsikyna/PhD/metric_learning_exp/dataset/vectors") +base = Path("/storage/brno2/home/anonymous/PhD/metric_learning_exp/dataset/vectors") filenames = sorted(str(p) for p in base.glob("*.txt")) # Create an empty dictionary to store the feature vectors @@ -114,8 +114,8 @@ def calculate_scaling_factor(A): N = len(feature_vectors_values) rng = np.random.default_rng(seed=42) selected_indices = rng.choice(N, size=1000, replace=False) -np.save('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) -selected_indices = np.load('/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') +np.save('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy', selected_indices) +selected_indices = np.load('/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/selected_indices.npy') k_list = [100] growth_factors = [1.0] @@ -124,15 +124,15 @@ def calculate_scaling_factor(A): indices = ["AutofaissL2"] # indices = ["FlatL2", "FlatIP"] methods = [ - ("Balance", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), - # ("Balance+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), - # ("Balance+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), - ("Min", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), - # ("Min+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), - # ("Min+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), - # ("Max", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), - ("Max+ALT", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) - # ("Max+Corr", np.load(open("/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) + ("Balance", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance.npy", 'rb'))), + # ("Balance+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+ALT.npy", 'rb'))), + # ("Balance+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Balance+Corr.npy", 'rb'))), + ("Min", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min.npy", 'rb'))), + # ("Min+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+ALT.npy", 'rb'))), + # ("Min+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Min+Corr.npy", 'rb'))), + # ("Max", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max.npy", 'rb'))), + ("Max+ALT", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+ALT.npy", 'rb'))) + # ("Max+Corr", np.load(open("/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/inputs/Max+Corr.npy", 'rb'))) ] IVFPQ_PARAM_GRID = { @@ -183,7 +183,7 @@ def params_tag(params: dict) -> str: def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict): """Build and persist the FAISS index if missing; return loaded index and the final filename used.""" - base_dir = "/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia" + base_dir = "/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia" os.makedirs(base_dir, exist_ok=True) # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params) @@ -202,7 +202,7 @@ def make_or_load_index(dataset_name: str, index_name: str, size: int, params: di ids = feature_vectors_keys.astype(str) emb = data - emb_path = f"/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" + emb_path = f"/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/autofaiss_data/{dataset_name}/" os.makedirs(emb_path, exist_ok=True) emb_file = os.path.join(emb_path, f"{size}_embeddings.npy") @@ -280,7 +280,7 @@ def describe_array(data, which): which + '_IQR': pct[3] - pct[1], } -CSV_PATH = '/storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' +CSV_PATH = '/storage/brno2/home/anonymous/PhD/metric_learning_exp_3/results/full_profimedia/results.csv' FIELDNAMES = [ 'dataset','method','k','growth', 'dataset_size', 'scaling_factor', 'index', 'index_params', 'count_superset', diff --git a/Implementation/Metacentrum_laion_large/run_experiment.sh b/Implementation/Metacentrum_laion_large/run_experiment.sh index 71a53a6..4912809 100755 --- a/Implementation/Metacentrum_laion_large/run_experiment.sh +++ b/Implementation/Metacentrum_laion_large/run_experiment.sh @@ -3,12 +3,12 @@ #PBS -l walltime=24:0:0 #PBS -l select=1:ncpus=4:mem=1600gb:scratch_local=400mb:spec=7.0 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp module add mambaforge mamba init -source /storage/brno2/home/xsikyna/.bashrc -mamba activate /storage/brno2/home/xsikyna/PhD/metric_learning_exp/faiss312 +source /storage/brno2/home/anonymous/.bashrc +mamba activate /storage/brno2/home/anonymous/PhD/metric_learning_exp/faiss312 -cd /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ +cd /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/ export PYTHONPATH=$PYTHONPATH:.. -python /storage/brno2/home/xsikyna/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file +python /storage/brno2/home/anonymous/PhD/metric_learning_exp_3/implementation/Metacentrum_full_profimedia/Metacentrum.py \ No newline at end of file diff --git a/Implementation/Untitled.ipynb b/Implementation/Untitled.ipynb index 0fd101b..1978013 100644 --- a/Implementation/Untitled.ipynb +++ b/Implementation/Untitled.ipynb @@ -49,20 +49,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "Finished processing /Users/matussikyna/PhD local/Paper2/dataset/combined_vectors.txt\n", - "Finished processing /Users/matussikyna/PhD local/Paper2/dataset/369.txt\n", + "Finished processing /Users/anonymous/PhD local/Paper2/dataset/combined_vectors.txt\n", + "Finished processing /Users/anonymous/PhD local/Paper2/dataset/369.txt\n", "768 52483\n" ] } ], "source": [ "filenames = [\n", - " \"/Users/matussikyna/PhD local/Paper2/dataset/combined_vectors.txt\",\n", - " \"/Users/matussikyna/PhD local/Paper2/dataset/369.txt\"\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/201.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/202.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/203.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/204.txt\"\n", + " \"/Users/anonymous/PhD local/Paper2/dataset/combined_vectors.txt\",\n", + " \"/Users/anonymous/PhD local/Paper2/dataset/369.txt\"\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/201.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/202.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/203.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/204.txt\"\n", "]\n", "\n", "# Create an empty dictionary to store the feature vectors\n", @@ -135,15 +135,15 @@ "outputs": [], "source": [ "methods = [\n", - " # (\"Balance\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", - " # (\"Balance+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " # (\"Balance+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", - " # (\"Min\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", - " # (\"Min+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " # (\"Min+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", - " # (\"Max\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", - " (\"Max+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " # (\"Max+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", + " # (\"Balance\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", + " # (\"Balance+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " # (\"Balance+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", + " # (\"Min\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", + " # (\"Min+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " # (\"Min+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", + " # (\"Max\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", + " (\"Max+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " # (\"Max+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", "]\n", "\n", "ideal_r_index = 100" @@ -165,9 +165,9 @@ "selected_indices = rng.choice(N, size=1000, replace=False)\n", "\n", "# # as a compact binary .npy file\n", - "np.save('/Users/matussikyna/PhD local/Paper2/dataset/Verify/selected_indices.npy', selected_indices)\n", + "np.save('/Users/anonymous/PhD local/Paper2/dataset/Verify/selected_indices.npy', selected_indices)\n", "\n", - "selected_indices = np.load('/Users/matussikyna/PhD local/Paper2/dataset/Verify/selected_indices.npy')" + "selected_indices = np.load('/Users/anonymous/PhD local/Paper2/dataset/Verify/selected_indices.npy')" ] }, { @@ -186,9 +186,9 @@ "# ids = feature_vectors_keys.astype(str)\n", "# emb = np.asarray(feature_vectors_values, dtype=np.float32)\n", "\n", - "# emb_path = \"/Users/matussikyna/PhD local/Paper3/results/autofaiss_data/\"\n", - "# index_path = \"/Users/matussikyna/PhD local/Paper3/results/autofaiss_data/autofaiss_data/my.index\"\n", - "# infos_path = \"/Users/matussikyna/PhD local/Paper3/results/autofaiss_data/autofaiss_data/my.index.json\"\n", + "# emb_path = \"/Users/anonymous/PhD local/Paper3/results/autofaiss_data/\"\n", + "# index_path = \"/Users/anonymous/PhD local/Paper3/results/autofaiss_data/autofaiss_data/my.index\"\n", + "# infos_path = \"/Users/anonymous/PhD local/Paper3/results/autofaiss_data/autofaiss_data/my.index.json\"\n", "\n", "# os.makedirs(emb_path, exist_ok=True)\n", "# np.save(emb_path + \"embeddings.npy\", emb) # or save 'emb' if using L2 metric\n", @@ -1120,7 +1120,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found 2 shard pair(s) under: /Users/matussikyna/PhD local/Paper3/laion/laionDir\n", + "Found 2 shard pair(s) under: /Users/anonymous/PhD local/Paper3/laion/laionDir\n", "\n", "================================================================================\n", "Shard token : 0000\n", @@ -1444,7 +1444,7 @@ " print(f\" {s}\")\n", " print()\n", "\n", - "peek_laion(\"/Users/matussikyna/PhD local/Paper3/laion/laionDir\", shards=2, rows=5)" + "peek_laion(\"/Users/anonymous/PhD local/Paper3/laion/laionDir\", shards=2, rows=5)" ] }, { @@ -1466,9 +1466,9 @@ "text": [ "/var/folders/fq/2vl6lxqx37d2246960sj33xm0000gn/T/ipykernel_77822/1807501945.py:359: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " desc = stats.describe(valid, ddof=1)\n", - "/Users/matussikyna/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1405: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", + "/Users/anonymous/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1405: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " sk = skew(a, axis, bias=bias)\n", - "/Users/matussikyna/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1406: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", + "/Users/anonymous/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1406: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " kurt = kurtosis(a, axis, bias=bias)\n" ] }, @@ -1723,15 +1723,15 @@ "k_additional = [0, 1, 2, 5, 10, 20]\n", "\n", "methods = [\n", - " # (\"Balance\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", - " # (\"Balance+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " # (\"Balance+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", - " # (\"Min\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", - " # (\"Min+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " # (\"Min+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", - " (\"Max\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", - " (\"Max+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Max+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", + " # (\"Balance\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", + " # (\"Balance+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " # (\"Balance+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", + " # (\"Min\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", + " # (\"Min+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " # (\"Min+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", + " (\"Max\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", + " (\"Max+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Max+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", "]\n", "\n", "indices = [\"FlatIP\"]\n", @@ -1740,7 +1740,7 @@ "\n", "dataset_name = \"LAION2B-en\" # will appear in CSV and index filenames\n", "# Point this to the folder that contains both file types\n", - "LAION_DIR = \"/Users/matussikyna/PhD local/Paper3/laion/laionDir\"\n", + "LAION_DIR = \"/Users/anonymous/PhD local/Paper3/laion/laionDir\"\n", "\n", "feature_vectors_keys, feature_vectors_values, selected_indices = load_laion_for_pipeline(\n", " dirpath=LAION_DIR,\n", @@ -1869,7 +1869,7 @@ "\n", "def make_or_load_index(dataset_name: str, index_name: str, size: int, params: dict):\n", " \"\"\"Build and persist the FAISS index if missing; return loaded index and the final filename used.\"\"\"\n", - " base_dir = \"/Users/matussikyna/PhD local/Paper3/results\"\n", + " base_dir = \"/Users/anonymous/PhD local/Paper3/results\"\n", " os.makedirs(base_dir, exist_ok=True)\n", "\n", " # Include params in filename for IVFPQ and Autofaiss* (Flat stays without params)\n", @@ -1890,7 +1890,7 @@ " ids = feature_vectors_keys.astype(str)\n", " emb = data\n", " \n", - " emb_path = f\"/Users/matussikyna/PhD local/Paper3/results/autofaiss_data/{dataset_name}/\"\n", + " emb_path = f\"/Users/anonymous/PhD local/Paper3/results/autofaiss_data/{dataset_name}/\"\n", " os.makedirs(emb_path, exist_ok=True)\n", " \n", " emb_file = os.path.join(emb_path, f\"{size}_embeddings.npy\")\n", @@ -2032,7 +2032,7 @@ " }\n", "\n", "\n", - "CSV_PATH = '/Users/matussikyna/PhD local/Paper3/results/results.csv'\n", + "CSV_PATH = '/Users/anonymous/PhD local/Paper3/results/results.csv'\n", "FIELDNAMES = [\n", " # NOTEEE\n", " 'dataset','method','k', 'growth', 'k_additional', 'dataset_size', 'scaling_factor', 'index', 'index_params',\n", @@ -2479,7 +2479,7 @@ } ], "source": [ - "with open('/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy', 'rb') as f:\n", + "with open('/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy', 'rb') as f:\n", " M = np.load(f)\n", "print(calculate_scaling_factor(M))" ] @@ -2496,13 +2496,13 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m text_query \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mloadtxt(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/matussikyna/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, delimiter\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m euclidean_nearest_indices, euclidean_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_euclidean(text_query, \u001b[43mdataset\u001b[49m, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n\u001b[1;32m 3\u001b[0m mahalanobis_nearest_indices, mahalanobis_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_mahalanobis(text_query, dataset, M, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n", + "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m text_query \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mloadtxt(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/anonymous/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, delimiter\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m euclidean_nearest_indices, euclidean_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_euclidean(text_query, \u001b[43mdataset\u001b[49m, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n\u001b[1;32m 3\u001b[0m mahalanobis_nearest_indices, mahalanobis_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_mahalanobis(text_query, dataset, M, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n", "\u001b[0;31mNameError\u001b[0m: name 'dataset' is not defined" ] } ], "source": [ - "text_query = np.loadtxt('/Users/matussikyna/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt', delimiter=\",\")\n", + "text_query = np.loadtxt('/Users/anonymous/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt', delimiter=\",\")\n", "euclidean_nearest_indices, euclidean_nearest_distances = nearest_indices_euclidean(text_query, dataset, num_indices=len(dataset))\n", "mahalanobis_nearest_indices, mahalanobis_nearest_distances = nearest_indices_mahalanobis(text_query, dataset, M, num_indices=len(dataset))" ] @@ -2541,7 +2541,7 @@ "outputs": [], "source": [ "plot_images_with_rank_changes(\n", - " image_path='/Users/matussikyna/PhD local/Paper2/dataset/1Place/',\n", + " image_path='/Users/anonymous/PhD local/Paper2/dataset/1Place/',\n", " image_names=image_names,\n", " nearest_indices=mahalanobis_nearest_indices,\n", " nearest_distances=mahalanobis_nearest_distances,\n", diff --git a/Implementation/Untitled_Cholesky.ipynb b/Implementation/Untitled_Cholesky.ipynb index e052054..6424119 100644 --- a/Implementation/Untitled_Cholesky.ipynb +++ b/Implementation/Untitled_Cholesky.ipynb @@ -49,20 +49,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "Finished processing /Users/matussikyna/PhD local/Paper2/dataset/combined_vectors.txt\n", - "Finished processing /Users/matussikyna/PhD local/Paper2/dataset/369.txt\n", + "Finished processing /Users/anonymous/PhD local/Paper2/dataset/combined_vectors.txt\n", + "Finished processing /Users/anonymous/PhD local/Paper2/dataset/369.txt\n", "768 52483\n" ] } ], "source": [ "filenames = [\n", - " \"/Users/matussikyna/PhD local/Paper2/dataset/combined_vectors.txt\",\n", - " \"/Users/matussikyna/PhD local/Paper2/dataset/369.txt\"\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/201.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/202.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/203.txt\",\n", - "# \"/Users/matussikyna/PhD local/Paper2/dataset/204.txt\"\n", + " \"/Users/anonymous/PhD local/Paper2/dataset/combined_vectors.txt\",\n", + " \"/Users/anonymous/PhD local/Paper2/dataset/369.txt\"\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/201.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/202.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/203.txt\",\n", + "# \"/Users/anonymous/PhD local/Paper2/dataset/204.txt\"\n", "]\n", "\n", "# Create an empty dictionary to store the feature vectors\n", @@ -135,15 +135,15 @@ "outputs": [], "source": [ "methods = [\n", - " (\"Balance\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", - " (\"Balance+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Balance+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", - " (\"Min\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", - " (\"Min+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Min+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", - " (\"Max\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", - " (\"Max+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Max+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", + " (\"Balance\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", + " (\"Balance+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Balance+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", + " (\"Min\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", + " (\"Min+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Min+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", + " (\"Max\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", + " (\"Max+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Max+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", "]\n", "\n", "ideal_r_index = 100" @@ -165,9 +165,9 @@ "selected_indices = rng.choice(N, size=1000, replace=False)\n", "\n", "# # as a compact binary .npy file\n", - "np.save('/Users/matussikyna/PhD local/Paper2/dataset/Verify/selected_indices.npy', selected_indices)\n", + "np.save('/Users/anonymous/PhD local/Paper2/dataset/Verify/selected_indices.npy', selected_indices)\n", "\n", - "selected_indices = np.load('/Users/matussikyna/PhD local/Paper2/dataset/Verify/selected_indices.npy')" + "selected_indices = np.load('/Users/anonymous/PhD local/Paper2/dataset/Verify/selected_indices.npy')" ] }, { @@ -231,7 +231,7 @@ } ], "source": [ - "with open('/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy', 'rb') as f:\n", + "with open('/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy', 'rb') as f:\n", " M = np.load(f)\n", "print(calculate_scaling_factor(M))" ] @@ -248,13 +248,13 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m text_query \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mloadtxt(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/matussikyna/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, delimiter\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m euclidean_nearest_indices, euclidean_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_euclidean(text_query, \u001b[43mdataset\u001b[49m, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n\u001b[1;32m 3\u001b[0m mahalanobis_nearest_indices, mahalanobis_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_mahalanobis(text_query, dataset, M, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n", + "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m text_query \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mloadtxt(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/anonymous/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, delimiter\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m euclidean_nearest_indices, euclidean_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_euclidean(text_query, \u001b[43mdataset\u001b[49m, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n\u001b[1;32m 3\u001b[0m mahalanobis_nearest_indices, mahalanobis_nearest_distances \u001b[38;5;241m=\u001b[39m nearest_indices_mahalanobis(text_query, dataset, M, num_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(dataset))\n", "\u001b[0;31mNameError\u001b[0m: name 'dataset' is not defined" ] } ], "source": [ - "text_query = np.loadtxt('/Users/matussikyna/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt', delimiter=\",\")\n", + "text_query = np.loadtxt('/Users/anonymous/PhD local/Paper2/dataset/Semantic ax/Cultural/q16_chinese_dragon/query/chinese dragon.txt', delimiter=\",\")\n", "euclidean_nearest_indices, euclidean_nearest_distances = nearest_indices_euclidean(text_query, dataset, num_indices=len(dataset))\n", "mahalanobis_nearest_indices, mahalanobis_nearest_distances = nearest_indices_mahalanobis(text_query, dataset, M, num_indices=len(dataset))" ] @@ -293,7 +293,7 @@ "outputs": [], "source": [ "plot_images_with_rank_changes(\n", - " image_path='/Users/matussikyna/PhD local/Paper2/dataset/1Place/',\n", + " image_path='/Users/anonymous/PhD local/Paper2/dataset/1Place/',\n", " image_names=image_names,\n", " nearest_indices=mahalanobis_nearest_indices,\n", " nearest_distances=mahalanobis_nearest_distances,\n", @@ -540,9 +540,9 @@ "text": [ "/var/folders/fq/2vl6lxqx37d2246960sj33xm0000gn/T/ipykernel_63566/3287027954.py:334: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " desc = stats.describe(arr, ddof=1)\n", - "/Users/matussikyna/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1405: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", + "/Users/anonymous/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1405: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " sk = skew(a, axis, bias=bias)\n", - "/Users/matussikyna/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1406: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", + "/Users/anonymous/PhD local/Paper2/venv/lib/python3.12/site-packages/scipy/stats/_stats_py.py:1406: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n", " kurt = kurtosis(a, axis, bias=bias)\n" ] }, @@ -576,7 +576,7 @@ "from scipy import stats # for describe, spearmanr, kendalltau\n", "\n", "# ------------------------- Logging like before -------------------------\n", - "RESULTS_ROOT = \"/Users/matussikyna/PhD local/Paper3/results/cholesky/profimedia\"\n", + "RESULTS_ROOT = \"/Users/anonymous/PhD local/Paper3/results/cholesky/profimedia\"\n", "os.makedirs(RESULTS_ROOT, exist_ok=True)\n", "log_file_path = os.path.join(RESULTS_ROOT, \"output_log.txt\")\n", "log_file = open(log_file_path, \"w\", buffering=1)\n", @@ -590,7 +590,7 @@ "print(\"=== Logging started ===\")\n", "\n", "USE_PREVIOUS_QUERIES = True # set False to keep the old random-sampling behavior\n", - "QUERY_ROOT_DIRECTORY = \"/Users/matussikyna/PhD local/Paper2/dataset/Semantic ax\"\n", + "QUERY_ROOT_DIRECTORY = \"/Users/anonymous/PhD local/Paper2/dataset/Semantic ax\"\n", "QUERY_AXES = [\"Activities\"] # same idea as your semantic_axes_list\n", "# QUERY_AXES = [\n", "# \"Activities\",\n", @@ -658,16 +658,16 @@ "# Two datasets (edit paths). Same shard layout as before.\n", "DATASETS = [\n", " # LAION (unchanged)\n", - " # (\"LAION2B-en\", \"/Users/matussikyna/PhD local/Paper3/laion/laionDir\")\n", + " # (\"LAION2B-en\", \"/Users/anonymous/PhD local/Paper3/laion/laionDir\")\n", "\n", " # Profimedia (NEW)\n", " (\"Profimedia\", {\n", " \"type\": \"profimedia\",\n", " \"filenames\": [\n", - " \"/Users/matussikyna/PhD local/Paper2/dataset/369.txt\",\n", + " \"/Users/anonymous/PhD local/Paper2/dataset/369.txt\",\n", " ],\n", " # optional: where to reuse/store the 1k random query indices\n", - " \"selected_indices_path\": \"/Users/matussikyna/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy\",\n", + " \"selected_indices_path\": \"/Users/anonymous/PhD local/Paper3/results/cholesky/profimedia/selected_indices.npy\",\n", " \"max_items\": 20000\n", " })\n", "]\n", @@ -686,15 +686,15 @@ "\n", "# Matrices: reuse your previous ones (edit paths if needed)\n", "methods = [\n", - " (\"Balance\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", - " (\"Balance+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Balance+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", - " (\"Min\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", - " (\"Min+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Min+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", - " (\"Max\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", - " (\"Max+ALT\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", - " (\"Max+Corr\", np.load(open(\"/Users/matussikyna/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", + " (\"Balance\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.001_C_0.004_gamma_0.007/matrix.npy\", 'rb'))),\n", + " (\"Balance+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_1_with_replacement_False_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Balance+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.07_C_0.001_gamma_0.0001/matrix.npy\", 'rb'))),\n", + " (\"Min\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_3_num_nn_20_dataset_size_5600_model_type_MLOML_pair_type_pairs_number_of_queries_5_params_n_layers_2_gamma_0.0001_activation_tanh_lambda_reg_7e-06/matrix.npy\", 'rb'))),\n", + " (\"Min+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_8_with_replacement_False_params_C_0.01_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Min+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_num_iterations_64_with_replacement_True_params_beta_1.0_C_0.0007_gamma_0.0004/matrix.npy\", 'rb'))),\n", + " (\"Max\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_False_params_beta_0.7_C_0.01_gamma_0.01/matrix.npy\", 'rb'))),\n", + " (\"Max+ALT\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_1_num_nn_20_dataset_size_5600_model_type_OASIS_pair_type_triplets_num_iterations_2_with_replacement_True_params_C_0.07_enforce_psd_False/matrix.npy\", 'rb'))),\n", + " (\"Max+Corr\", np.load(open(\"/Users/anonymous/PhD local/Paper2/dataset/grid_search_results/AROMA/Activities/random_order_False_random_order_iteration_0_feedback_type_2_num_nn_20_dataset_size_5600_model_type_OMDML_pair_type_pairs_batch_mode_True_params_beta_0.04_C_0.01_gamma_0.0001/matrix.npy\", 'rb')))\n", "]\n", "\n", "# ------------------------- Shard helpers (same spirit as before) -------------------------\n", diff --git a/Implementation/Untitled_large.ipynb b/Implementation/Untitled_large.ipynb index 5b5e618..1b65688 100644 --- a/Implementation/Untitled_large.ipynb +++ b/Implementation/Untitled_large.ipynb @@ -90,7 +90,7 @@ "# --- NEW: knobs for big runs (place near imports) ---\n", "CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying\n", "TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap)\n", - "EMB_MEMMAP_DIR = \"/Users/matussikyna/PhD local/Paper3/results/laion_large/memmap\"\n", + "EMB_MEMMAP_DIR = \"/Users/anonymous/PhD local/Paper3/results/laion_large/memmap\"\n", "EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk\n", "FORCE_INT_KEYS = True # store keys as int indices for huge runs (saves tens of GB)\n", "FAISS_THREADS = int(os.environ.get(\"FAISS_THREADS\", \"16\"))\n", @@ -158,7 +158,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found 1 shard pair(s) under: /Users/matussikyna/PhD local/Paper3/laion/laionDir\n", + "Found 1 shard pair(s) under: /Users/anonymous/PhD local/Paper3/laion/laionDir\n", "\n", "================================================================================\n", "Shard token : 0000\n", @@ -231,7 +231,7 @@ "# fields we prefer (first that exists wins) to create stable IDs\n", "_DEFAULT_ID_FIELDS: Sequence[str] = (\"sha256\", \"id\", \"uid\", \"image_id\", \"URL\", \"url\")\n", "\n", - "EMB_MEMMAP_DIR = \"/Users/matussikyna/PhD local/Paper3/results/memmap\"\n", + "EMB_MEMMAP_DIR = \"/Users/anonymous/PhD local/Paper3/results/memmap\"\n", "CHUNK_SIZE = 2_000_000 # how many vectors to load per batch when building/querying\n", "TRAIN_SAMPLE = 1_000_000 # how many vectors to train IVFPQ on (cap)\n", "EMB_MEMMAP_DTYPE = np.float16 # use float16 on disk; upcast to float32 per chunk\n", @@ -477,7 +477,7 @@ " print(f\" {s}\")\n", " print()\n", "\n", - "peek_laion(\"/Users/matussikyna/PhD local/Paper3/laion/laionDir\", shards=2, rows=5)" + "peek_laion(\"/Users/anonymous/PhD local/Paper3/laion/laionDir\", shards=2, rows=5)" ] }, { @@ -592,8 +592,8 @@ "2025-09-10 22:41:27,625 [INFO]: The memory available for adding the vectors is 8.0GB(total available - used by the index)\n", "2025-09-10 22:41:27,626 [INFO]: Using a batch size of 325520 (memory overhead 953.7MB)\n", "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00, 1.63it/s]\n", - "2025-09-10 22:41:28,246 [INFO]: \tComputing best hyperparameters for index /Users/matussikyna/PhD local/Paper3/results/LAION2B-en_AutofaissIP_10000_current_memory_available-8G_max_index_memory_usage-4G_max_index_query_time_ms-1_min_nearest_neighbors_to_retrieve-5.faiss 09/10/2025, 22:41:28\n", - "2025-09-10 22:41:32,261 [INFO]: \t>>> Finished \"Computing best hyperparameters for index /Users/matussikyna/PhD local/Paper3/results/LAION2B-en_AutofaissIP_10000_current_memory_available-8G_max_index_memory_usage-4G_max_index_query_time_ms-1_min_nearest_neighbors_to_retrieve-5.faiss\" in 4.0145 secs\n", + "2025-09-10 22:41:28,246 [INFO]: \tComputing best hyperparameters for index /Users/anonymous/PhD local/Paper3/results/LAION2B-en_AutofaissIP_10000_current_memory_available-8G_max_index_memory_usage-4G_max_index_query_time_ms-1_min_nearest_neighbors_to_retrieve-5.faiss 09/10/2025, 22:41:28\n", + "2025-09-10 22:41:32,261 [INFO]: \t>>> Finished \"Computing best hyperparameters for index /Users/anonymous/PhD local/Paper3/results/LAION2B-en_AutofaissIP_10000_current_memory_available-8G_max_index_memory_usage-4G_max_index_query_time_ms-1_min_nearest_neighbors_to_retrieve-5.faiss\" in 4.0145 secs\n", "2025-09-10 22:41:32,261 [INFO]: The best hyperparameters are: efSearch=596\n", "2025-09-10 22:41:32,262 [INFO]: \tCompute fast metrics 09/10/2025, 22:41:32\n", " 0%| | 0/1 [00:00