20251126_v0.10.12_lza_investigation

production_configs/20251126_v0.10.12_LZA_investigation/README.md

@@ -0,0 +1,7 @@
+# LZA investigation
+
+We are currently investigation some discrepancy observed at LZA with the Crab. For this purpose a few alternative simulations with different atmosphere have been simulated.
+The objective is to process the same way as the standard production
+
+Parameters used for production are identical to 20240918_v0.10.12_allsky_nsb_grid with modification of 20250212_v0.10.17_allsky_interp_dl2_irfs.
+The RF used are the one of the standard productions.

production_configs/20251126_v0.10.12_LZA_investigation/add_slurm_options.py

@@ -0,0 +1,40 @@
+import sys
+from pathlib import Path
+from ruamel.yaml import YAML
+
+
+def add_slurm_options(config_file):
+    """
+    Add SLURM options to each step in the given YML config file.
+    This should ease job handling on the cluster, allowing some jobs to be scheduled in a 12h window.
+    It also uses the `--nice` SLURM option to give priority to lstosa jobs.
+    """
+    yaml = YAML()
+    yaml.preserve_quotes = True
+    config = yaml.load(open(config_file))
+
+    for stage_name, stage in config['stages'].items():
+        for ii, step in enumerate(stage):
+            slurm_options = step.get('extra_slurm_options', {})
+            if 'time' not in slurm_options:
+                if stage_name == 'r0_to_dl1':
+                    slurm_options['time'] = '11:30:00'
+                elif stage_name == 'train_pipe':
+                    slurm_options['time'] = '02-00:00:00'
+                elif stage_name == 'dl1ab':
+                    slurm_options['time'] = '08:00:00'
+                elif stage_name == 'merge_dl1' and ('GammaDiffuse' in step['input'] or 'Protons' in step['input']):
+                    slurm_options['time'] = '08:00:00'
+                else:
+                    slurm_options['partition'] = 'short'
+            if 'nice' not in slurm_options:
+                slurm_options['nice'] = 10
+            stage[ii]['extra_slurm_options'] = slurm_options
+
+    with open(config_file, 'w') as f:
+        yaml.dump(config, f)
+
+
+if __name__ == '__main__':
+    for cfg_file in Path('.').glob('NSB-*/lstmcpipe*.yml'):
+        add_slurm_options(cfg_file)

production_configs/20251126_v0.10.12_LZA_investigation/change_nestimators.py

@@ -0,0 +1,25 @@
+import json
+import os
+
+# Directory path
+directory = '.'
+
+# Iterate over all directories starting with 'NSB-'
+for root, dirs, files in os.walk(directory):
+    if root.startswith(os.path.join(directory, 'NSB-')):
+        # Iterate over all JSON files in the directory
+        for file in files:
+            if file.endswith('.json'):
+                file_path = os.path.join(root, file)
+
+                # Open the JSON file
+                with open(file_path, 'r') as f:
+                    data = json.load(f)
+
+                # Modify the 'n_estimators' values
+                data['random_forest_energy_regressor_args']['n_estimators'] = 50
+                data['random_forest_disp_regressor_args']['n_estimators'] = 50
+
+               # Write the modified data back to the JSON file
+                with open(file_path, 'w') as f:
+                    json.dump(data, f, indent=4)