diff --git a/database/db_queries.py b/database/db_queries.py new file mode 100644 index 0000000..863fb3b --- /dev/null +++ b/database/db_queries.py @@ -0,0 +1,68 @@ +from models import TileIndex, TileFull, HouseIndex, HouseFull + +# ────────────────────────────────────────────────────────────────────────────── +# TWO-TABLE SYSTEM +# Query the INDEX table (fast, lightweight) +# Only fetch FULL table if you need WKT / polygons / full metadata +# ────────────────────────────────────────────────────────────────────────────── + + +#Get all tiles for one image (INDEX only) +print("=== All tiles for one image ===") +image_id = "hurricane-florence_00000004_post_disaster" + +for tile in TileIndex.query(image_id): + print(f" {tile.tile_id} | buildings: {tile.building_count} | damage: {tile.max_damage_level}") + print(f" post → {tile.post_image_path}") + + +# Get all destroyed tiles across ALL images +print("\n=== All destroyed tiles ===") +for tile in TileIndex.scan(TileIndex.max_damage_level == "destroyed"): + print(f" {tile.image_id} / {tile.tile_id} → {tile.post_image_path}") + + +# Get full polygon data for a specific tile +print("\n=== Full data for one tile ===") +full = TileFull.get("hurricane-florence_00000004_post_disaster", "tile_001") +print(f" Sensor: {full.sensor}") +print(f" Crop bounds: {full.crop_bounds}") +print(f" Buildings: {len(full.buildings)} found") +for b in full.buildings: + print(f" → damage: {b['damage_level']} | wkt: {b['local_wkt'][:40]}...") + + +# Use INDEX pointer to load FULL data +print("\n=== Index → Full data lookup ===") +tile = TileIndex.get("hurricane-florence_00000004_post_disaster", "tile_002") +print(f" Index record: {tile.tile_id}, damage={tile.max_damage_level}") + +# Use the full_record_key pointer to fetch full record +img_id, tile_id = tile.full_record_key.split("#") +full = TileFull.get(img_id, tile_id) +print(f" Full record has {len(full.buildings)} buildings") +print(f" GSD: {full.gsd}, Sensor: {full.sensor}") + + +# Training loop pattern +print("\n=== Training loop example ===") +from PIL import Image # pip install Pillow + +for tile in TileIndex.scan(TileIndex.disaster_type == "flooding"): + # Load the actual images using the paths stored in the index + post_img = Image.open(tile.post_image_path) + pre_img = Image.open(tile.pre_image_path) + label = tile.max_damage_level + + print(f" Loaded {tile.tile_id}: {post_img.size}, label={label}") + + # Only fetch full data if you need polygon details + if label == "destroyed": + full = TileFull.get(tile.image_id, tile.tile_id) + print(f" → {len(full.buildings)} building polygons available") + + +#Example 6: Houses queries +print("\n=== All un-classified houses ===") +for house in HouseIndex.scan(HouseIndex.damage_level == "un-classified"): + print(f" {house.image_id} / {house.house_id} → {house.post_image_path}") \ No newline at end of file diff --git a/database/debug_paths.py b/database/debug_paths.py new file mode 100644 index 0000000..c4287b1 --- /dev/null +++ b/database/debug_paths.py @@ -0,0 +1,16 @@ +import os + +TILES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Grid_Final\CS4485_Hurricane_Grid_Final" +HOUSES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Org_final\CS4485_Hurricane_Org_final" + +print("=== TILES folder contents ===") +for f in os.listdir(TILES_PATH): + full = os.path.join(TILES_PATH, f) + json_path = os.path.join(full, "master_grid_labels.json") + print(f" {f} → json exists: {os.path.exists(json_path)}") + +print("\n=== HOUSES folder contents ===") +for f in os.listdir(HOUSES_PATH): + full = os.path.join(HOUSES_PATH, f) + json_path = os.path.join(full, "labels.json") + print(f" {f} → json exists: {os.path.exists(json_path)}") \ No newline at end of file diff --git a/database/models.py b/database/models.py new file mode 100644 index 0000000..2717440 --- /dev/null +++ b/database/models.py @@ -0,0 +1,162 @@ +from pynamodb.models import Model +from pynamodb.attributes import UnicodeAttribute, NumberAttribute, JSONAttribute + + +# FAST INDEX TABLES + +# Only holds labels, counts, damage levels, and file paths + + +class TileIndex(Model): + """ + Fast index for tiles dataset. + Query this to find tiles by damage level, disaster type, etc. + Use tile.full_record_key to fetch full data from TileFull if needed. + """ + class Meta: + table_name = "TileIndex" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + # Keys + image_id = UnicodeAttribute(hash_key=True) + tile_id = UnicodeAttribute(range_key=True) + + # Quick-filter labels + disaster = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + building_count = NumberAttribute(default=0) + max_damage_level = UnicodeAttribute(null=True) + + # File paths + pre_image_path = UnicodeAttribute(null=True) + post_image_path = UnicodeAttribute(null=True) + target_image_path = UnicodeAttribute(null=True) + + # Pointer to full data record + full_record_key = UnicodeAttribute(null=True) + + +class HouseIndex(Model): + """ + Fast index for houses dataset. + Query this to find houses by damage level, disaster type, etc. + Use house.full_record_key to fetch full data from HouseFull if needed. + """ + class Meta: + table_name = "HouseIndex" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + # Keys + image_id = UnicodeAttribute(hash_key=True) + house_id = UnicodeAttribute(range_key=True) + + # Quick-filter labels + disaster = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + damage_level = UnicodeAttribute(null=True) + + # File paths + pre_image_path = UnicodeAttribute(null=True) + post_image_path = UnicodeAttribute(null=True) + target_image_path = UnicodeAttribute(null=True) + + # Pointer to full data record + full_record_key = UnicodeAttribute(null=True) + + + +# FULL DATA TABLES +# Heavy records with all WKT polygons, buildings arrays, and complete metadata +# Only load these when you actually need the polygon/geometry details + + +class TileFull(Model): + """ + Full data record for a tile. + Contains all building polygons, WKT geometry, crop bounds, and complete metadata. + Access via: TileFull.get(image_id, tile_id) + """ + class Meta: + table_name = "TileFull" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + # Keys (same as TileIndex so you can cross-reference easily) + image_id = UnicodeAttribute(hash_key=True) + tile_id = UnicodeAttribute(range_key=True) + + # Complete metadata from JSON + sensor = UnicodeAttribute(null=True) + provider_asset_type = UnicodeAttribute(null=True) + gsd = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + off_nadir_angle = UnicodeAttribute(null=True) + pan_resolution = UnicodeAttribute(null=True) + sun_azimuth = UnicodeAttribute(null=True) + sun_elevation = UnicodeAttribute(null=True) + target_azimuth = UnicodeAttribute(null=True) + disaster = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + catalog_id = UnicodeAttribute(null=True) + original_width = NumberAttribute(null=True) + original_height = NumberAttribute(null=True) + + # Tile geometry + crop_bounds = JSONAttribute() # {x1, y1, x2, y2} + buildings = JSONAttribute() # full list of {local_wkt, damage_level} + + # Path to raw JSON on disk (load this if you need anything else) + json_path = UnicodeAttribute(null=True) + + +class HouseFull(Model): + """ + Full data record for a house crop. + Contains WKT polygons, crop bounds, and complete metadata. + Access via: HouseFull.get(image_id, house_id) + """ + class Meta: + table_name = "HouseFull" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + # Keys + image_id = UnicodeAttribute(hash_key=True) + house_id = UnicodeAttribute(range_key=True) + + # Complete metadata + sensor = UnicodeAttribute(null=True) + provider_asset_type = UnicodeAttribute(null=True) + gsd = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + off_nadir_angle = UnicodeAttribute(null=True) + pan_resolution = UnicodeAttribute(null=True) + sun_azimuth = UnicodeAttribute(null=True) + sun_elevation = UnicodeAttribute(null=True) + target_azimuth = UnicodeAttribute(null=True) + disaster = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + catalog_id = UnicodeAttribute(null=True) + original_width = NumberAttribute(null=True) + original_height = NumberAttribute(null=True) + + # House geometry + local_wkt = UnicodeAttribute(null=True) + original_wkt = UnicodeAttribute(null=True) + crop_bounds = JSONAttribute() + damage_level = UnicodeAttribute(null=True) + + # Path to raw JSON on disk + json_path = UnicodeAttribute(null=True) \ No newline at end of file diff --git a/database/models_2.py b/database/models_2.py new file mode 100644 index 0000000..bae0bae --- /dev/null +++ b/database/models_2.py @@ -0,0 +1,109 @@ +from pynamodb.models import Model +from pynamodb.attributes import UnicodeAttribute, NumberAttribute, JSONAttribute + +class TileIndex(Model): + class Meta: + table_name = "tile_cursor_index" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + image_uid = UnicodeAttribute(hash_key=True) + tile_uid = UnicodeAttribute(range_key=True) + disaster_id = UnicodeAttribute(null=True) + pair_id = UnicodeAttribute(null=True) + tile_id = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + building_count = NumberAttribute(default=0) + classification = UnicodeAttribute(null=True) + prediction = UnicodeAttribute(null=True) + pre_image_path = UnicodeAttribute(null=True) + post_image_path = UnicodeAttribute(null=True) + target_image_path = UnicodeAttribute(null=True) + full_record_key = UnicodeAttribute(null=True) + +class HouseIndex(Model): + class Meta: + table_name = "house_cursor_index" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + image_uid = UnicodeAttribute(hash_key=True) + house_uid = UnicodeAttribute(range_key=True) + disaster_id = UnicodeAttribute(null=True) + pair_id = UnicodeAttribute(null=True) + house_id = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + classification = UnicodeAttribute(null=True) + prediction = UnicodeAttribute(null=True) + pre_image_path = UnicodeAttribute(null=True) + post_image_path = UnicodeAttribute(null=True) + target_image_path = UnicodeAttribute(null=True) + full_record_key = UnicodeAttribute(null=True) + +class TileFull(Model): + class Meta: + table_name = "tile_cursor_full" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + image_uid = UnicodeAttribute(hash_key=True) + tile_uid = UnicodeAttribute(range_key=True) + disaster_id = UnicodeAttribute(null=True) + pair_id = UnicodeAttribute(null=True) + tile_id = UnicodeAttribute(null=True) + sensor = UnicodeAttribute(null=True) + provider_asset_type = UnicodeAttribute(null=True) + gsd = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + off_nadir_angle = UnicodeAttribute(null=True) + pan_resolution = UnicodeAttribute(null=True) + sun_azimuth = UnicodeAttribute(null=True) + sun_elevation = UnicodeAttribute(null=True) + target_azimuth = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + catalog_id = UnicodeAttribute(null=True) + original_width = NumberAttribute(null=True) + original_height = NumberAttribute(null=True) + crop_bounds = JSONAttribute() + buildings = JSONAttribute() + json_path = UnicodeAttribute(null=True) + +class HouseFull(Model): + class Meta: + table_name = "house_cursor_full" + host = "http://localhost:8000" + region = "us-west-2" + aws_access_key_id = "fake" + aws_secret_access_key = "fake" + + image_uid = UnicodeAttribute(hash_key=True) + house_uid = UnicodeAttribute(range_key=True) + disaster_id = UnicodeAttribute(null=True) + pair_id = UnicodeAttribute(null=True) + house_id = UnicodeAttribute(null=True) + sensor = UnicodeAttribute(null=True) + provider_asset_type = UnicodeAttribute(null=True) + gsd = UnicodeAttribute(null=True) + capture_date = UnicodeAttribute(null=True) + off_nadir_angle = UnicodeAttribute(null=True) + pan_resolution = UnicodeAttribute(null=True) + sun_azimuth = UnicodeAttribute(null=True) + sun_elevation = UnicodeAttribute(null=True) + target_azimuth = UnicodeAttribute(null=True) + disaster_type = UnicodeAttribute(null=True) + catalog_id = UnicodeAttribute(null=True) + original_width = NumberAttribute(null=True) + original_height = NumberAttribute(null=True) + crop_bounds = JSONAttribute() + points = JSONAttribute() + classification = UnicodeAttribute(null=True) + prediction = UnicodeAttribute(null=True) + json_path = UnicodeAttribute(null=True) \ No newline at end of file diff --git a/database/populate_db.py b/database/populate_db.py new file mode 100644 index 0000000..ae4544d --- /dev/null +++ b/database/populate_db.py @@ -0,0 +1,175 @@ +import os +import json +from models import TileIndex, TileFull, HouseIndex, HouseFull + +# ✏️ UPDATE THESE to your actual folder paths +TILES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Grid_Final\CS4485_Hurricane_Grid_Final" +HOUSES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Org_final\CS4485_Hurricane_Org_final" + + + + +# ── Create all 4 tables ──────────────────────────────────────────────────── +print("🔧 Setting up tables...") +tables = [ + (TileIndex, "TileIndex"), + (TileFull, "TileFull"), + (HouseIndex, "HouseIndex"), + (HouseFull, "HouseFull"), +] +for ModelClass, name in tables: + if not ModelClass.exists(): + ModelClass.create_table(wait=True, read_capacity_units=1, write_capacity_units=1) + print(f" ✅ Created: {name}") + else: + print(f" ✅ Already exists: {name}") + + +# ── Load ALL TILES ───────────────────────────────────────────────────────── +print("\n📦 Loading tiles dataset...") +tiles_loaded = 0 +tiles_skipped = 0 + +for image_folder in os.listdir(TILES_PATH): + image_folder_path = os.path.join(TILES_PATH, image_folder) + + if not os.path.isdir(image_folder_path): + continue + + json_path = os.path.join(image_folder_path, "master_grid_labels.json") + + if not os.path.exists(json_path): + print(f" ⚠️ No master_grid_labels.json in {image_folder}, skipping.") + tiles_skipped += 1 + continue + + with open(json_path, 'r') as f: + data = json.load(f) + + image_id = data["source_image"].replace(".png", "") + meta = data.get("metadata", {}) + + for tile_id, tile_info in data["tiles"].items(): + buildings = tile_info.get("buildings", []) + tile_folder = os.path.join(image_folder_path, tile_id) + record_key = f"{image_id}#{tile_id}" + + # ── Save to FAST INDEX (lightweight) ────────────────────────────── + TileIndex( + image_id = image_id, + tile_id = tile_id, + disaster = meta.get("disaster"), + disaster_type = meta.get("disaster_type"), + capture_date = meta.get("capture_date"), + building_count = len(buildings), + max_damage_level = "pending", + pre_image_path = os.path.join(tile_folder, "pre.png"), + post_image_path = os.path.join(tile_folder, "post.png"), + target_image_path = os.path.join(tile_folder, "target.png"), + full_record_key = record_key, + ).save() + + # ── Save to FULL DATA (everything) ──────────────────────────────── + TileFull( + image_id = image_id, + tile_id = tile_id, + sensor = meta.get("sensor"), + provider_asset_type = meta.get("provider_asset_type"), + gsd = str(meta.get("gsd", "")), + capture_date = meta.get("capture_date"), + off_nadir_angle = str(meta.get("off_nadir_angle", "")), + pan_resolution = str(meta.get("pan_resolution", "")), + sun_azimuth = str(meta.get("sun_azimuth", "")), + sun_elevation = str(meta.get("sun_elevation", "")), + target_azimuth = str(meta.get("target_azimuth", "")), + disaster = meta.get("disaster"), + disaster_type = meta.get("disaster_type"), + catalog_id = meta.get("catalog_id"), + original_width = meta.get("original_width"), + original_height = meta.get("original_height"), + crop_bounds = tile_info.get("crop_bounds", {}), + buildings = buildings, + json_path = json_path, + ).save() + + tiles_loaded += 1 + + print(f" 🚀 {image_folder} → {len(data['tiles'])} tiles") + +print(f"\n ✅ Tiles done: {tiles_loaded} records loaded, {tiles_skipped} skipped") + + +# ── Load ALL HOUSES ──────────────────────────────────────────────────────── +print("\n🏠 Loading houses dataset...") +houses_loaded = 0 +houses_skipped = 0 + +for image_folder in os.listdir(HOUSES_PATH): + image_folder_path = os.path.join(HOUSES_PATH, image_folder) + + if not os.path.isdir(image_folder_path): + continue + + json_path = os.path.join(image_folder_path, "labels.json") + + if not os.path.exists(json_path): + print(f" ⚠️ No labels.json in {image_folder}, skipping.") + houses_skipped += 1 + continue + + with open(json_path, 'r') as f: + data = json.load(f) + + image_id = data["source_image"].replace(".png", "") + meta = data.get("metadata", {}) + + for house_id, house_info in data["houses"].items(): + house_folder = os.path.join(image_folder_path, house_id) + record_key = f"{image_id}#{house_id}" + + # ── Save to FAST INDEX (lightweight) ────────────────────────────── + HouseIndex( + image_id = image_id, + house_id = house_id, + disaster = meta.get("disaster"), + disaster_type = meta.get("disaster_type"), + capture_date = meta.get("capture_date"), + damage_level = "pending", + pre_image_path = os.path.join(house_folder, "pre.png"), + post_image_path = os.path.join(house_folder, "post.png"), + target_image_path = os.path.join(house_folder, "target.png"), + full_record_key = record_key, + ).save() + + # ── Save to FULL DATA (everything) ──────────────────────────────── + HouseFull( + image_id = image_id, + house_id = house_id, + sensor = meta.get("sensor"), + provider_asset_type = meta.get("provider_asset_type"), + gsd = str(meta.get("gsd", "")), + capture_date = meta.get("capture_date"), + off_nadir_angle = str(meta.get("off_nadir_angle", "")), + pan_resolution = str(meta.get("pan_resolution", "")), + sun_azimuth = str(meta.get("sun_azimuth", "")), + sun_elevation = str(meta.get("sun_elevation", "")), + target_azimuth = str(meta.get("target_azimuth", "")), + disaster = meta.get("disaster"), + disaster_type = meta.get("disaster_type"), + catalog_id = meta.get("catalog_id"), + original_width = meta.get("original_width"), + original_height = meta.get("original_height"), + local_wkt = house_info.get("local_wkt"), + original_wkt = house_info.get("original_wkt"), + crop_bounds = house_info.get("crop_bounds", {}), + damage_level = house_info.get("damage_level"), + json_path = json_path, + ).save() + + houses_loaded += 1 + + print(f" 🚀 {image_folder} → {len(data['houses'])} houses") + +print(f"\n ✅ Houses done: {houses_loaded} records loaded, {houses_skipped} skipped") + +print("\n✨ All done! Both index and full tables are loaded.") \ No newline at end of file diff --git a/database/populate_db2.py b/database/populate_db2.py new file mode 100644 index 0000000..ac9568f --- /dev/null +++ b/database/populate_db2.py @@ -0,0 +1,171 @@ +from models_2 import TileIndex, TileFull, HouseIndex, HouseFull +import os +import json + +TILES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Crop_Tiles\CS4485_Hurricane_Crop_Tiles" +HOUSES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Cursor_House\CS4485_Hurricane_Cursor_House" + +# ── Create / reset all 4 tables ──────────────────────────────────────────── +print("🔧 Setting up tables...") +tables = [(TileIndex,"tile_cursor_index"),(TileFull,"tile_cursor_full"),(HouseIndex,"house_cursor_index"),(HouseFull,"house_cursor_full")] + +for ModelClass, name in tables: + if ModelClass.exists(): + ModelClass.delete_table() + print(f" 🗑️ Cleared: {name}") + ModelClass.create_table(wait=True, read_capacity_units=1, write_capacity_units=1) + print(f" ✅ Created: {name}") + +# ── Load TILES ───────────────────────────────────────────────────────────── +print("\n📦 Loading tiles...") +tiles_loaded = 0 + +for image_folder in os.listdir(TILES_PATH): + folder_path = os.path.join(TILES_PATH, image_folder) + if not os.path.isdir(folder_path): + continue + + json_path = os.path.join(folder_path, "master_grid_labels.json") + if not os.path.exists(json_path): + print(f" ⚠️ No master_grid_labels.json in {image_folder}, skipping.") + continue + + with open(json_path, 'r') as f: + data = json.load(f) + + meta = data.get("metadata", {}) + disaster_id = data.get("disasterId") or meta.get("disaster", "hurricane-florence") + pair_id = data.get("pairId", "unknown") + image_uid = image_folder + + for tile_id, tile_info in data["tiles"].items(): + tile_uid = tile_id + buildings = tile_info.get("buildings", []) + tile_folder = os.path.join(folder_path, tile_id) + record_key = f"{image_uid}#{tile_uid}" + + # FAST INDEX + TileIndex( + image_uid = image_uid, + tile_uid = tile_uid, + disaster_id = disaster_id, + pair_id = pair_id, + tile_id = tile_id, + disaster_type = meta.get("disaster_type"), + capture_date = meta.get("capture_date"), + building_count = len(buildings), + classification = "unknown", + prediction = None, + pre_image_path = os.path.join(tile_folder, "pre.png"), + post_image_path = os.path.join(tile_folder, "post.png"), + target_image_path = os.path.join(tile_folder, "target.png"), + full_record_key = record_key, + ).save() + + # FULL DATA + TileFull( + image_uid = image_uid, + tile_uid = tile_uid, + disaster_id = disaster_id, + pair_id = pair_id, + tile_id = tile_id, + sensor = meta.get("sensor"), + provider_asset_type = meta.get("provider_asset_type"), + gsd = str(meta.get("gsd", "")), + capture_date = meta.get("capture_date"), + off_nadir_angle = str(meta.get("off_nadir_angle", "")), + pan_resolution = str(meta.get("pan_resolution", "")), + sun_azimuth = str(meta.get("sun_azimuth", "")), + sun_elevation = str(meta.get("sun_elevation", "")), + target_azimuth = str(meta.get("target_azimuth", "")), + disaster_type = meta.get("disaster_type"), + catalog_id = meta.get("catalog_id"), + original_width = meta.get("original_width"), + original_height = meta.get("original_height"), + crop_bounds = tile_info.get("crop_bounds", {}), + buildings = buildings, + json_path = json_path, + ).save() + + tiles_loaded += 1 + + print(f" 🚀 {image_folder} → {len(data['tiles'])} tiles") + +print(f"\n ✅ Tiles done: {tiles_loaded} records loaded") + +# ── Load HOUSES ──────────────────────────────────────────────────────────── +print("\n🏠 Loading houses...") +houses_loaded = 0 + +for image_folder in os.listdir(HOUSES_PATH): + folder_path = os.path.join(HOUSES_PATH, image_folder) + if not os.path.isdir(folder_path): + continue + + json_path = os.path.join(folder_path, "labels.json") + if not os.path.exists(json_path): + print(f" ⚠️ No labels.json in {image_folder}, skipping.") + continue + + with open(json_path, 'r') as f: + data = json.load(f) + + meta = data.get("metadata", {}) + disaster_id = data.get("disasterId") or meta.get("disaster", "hurricane-florence") + pair_id = data.get("pairId", "unknown") + image_uid = image_folder + for house_id, house_info in data["houses"].items(): + house_uid = house_id + house_folder = os.path.join(folder_path, house_id) + record_key = f"{image_uid}#{house_uid}" + + # FAST INDEX + HouseIndex( + image_uid = image_uid, + house_uid = house_uid, + disaster_id = disaster_id, + pair_id = pair_id, + house_id = house_id, + disaster_type = meta.get("disaster_type"), + capture_date = meta.get("capture_date"), + classification = "unknown", + prediction = None, + pre_image_path = os.path.join(house_folder, "pre.png"), + post_image_path = os.path.join(house_folder, "post.png"), + target_image_path = os.path.join(house_folder, "target.png"), + full_record_key = record_key, + ).save() + + # FULL DATA + HouseFull( + image_uid = image_uid, + house_uid = house_uid, + disaster_id = disaster_id, + pair_id = pair_id, + house_id = house_id, + sensor = meta.get("sensor"), + provider_asset_type = meta.get("provider_asset_type"), + gsd = str(meta.get("gsd", "")), + capture_date = meta.get("capture_date"), + off_nadir_angle = str(meta.get("off_nadir_angle", "")), + pan_resolution = str(meta.get("pan_resolution", "")), + sun_azimuth = str(meta.get("sun_azimuth", "")), + sun_elevation = str(meta.get("sun_elevation", "")), + target_azimuth = str(meta.get("target_azimuth", "")), + disaster_type = meta.get("disaster_type"), + catalog_id = meta.get("catalog_id"), + original_width = meta.get("original_width"), + original_height = meta.get("original_height"), + crop_bounds = house_info.get("crop_bounds", {}), + points = house_info.get("points", {}), + classification = house_info.get("classification", "unknown"), + prediction = house_info.get("prediction"), + json_path = json_path, + ).save() + + houses_loaded += 1 + + print(f" 🚀 {image_folder} → {len(data['houses'])} houses") + +print(f"\n ✅ Houses done: {houses_loaded} records loaded") +print("\n✨ All done!") \ No newline at end of file diff --git a/database/verify.py b/database/verify.py new file mode 100644 index 0000000..1d9edd4 --- /dev/null +++ b/database/verify.py @@ -0,0 +1,31 @@ +from models import TileIndex, TileFull, HouseIndex, HouseFull + +print("=== TABLE RECORD COUNTS ===") +tile_index_count = sum(1 for _ in TileIndex.scan()) +tile_full_count = sum(1 for _ in TileFull.scan()) +house_index_count = sum(1 for _ in HouseIndex.scan()) +house_full_count = sum(1 for _ in HouseFull.scan()) + +print(f" TileIndex : {tile_index_count} records") +print(f" TileFull : {tile_full_count} records") +print(f" HouseIndex : {house_index_count} records") +print(f" HouseFull : {house_full_count} records") + +print("\n=== SAMPLE TILE INDEX RECORD ===") +sample_tile = next(TileIndex.scan()) +print(f" image_id : {sample_tile.image_id}") +print(f" tile_id : {sample_tile.tile_id}") +print(f" disaster : {sample_tile.disaster}") +print(f" building_count : {sample_tile.building_count}") +print(f" damage_level : {sample_tile.max_damage_level}") +print(f" post_image_path : {sample_tile.post_image_path}") +print(f" full_record_key : {sample_tile.full_record_key}") + +print("\n=== SAMPLE HOUSE INDEX RECORD ===") +sample_house = next(HouseIndex.scan()) +print(f" image_id : {sample_house.image_id}") +print(f" house_id : {sample_house.house_id}") +print(f" damage_level : {sample_house.damage_level}") +print(f" post_image_path : {sample_house.post_image_path}") + +print("\n✅ All tables verified!") \ No newline at end of file