Issue 21: added dynamodb database setup files

database/db_queries.py

@@ -0,0 +1,68 @@
+from models import TileIndex, TileFull, HouseIndex, HouseFull
+
+# ──────────────────────────────────────────────────────────────────────────────
+# TWO-TABLE SYSTEM
+# Query the INDEX table (fast, lightweight)
+# Only fetch FULL table if you need WKT / polygons / full metadata
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+#Get all tiles for one image (INDEX only) 
+print("=== All tiles for one image ===")
+image_id = "hurricane-florence_00000004_post_disaster"
+
+for tile in TileIndex.query(image_id):
+    print(f"  {tile.tile_id} | buildings: {tile.building_count} | damage: {tile.max_damage_level}")
+    print(f"    post → {tile.post_image_path}")
+
+
+# Get all destroyed tiles across ALL images 
+print("\n=== All destroyed tiles ===")
+for tile in TileIndex.scan(TileIndex.max_damage_level == "destroyed"):
+    print(f"  {tile.image_id} / {tile.tile_id} → {tile.post_image_path}")
+
+
+# Get full polygon data for a specific tile 
+print("\n=== Full data for one tile ===")
+full = TileFull.get("hurricane-florence_00000004_post_disaster", "tile_001")
+print(f"  Sensor: {full.sensor}")
+print(f"  Crop bounds: {full.crop_bounds}")
+print(f"  Buildings: {len(full.buildings)} found")
+for b in full.buildings:
+    print(f"    → damage: {b['damage_level']} | wkt: {b['local_wkt'][:40]}...")
+
+
+# Use INDEX pointer to load FULL data 
+print("\n=== Index → Full data lookup ===")
+tile = TileIndex.get("hurricane-florence_00000004_post_disaster", "tile_002")
+print(f"  Index record: {tile.tile_id}, damage={tile.max_damage_level}")
+
+# Use the full_record_key pointer to fetch full record
+img_id, tile_id = tile.full_record_key.split("#")
+full = TileFull.get(img_id, tile_id)
+print(f"  Full record has {len(full.buildings)} buildings")
+print(f"  GSD: {full.gsd}, Sensor: {full.sensor}")
+
+
+#  Training loop pattern 
+print("\n=== Training loop example ===")
+from PIL import Image  # pip install Pillow
+
+for tile in TileIndex.scan(TileIndex.disaster_type == "flooding"):
+    # Load the actual images using the paths stored in the index
+    post_img = Image.open(tile.post_image_path)
+    pre_img  = Image.open(tile.pre_image_path)
+    label    = tile.max_damage_level
+
+    print(f"  Loaded {tile.tile_id}: {post_img.size}, label={label}")
+
+    # Only fetch full data if you need polygon details
+    if label == "destroyed":
+        full = TileFull.get(tile.image_id, tile.tile_id)
+        print(f"    → {len(full.buildings)} building polygons available")
+
+
+#Example 6: Houses queries
+print("\n=== All un-classified houses ===")
+for house in HouseIndex.scan(HouseIndex.damage_level == "un-classified"):
+    print(f"  {house.image_id} / {house.house_id} → {house.post_image_path}")

database/debug_paths.py

@@ -0,0 +1,16 @@
+import os
+
+TILES_PATH  = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Grid_Final\CS4485_Hurricane_Grid_Final"
+HOUSES_PATH = r"C:\Users\furkm\OneDrive\Desktop\database_setup\CS4485_Hurricane_Org_final\CS4485_Hurricane_Org_final"
+
+print("=== TILES folder contents ===")
+for f in os.listdir(TILES_PATH):
+    full = os.path.join(TILES_PATH, f)
+    json_path = os.path.join(full, "master_grid_labels.json")
+    print(f"  {f} → json exists: {os.path.exists(json_path)}")
+
+print("\n=== HOUSES folder contents ===")
+for f in os.listdir(HOUSES_PATH):
+    full = os.path.join(HOUSES_PATH, f)
+    json_path = os.path.join(full, "labels.json")
+    print(f"  {f} → json exists: {os.path.exists(json_path)}")

database/models.py

@@ -0,0 +1,162 @@
+from pynamodb.models import Model
+from pynamodb.attributes import UnicodeAttribute, NumberAttribute, JSONAttribute
+
+
+# FAST INDEX TABLES
+
+# Only holds labels, counts, damage levels, and file paths
+
+
+class TileIndex(Model):
+    """
+    Fast index for tiles dataset.
+    Query this to find tiles by damage level, disaster type, etc.
+    Use tile.full_record_key to fetch full data from TileFull if needed.
+    """
+    class Meta:
+        table_name  = "TileIndex"
+        host        = "http://localhost:8000"
+        region      = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    # Keys
+    image_id         = UnicodeAttribute(hash_key=True)   
+    tile_id          = UnicodeAttribute(range_key=True)  
+
+    # Quick-filter labels
+    disaster         = UnicodeAttribute(null=True)        
+    disaster_type    = UnicodeAttribute(null=True)        
+    capture_date     = UnicodeAttribute(null=True)
+    building_count   = NumberAttribute(default=0)
+    max_damage_level = UnicodeAttribute(null=True)        
+
+    # File paths 
+    pre_image_path    = UnicodeAttribute(null=True)
+    post_image_path   = UnicodeAttribute(null=True)
+    target_image_path = UnicodeAttribute(null=True)
+
+    # Pointer to full data record
+    full_record_key  = UnicodeAttribute(null=True)       
+
+
+class HouseIndex(Model):
+    """
+    Fast index for houses dataset.
+    Query this to find houses by damage level, disaster type, etc.
+    Use house.full_record_key to fetch full data from HouseFull if needed.
+    """
+    class Meta:
+        table_name  = "HouseIndex"
+        host        = "http://localhost:8000"
+        region      = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    # Keys
+    image_id         = UnicodeAttribute(hash_key=True)
+    house_id         = UnicodeAttribute(range_key=True)  
+
+    # Quick-filter labels
+    disaster         = UnicodeAttribute(null=True)
+    disaster_type    = UnicodeAttribute(null=True)
+    capture_date     = UnicodeAttribute(null=True)
+    damage_level     = UnicodeAttribute(null=True)
+
+    # File paths
+    pre_image_path    = UnicodeAttribute(null=True)
+    post_image_path   = UnicodeAttribute(null=True)
+    target_image_path = UnicodeAttribute(null=True)
+
+    # Pointer to full data record
+    full_record_key  = UnicodeAttribute(null=True)        
+
+
+
+# FULL DATA TABLES
+# Heavy records with all WKT polygons, buildings arrays, and complete metadata
+# Only load these when you actually need the polygon/geometry details
+
+
+class TileFull(Model):
+    """
+    Full data record for a tile.
+    Contains all building polygons, WKT geometry, crop bounds, and complete metadata.
+    Access via: TileFull.get(image_id, tile_id)
+    """
+    class Meta:
+        table_name  = "TileFull"
+        host        = "http://localhost:8000"
+        region      = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    # Keys (same as TileIndex so you can cross-reference easily)
+    image_id  = UnicodeAttribute(hash_key=True)
+    tile_id   = UnicodeAttribute(range_key=True)
+
+    # Complete metadata from JSON
+    sensor               = UnicodeAttribute(null=True)
+    provider_asset_type  = UnicodeAttribute(null=True)
+    gsd                  = UnicodeAttribute(null=True)   
+    capture_date         = UnicodeAttribute(null=True)
+    off_nadir_angle      = UnicodeAttribute(null=True)
+    pan_resolution       = UnicodeAttribute(null=True)
+    sun_azimuth          = UnicodeAttribute(null=True)
+    sun_elevation        = UnicodeAttribute(null=True)
+    target_azimuth       = UnicodeAttribute(null=True)
+    disaster             = UnicodeAttribute(null=True)
+    disaster_type        = UnicodeAttribute(null=True)
+    catalog_id           = UnicodeAttribute(null=True)
+    original_width       = NumberAttribute(null=True)
+    original_height      = NumberAttribute(null=True)
+
+    # Tile geometry
+    crop_bounds = JSONAttribute()   # {x1, y1, x2, y2}
+    buildings   = JSONAttribute()   # full list of {local_wkt, damage_level}
+
+    # Path to raw JSON on disk (load this if you need anything else)
+    json_path   = UnicodeAttribute(null=True)
+
+
+class HouseFull(Model):
+    """
+    Full data record for a house crop.
+    Contains WKT polygons, crop bounds, and complete metadata.
+    Access via: HouseFull.get(image_id, house_id)
+    """
+    class Meta:
+        table_name  = "HouseFull"
+        host        = "http://localhost:8000"
+        region      = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    # Keys
+    image_id  = UnicodeAttribute(hash_key=True)
+    house_id  = UnicodeAttribute(range_key=True)
+
+    # Complete metadata
+    sensor               = UnicodeAttribute(null=True)
+    provider_asset_type  = UnicodeAttribute(null=True)
+    gsd                  = UnicodeAttribute(null=True)
+    capture_date         = UnicodeAttribute(null=True)
+    off_nadir_angle      = UnicodeAttribute(null=True)
+    pan_resolution       = UnicodeAttribute(null=True)
+    sun_azimuth          = UnicodeAttribute(null=True)
+    sun_elevation        = UnicodeAttribute(null=True)
+    target_azimuth       = UnicodeAttribute(null=True)
+    disaster             = UnicodeAttribute(null=True)
+    disaster_type        = UnicodeAttribute(null=True)
+    catalog_id           = UnicodeAttribute(null=True)
+    original_width       = NumberAttribute(null=True)
+    original_height      = NumberAttribute(null=True)
+
+    # House geometry
+    local_wkt    = UnicodeAttribute(null=True)
+    original_wkt = UnicodeAttribute(null=True)
+    crop_bounds  = JSONAttribute()
+    damage_level = UnicodeAttribute(null=True)
+
+    # Path to raw JSON on disk
+    json_path    = UnicodeAttribute(null=True)

database/models_2.py

@@ -0,0 +1,109 @@
+from pynamodb.models import Model
+from pynamodb.attributes import UnicodeAttribute, NumberAttribute, JSONAttribute
+
+class TileIndex(Model):
+    class Meta:
+        table_name            = "tile_cursor_index"
+        host                  = "http://localhost:8000"
+        region                = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    image_uid         = UnicodeAttribute(hash_key=True)
+    tile_uid          = UnicodeAttribute(range_key=True)
+    disaster_id       = UnicodeAttribute(null=True)
+    pair_id           = UnicodeAttribute(null=True)
+    tile_id           = UnicodeAttribute(null=True)
+    disaster_type     = UnicodeAttribute(null=True)
+    capture_date      = UnicodeAttribute(null=True)
+    building_count    = NumberAttribute(default=0)
+    classification    = UnicodeAttribute(null=True)
+    prediction        = UnicodeAttribute(null=True)
+    pre_image_path    = UnicodeAttribute(null=True)
+    post_image_path   = UnicodeAttribute(null=True)
+    target_image_path = UnicodeAttribute(null=True)
+    full_record_key   = UnicodeAttribute(null=True)
+
+class HouseIndex(Model):
+    class Meta:
+        table_name            = "house_cursor_index"
+        host                  = "http://localhost:8000"
+        region                = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    image_uid         = UnicodeAttribute(hash_key=True)
+    house_uid         = UnicodeAttribute(range_key=True)
+    disaster_id       = UnicodeAttribute(null=True)
+    pair_id           = UnicodeAttribute(null=True)
+    house_id          = UnicodeAttribute(null=True)
+    disaster_type     = UnicodeAttribute(null=True)
+    capture_date      = UnicodeAttribute(null=True)
+    classification    = UnicodeAttribute(null=True)
+    prediction        = UnicodeAttribute(null=True)
+    pre_image_path    = UnicodeAttribute(null=True)
+    post_image_path   = UnicodeAttribute(null=True)
+    target_image_path = UnicodeAttribute(null=True)
+    full_record_key   = UnicodeAttribute(null=True)
+
+class TileFull(Model):
+    class Meta:
+        table_name            = "tile_cursor_full"
+        host                  = "http://localhost:8000"
+        region                = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    image_uid           = UnicodeAttribute(hash_key=True)
+    tile_uid            = UnicodeAttribute(range_key=True)
+    disaster_id         = UnicodeAttribute(null=True)
+    pair_id             = UnicodeAttribute(null=True)
+    tile_id             = UnicodeAttribute(null=True)
+    sensor              = UnicodeAttribute(null=True)
+    provider_asset_type = UnicodeAttribute(null=True)
+    gsd                 = UnicodeAttribute(null=True)
+    capture_date        = UnicodeAttribute(null=True)
+    off_nadir_angle     = UnicodeAttribute(null=True)
+    pan_resolution      = UnicodeAttribute(null=True)
+    sun_azimuth         = UnicodeAttribute(null=True)
+    sun_elevation       = UnicodeAttribute(null=True)
+    target_azimuth      = UnicodeAttribute(null=True)
+    disaster_type       = UnicodeAttribute(null=True)
+    catalog_id          = UnicodeAttribute(null=True)
+    original_width      = NumberAttribute(null=True)
+    original_height     = NumberAttribute(null=True)
+    crop_bounds         = JSONAttribute()
+    buildings           = JSONAttribute()
+    json_path           = UnicodeAttribute(null=True)
+
+class HouseFull(Model):
+    class Meta:
+        table_name            = "house_cursor_full"
+        host                  = "http://localhost:8000"
+        region                = "us-west-2"
+        aws_access_key_id     = "fake"
+        aws_secret_access_key = "fake"
+
+    image_uid           = UnicodeAttribute(hash_key=True)
+    house_uid           = UnicodeAttribute(range_key=True)
+    disaster_id         = UnicodeAttribute(null=True)
+    pair_id             = UnicodeAttribute(null=True)
+    house_id            = UnicodeAttribute(null=True)
+    sensor              = UnicodeAttribute(null=True)
+    provider_asset_type = UnicodeAttribute(null=True)
+    gsd                 = UnicodeAttribute(null=True)
+    capture_date        = UnicodeAttribute(null=True)
+    off_nadir_angle     = UnicodeAttribute(null=True)
+    pan_resolution      = UnicodeAttribute(null=True)
+    sun_azimuth         = UnicodeAttribute(null=True)
+    sun_elevation       = UnicodeAttribute(null=True)
+    target_azimuth      = UnicodeAttribute(null=True)
+    disaster_type       = UnicodeAttribute(null=True)
+    catalog_id          = UnicodeAttribute(null=True)
+    original_width      = NumberAttribute(null=True)
+    original_height     = NumberAttribute(null=True)
+    crop_bounds         = JSONAttribute()
+    points              = JSONAttribute()
+    classification      = UnicodeAttribute(null=True)
+    prediction          = UnicodeAttribute(null=True)
+    json_path           = UnicodeAttribute(null=True)