BatBot initial testing and integration

README.md

@@ -11,9 +11,6 @@ This is the simplest configuration for developers to start with.
    and follow the prompts to create your own user
 3. Run `docker compose run --rm django ./manage.py loaddata species` to load species
    data into the database
-4. Run `docker compose run --rm django ./manage.py makeclient \
-                            --username your.super.user@email.address \
-                            --uri http://localhost:3000/`
 
 ### Run Vue Frontend
 
@@ -38,7 +35,7 @@ To non-destructively update your development stack at any time:
 
 ## Dev Tool Endpoints
 
-1. Main Site Interface [http://localhost:3000/](http://localhost:3000/)
+1. Main Site Interface [http://localhost:8080/](http://localhost:8080/)
 2. Site Administration [http://localhost:8000/admin/](http://localhost:8000/admin/)
 3. Swagger API (These are default swagger endpoints using Django-REST) [http://localhost:8000/api/docs/swagger/](http://localhost:8000/api/docs/swagger/)
 4. Django Ninja API [http://localhost:8000/api/v1/docs#/](http://localhost:8000/api/v1/docs#/)

bats_ai/core/management/commands/generateNABat.py

@@ -12,10 +12,10 @@
 
 from bats_ai.core.models import Species
 from bats_ai.core.models.nabat import NABatRecording, NABatRecordingAnnotation
+from bats_ai.core.utils.batbot_metadata import generate_spectrogram_assets
 from bats_ai.utils.spectrogram_utils import (
     generate_nabat_compressed_spectrogram,
     generate_nabat_spectrogram,
-    generate_spectrogram_assets,
 )
 
 fake = Faker()

bats_ai/core/management/commands/importRecordings.py

@@ -40,10 +40,6 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        import matplotlib
-
-        matplotlib.use('Agg')
-
         directory_path = Path(options['directory'])
         owner_username = options.get('owner')
         is_public = options.get('public', False)
@@ -78,7 +74,11 @@ def handle(self, *args, **options):
             self.stdout.write(self.style.WARNING(f'Using default owner: {owner.username}'))
 
         # Find all WAV files
-        wav_files = list(directory_path.rglob('*.wav', case_sensitive=False))
+        wav_files = list(
+            directory_path.rglob(
+                '*.wav',
+            )
+        )
 
         if not wav_files:
             self.stdout.write(

bats_ai/core/management/commands/loadGRTS.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import tempfile
+import urllib
 from urllib.request import urlretrieve
 import zipfile
 
@@ -18,27 +19,30 @@
         'https://www.sciencebase.gov/catalog/file/get/5b7753bde4b0f5d578820455?facet=conus_mastersample_10km_GRTS',  # noqa: E501
         14,
         'CONUS',
+        # Backup URL
+        'https://data.kitware.com/api/v1/item/697cc601e7dea9be44ec5aee/download',  # noqa: E501
     ),  # CONUS
-    (
-        'https://www.sciencebase.gov/catalog/file/get/5b7753a8e4b0f5d578820452?facet=akcan_mastersample_10km_GRTS',  # noqa: E501
-        20,
-        'Alaska/Canada',
-    ),  # Alaska/Canada
-    (
-        'https://www.sciencebase.gov/catalog/file/get/5b7753c2e4b0f5d578820457?facet=HI_mastersample_5km_GRTS',  # noqa: E501
-        15,
-        'Hawaii',
-    ),  # Hawaii
-    (
-        'https://www.sciencebase.gov/catalog/file/get/5b7753d3e4b0f5d578820459?facet=mex_mastersample_10km_GRTS',  # noqa: E501
-        12,
-        'Mexico',
-    ),  # Mexico
-    (
-        'https://www.sciencebase.gov/catalog/file/get/5b7753d8e4b0f5d57882045b?facet=PR_mastersample_5km_GRTS',  # noqa: E501
-        21,
-        'Puerto Rico',
-    ),  # Puerto Rico
+    # Removed other regions for now because of sciencebase.gov being down
+    # (
+    #     'https://www.sciencebase.gov/catalog/file/get/5b7753a8e4b0f5d578820452?facet=akcan_mastersample_10km_GRTS',  # noqa: E501
+    #     20,
+    #     'Alaska/Canada',
+    # ),  # Alaska/Canada
+    # (
+    #     'https://www.sciencebase.gov/catalog/file/get/5b7753c2e4b0f5d578820457?facet=HI_mastersample_5km_GRTS',  # noqa: E501
+    #     15,
+    #     'Hawaii',
+    # ),  # Hawaii
+    # (
+    #     'https://www.sciencebase.gov/catalog/file/get/5b7753d3e4b0f5d578820459?facet=mex_mastersample_10km_GRTS',  # noqa: E501
+    #     12,
+    #     'Mexico',
+    # ),  # Mexico
+    # (
+    #     'https://www.sciencebase.gov/catalog/file/get/5b7753d8e4b0f5d57882045b?facet=PR_mastersample_5km_GRTS',  # noqa: E501
+    #     21,
+    #     'Puerto Rico',
+    # ),  # Puerto Rico
 ]
 
 
@@ -56,11 +60,26 @@ def handle(self, *args, **options):
         # Track existing IDs to avoid duplicates
         existing_ids = set(GRTSCells.objects.values_list('id', flat=True))
 
-        for url, sample_frame_id, name in SHAPEFILES:
+        for url, sample_frame_id, name, backup_url in SHAPEFILES:
             logger.info(f'Downloading shapefile for Location {name}...')
             with tempfile.TemporaryDirectory() as tmpdir:
                 zip_path = os.path.join(tmpdir, 'file.zip')
-                urlretrieve(url, zip_path)
+                try:
+                    urlretrieve(url, zip_path)
+                except urllib.error.URLError as e:
+                    logger.warning(
+                        f'Failed to download from primary URL: {e}. \
+                            Attempting backup URL...'
+                    )
+                    if backup_url is None:
+                        logger.warning('No backup URL provided, skipping this shapefile.')
+                        continue
+                    try:
+                        urlretrieve(backup_url, zip_path)
+                    except urllib.error.URLError as e2:
+                        raise CommandError(
+                            f'Failed to download from backup URL as well: {e2}'
+                        ) from e2
                 logger.info(f'Downloaded to {zip_path}')
 
                 logger.info('Extracting zip file...')

bats_ai/core/migrations/0027_alter_annotations_end_time_and_more.py

@@ -0,0 +1,143 @@
+# Generated by Django 4.2.23 on 2026-02-03 13:05
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('core', '0026_merge'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='annotations',
+            name='end_time',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='annotations',
+            name='high_freq',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='annotations',
+            name='low_freq',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='annotations',
+            name='start_time',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='compressedspectrogram',
+            name='length',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='compressedspectrogram',
+            name='starts',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='compressedspectrogram',
+            name='stops',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='compressedspectrogram',
+            name='widths',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='nabatcompressedspectrogram',
+            name='length',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='nabatcompressedspectrogram',
+            name='starts',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='nabatcompressedspectrogram',
+            name='stops',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='nabatcompressedspectrogram',
+            name='widths',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=django.contrib.postgres.fields.ArrayField(
+                    base_field=models.FloatField(), size=None
+                ),
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='nabatspectrogram',
+            name='duration',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='nabatspectrogram',
+            name='frequency_max',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='nabatspectrogram',
+            name='frequency_min',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='sequenceannotations',
+            name='end_time',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='sequenceannotations',
+            name='start_time',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='spectrogram',
+            name='duration',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='spectrogram',
+            name='frequency_max',
+            field=models.FloatField(),
+        ),
+        migrations.AlterField(
+            model_name='spectrogram',
+            name='frequency_min',
+            field=models.FloatField(),
+        ),
+    ]

bats_ai/core/models/annotations.py

@@ -10,10 +10,10 @@
 class Annotations(TimeStampedModel, models.Model):
     recording = models.ForeignKey(Recording, on_delete=models.CASCADE)
     owner = models.ForeignKey(User, on_delete=models.CASCADE)
-    start_time = models.IntegerField(blank=True, null=True)
-    end_time = models.IntegerField(blank=True, null=True)
-    low_freq = models.IntegerField(blank=True, null=True)
-    high_freq = models.IntegerField(blank=True, null=True)
+    start_time = models.FloatField(blank=True, null=True)
+    end_time = models.FloatField(blank=True, null=True)
+    low_freq = models.FloatField(blank=True, null=True)
+    high_freq = models.FloatField(blank=True, null=True)
     type = models.TextField(blank=True, null=True)
     species = models.ManyToManyField(Species)
     comments = models.TextField(blank=True, null=True)

bats_ai/core/models/compressed_spectrogram.py

@@ -15,11 +15,11 @@
 class CompressedSpectrogram(TimeStampedModel, models.Model):
     recording = models.ForeignKey(Recording, on_delete=models.CASCADE)
     spectrogram = models.ForeignKey(Spectrogram, on_delete=models.CASCADE)
-    length = models.IntegerField()
+    length = models.FloatField()
     images = GenericRelation(SpectrogramImage)
-    starts = ArrayField(ArrayField(models.IntegerField()))
-    stops = ArrayField(ArrayField(models.IntegerField()))
-    widths = ArrayField(ArrayField(models.IntegerField()))
+    starts = ArrayField(ArrayField(models.FloatField()))
+    stops = ArrayField(ArrayField(models.FloatField()))
+    widths = ArrayField(ArrayField(models.FloatField()))
     cache_invalidated = models.BooleanField(default=True)
 
     @property

bats_ai/core/models/nabat/nabat_compressed_spectrogram.py

@@ -17,10 +17,10 @@ class NABatCompressedSpectrogram(TimeStampedModel, models.Model):
     nabat_recording = models.ForeignKey(NABatRecording, on_delete=models.CASCADE)
     spectrogram = models.ForeignKey(NABatSpectrogram, on_delete=models.CASCADE)
     images = GenericRelation(SpectrogramImage)
-    length = models.IntegerField()
-    starts = ArrayField(ArrayField(models.IntegerField()))
-    stops = ArrayField(ArrayField(models.IntegerField()))
-    widths = ArrayField(ArrayField(models.IntegerField()))
+    length = models.FloatField()
+    starts = ArrayField(ArrayField(models.FloatField()))
+    stops = ArrayField(ArrayField(models.FloatField()))
+    widths = ArrayField(ArrayField(models.FloatField()))
     cache_invalidated = models.BooleanField(default=True)
 
     @property

bats_ai/core/models/nabat/nabat_spectrogram.py

@@ -20,9 +20,9 @@ class NABatSpectrogram(TimeStampedModel, models.Model):
     images = GenericRelation(SpectrogramImage)
     width = models.IntegerField()  # pixels
     height = models.IntegerField()  # pixels
-    duration = models.IntegerField()  # milliseconds
-    frequency_min = models.IntegerField()  # hz
-    frequency_max = models.IntegerField()  # hz
+    duration = models.FloatField()  # milliseconds
+    frequency_min = models.FloatField()  # hz
+    frequency_max = models.FloatField()  # hz
 
     @property
     def image_url_list(self):

bats_ai/core/models/sequence_annotations.py

@@ -8,8 +8,8 @@
 class SequenceAnnotations(models.Model):
     recording = models.ForeignKey(Recording, on_delete=models.CASCADE)
     owner = models.ForeignKey(User, on_delete=models.CASCADE)
-    start_time = models.IntegerField(blank=True, null=True)
-    end_time = models.IntegerField(blank=True, null=True)
+    start_time = models.FloatField(blank=True, null=True)
+    end_time = models.FloatField(blank=True, null=True)
     type = models.TextField(blank=True, null=True)
     comments = models.TextField(blank=True, null=True)
     species = models.ManyToManyField(Species)

bats_ai/core/models/spectrogram.py

@@ -14,9 +14,9 @@ class Spectrogram(TimeStampedModel, models.Model):
     images = GenericRelation(SpectrogramImage)
     width = models.IntegerField()  # pixels
     height = models.IntegerField()  # pixels
-    duration = models.IntegerField()  # milliseconds
-    frequency_min = models.IntegerField()  # hz
-    frequency_max = models.IntegerField()  # hz
+    duration = models.FloatField()  # milliseconds
+    frequency_min = models.FloatField()  # hz
+    frequency_max = models.FloatField()  # hz
 
     @property
     def image_url_list(self):

bats_ai/core/tasks/nabat/tasks.py

@@ -6,10 +6,10 @@
 
 from bats_ai.core.models import Configuration, ProcessingTask, Species
 from bats_ai.core.models.nabat import NABatRecording, NABatRecordingAnnotation
+from bats_ai.core.utils.batbot_metadata import generate_spectrogram_assets
 from bats_ai.utils.spectrogram_utils import (
     generate_nabat_compressed_spectrogram,
     generate_nabat_spectrogram,
-    generate_spectrogram_assets,
     predict_from_compressed,
 )
 
@@ -60,7 +60,9 @@ def generate_spectrograms(
 
         try:
             config = Configuration.objects.first()
-            if config and config.run_inference_on_upload:
+            # TODO: Disabled until prediction is in batbot
+            # https://github.com/Kitware/batbot/issues/29
+            if config and config.run_inference_on_upload and False:
                 self.update_state(
                     state='Progress',
                     meta={'description': 'Running Prediction on Spectrogram'},