Async thumbnails generation (#372)

Author: DenysIsaiev

backend/database/images.py

@@ -70,6 +70,7 @@ def create_from_path(cls, path, dataset_id=None):
         image.path = path
         image.width = pil_image.size[0]
         image.height = pil_image.size[1]
+        image.regenerate_thumbnail = True
 
         if dataset_id is not None:
             image.dataset_id = dataset_id
@@ -94,15 +95,12 @@ def delete(self, *args, **kwargs):
 
     def thumbnail(self):
         """
-        Generates (if required) and returns thumbnail
+        Generates (if required) thumbnail
         """
 
         thumbnail_path = self.thumbnail_path()
 
-        if self.regenerate_thumbnail or \
-            not os.path.isfile(thumbnail_path):
-            
-            # logger.debug(f'Generating thumbnail for {self.id}')
+        if self.regenerate_thumbnail:
 
             pil_image = self.generate_thumbnail()
             pil_image = pil_image.convert("RGB")
@@ -116,9 +114,14 @@ def thumbnail(self):
 
             self.update(is_modified=False)
             return pil_image
-        else:
-            return Image.open(thumbnail_path)
-    
+
+    def open_thumbnail(self):
+        """
+        Return thumbnail
+        """
+        thumbnail_path = self.thumbnail_path()
+        return Image.open(thumbnail_path)
+
     def thumbnail_path(self):
         folders = self.path.split('/')
         folders.insert(len(folders)-1, self.THUMBNAIL_DIRECTORY)

backend/webserver/__init__.py

@@ -20,7 +20,7 @@
 
 from .watcher import run_watcher
 from .api import blueprint as api
-from .util import query_util
+from .util import query_util, thumbnails
 from .authentication import login_manager
 from .sockets import socketio
 
@@ -55,6 +55,7 @@ def create_app():
     # Remove all poeple who were annotating when
     # the server shutdown
     ImageModel.objects.update(annotating=[])
+    thumbnails.generate_thumbnails()
 
     return flask
 

backend/webserver/api/annotator.py

@@ -4,7 +4,7 @@
 from flask_login import login_required, current_user
 from flask import request
 
-from ..util import query_util, coco_util, profile
+from ..util import query_util, coco_util, profile, thumbnails
 
 from config import Config
 from database import (
@@ -141,6 +141,8 @@ def post(self):
             set__num_annotations=num_annotations
         )
 
+        thumbnails.generate_thumbnail(image_model)
+
         return {"success": True}
 
 

backend/webserver/api/images.py

@@ -132,9 +132,8 @@ def get(self, image_id):
         if not height:
             height = image.height
 
-        pil_image = image.thumbnail() if thumbnail else Image.open(image.path)
-        image.flag_thumbnail(flag=False)
-        
+        pil_image = image.open_thumbnail() if thumbnail else Image.open(image.path)
+
         pil_image.thumbnail((width, height), Image.ANTIALIAS)
         image_io = io.BytesIO()
         pil_image = pil_image.convert("RGB")

backend/webserver/util/thumbnails.py

@@ -0,0 +1,12 @@
+from database import ImageModel
+
+
+def generate_thumbnails():
+    PREFIX = "[Thumbnails]"
+    print(f'{PREFIX} Sending request for regenerating images with non actual thumbnails', flush=True)
+    [generate_thumbnail(image) for image in ImageModel.objects(regenerate_thumbnail=True).all()]
+
+
+def generate_thumbnail(image):
+    from workers.tasks import thumbnail_generate_single_image
+    thumbnail_generate_single_image.delay(image.id)

backend/webserver/watcher.py

@@ -3,6 +3,7 @@
 
 from config import Config
 from database import ImageModel
+from .util.thumbnails import generate_thumbnail
 
 import re
 
@@ -32,11 +33,13 @@ def on_any_event(self, event):
 
         if image is None and event.event_type != 'deleted':
             self._log(f'Adding new file to database: {path}')
-            ImageModel.create_from_path(path).save()
+            image = ImageModel.create_from_path(path).save()
+            generate_thumbnail(image)
 
         elif event.event_type == 'moved':
             self._log(f'Moving image from {event.src_path} to {path}')
             image.update(path=path)
+            generate_thumbnail(image)
 
         elif event.event_type == 'deleted':
             self._log(f'Deleting image from database {path}')

backend/workers/tasks/__init__.py

@@ -1,4 +1,5 @@
 
 from .data import *
 from .test import *
-from .scan import *
+from .scan import *
+from .thumbnails import *

backend/workers/tasks/scan.py

@@ -6,6 +6,7 @@
 
 from celery import shared_task
 from ..socket import create_socket
+from .thumbnails import thumbnail_generate_single_image
 
 import os
 
@@ -52,6 +53,8 @@ def scan_dataset(task_id, dataset_id):
                 except:
                     task.warning(f"Could not read {path}")
 
+    [thumbnail_generate_single_image.delay(image.id) for image in ImageModel.objects(regenerate_thumbnail=True).all()]
+
     task.info(f"Created {count} new image(s)")
     task.set_progress(100, socket=socket)
 

backend/workers/tasks/thumbnails.py

@@ -0,0 +1,12 @@
+from database import ImageModel
+from celery import task
+
+
+@task
+def thumbnail_generate_single_image(image_id):
+    image = ImageModel.objects(id=image_id).first()
+    image.thumbnail()
+    image.flag_thumbnail(flag=False)
+
+
+__all__ = ["thumbnail_generate_single_image"]