Feature/thumbnails

CHANGELOG

@@ -1,3 +1,7 @@
+2.5.0
+2024-06-03
+Add support for storing thumbnail images associated with a frame
+
 2.4.1
 2024-04-15
 

archive/frames/filters.py

@@ -1,10 +1,11 @@
-from archive.frames.models import Frame
+from archive.frames.models import Frame, Thumbnail
 from archive.frames.utils import get_configuration_type_tuples
 from archive.settings import SCIENCE_CONFIGURATION_TYPES
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.gis.geos.error import GEOSException
 from rest_framework.exceptions import ValidationError
 from django.utils import timezone
+from django.conf import settings
 from django_filters import rest_framework as django_filters
 
 
@@ -88,3 +89,15 @@ class Meta:
         fields = ['proposal_id', 'configuration_type', 'instrument_id',
                   'reduction_level', 'site_id', 'telescope_id', 'primary_optical_element',
                   'observation_id', 'request_id']
+
+
+class ThumbnailFilter(django_filters.FilterSet):
+    frame_basename = django_filters.CharFilter(field_name='frame__basename', lookup_expr='exact')
+    proposal_id = django_filters.CharFilter(field_name='frame__proposal_id', lookup_expr='exact')
+    observation_id = django_filters.NumberFilter(field_name='frame__observation_id', lookup_expr='exact')
+    request_id = django_filters.NumberFilter(field_name='frame__request_id', lookup_expr='exact')
+    size = django_filters.ChoiceFilter(choices=[(size,size) for size in settings.THUMBNAIL_SIZE_CHOICES], field_name='size', lookup_expr='exact')
+
+    class Meta:
+        model = Thumbnail
+        fields = ['frame_basename', 'proposal_id', 'observation_id', 'request_id', 'size']

archive/frames/migrations/0021_add_thumbnail_model.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.9 on 2024-05-20 23:28
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('frames', '0020_alter_instrument_id_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Thumbnail',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('size', models.CharField(help_text='String description of the size of the thumbnail', max_length=20)),
+                ('basename', models.CharField(db_index=True, help_text='The basename of the thumbnail', max_length=1000, unique=True)),
+                ('extension', models.CharField(help_text='The file extension of the thumbnail', max_length=20)),
+                ('key', models.CharField(default='', help_text='The key used to store the thumbnail in the file store', max_length=32)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='thumbnail',
+            name='frame',
+            field=models.ForeignKey(help_text='The frame this thumbnail is associated with', on_delete=django.db.models.deletion.CASCADE, related_name='thumbnails', to='frames.frame'),
+        ),
+    ]

archive/frames/models.py

@@ -133,11 +133,13 @@ def get_header_dict(self):
             archive_settings.PUBLIC_DATE_KEY: self.public_date,
         }
 
-    def as_dict(self):
+    def as_dict(self, include_thumbnails=False):
         ret_dict = model_to_dict(self, exclude=('related_frames', 'area'))
         ret_dict['version_set'] = [v.as_dict() for v in self.version_set.all()]
-        ret_dict['url'] = self.url
-        ret_dict['filename'] = self.filename
+        ret_dict['url'] = self.url if self.version_set.exists() else None
+        ret_dict['filename'] = self.filename if self.version_set.exists() else None
+        if include_thumbnails:
+            ret_dict['thumbnails'] = [t.as_dict() for t in Thumbnail.objects.filter(frame=self)]
         # TODO: Remove these old model field names once users have migrated their code
         ret_dict['DATE_OBS'] = ret_dict['observation_date']
         ret_dict['DAY_OBS'] = ret_dict['observation_day']
@@ -159,6 +161,64 @@ def as_dict(self):
         ret_dict['related_frames'] = list(self.related_frames.all().values_list('id', flat=True))
         return ret_dict
 
+
+class Thumbnail(models.Model):
+    frame = models.ForeignKey(
+        Frame,
+        on_delete=models.CASCADE,
+        related_name='thumbnails',
+        help_text="The frame this thumbnail is associated with"
+    )
+    size = models.CharField(
+        max_length=20,
+        help_text="String description of the size of the thumbnail"
+    )
+    basename = models.CharField(
+        max_length=1000,
+        db_index=True,
+        unique=True,
+        help_text="The basename of the thumbnail"
+    )
+    extension = models.CharField(
+        max_length=20,
+        help_text="The file extension of the thumbnail"
+    )
+    key = models.CharField(
+        max_length=32,
+        help_text="The key used to store the thumbnail in the file store",
+        default=''
+    )
+
+    def as_dict(self):
+        ret_dict = model_to_dict(self)
+        ret_dict['url'] = self.url
+        ret_dict['size'] = self.size
+        return ret_dict
+
+    @property
+    def filename(self):
+        """
+        Returns the full filename for the thumbnail
+        """
+        return '{0}{1}'.format(self.basename, self.extension)
+
+    @cached_property
+    def url(self):
+        path = get_file_store_path(self.filename, {'SITEID': self.frame.site_id, 'INSTRUME': self.frame.instrument_id, 'TELID': self.frame.telescope_id,
+                                                   'DAY-OBS': self.frame.observation_day.strftime('%Y%m%d'), 'DATE-OBS': self.frame.observation_date.isoformat(),
+                                                   'frame_basename': self.frame.basename, 'size': self.size})
+        file_store = FileStoreFactory.get_file_store_class()()
+        return file_store.get_url(path, self.key, expiration=3600 * 48)
+
+    def delete_data(self):
+        logger.info('Deleting thumbnail', extra={'tags': {'key': self.key, 'frame': self.frame.id, 'thumbnail': self.basename}})
+        path = get_file_store_path(self.filename, {'SITEID': self.frame.site_id, 'INSTRUME': self.frame.instrument_id, 'TELID': self.frame.telescope_id,
+                                                   'DAY-OBS': self.frame.observation_day.strftime('%Y%m%d'), 'DATE-OBS': self.frame.observation_date.isoformat(),
+                                                   'frame_basename': self.frame.basename, 'size': self.size})
+        file_store = FileStoreFactory.get_file_store_class()()
+        file_store.delete_file(path, self.key)
+
+
 class Headers(models.Model):
     data = JSONField(default=dict)
     frame = models.OneToOneField(Frame, on_delete=models.CASCADE)

archive/frames/serializers.py

@@ -1,6 +1,6 @@
 import json
 from rest_framework import serializers
-from archive.frames.models import Frame, Version, Headers
+from archive.frames.models import Frame, Version, Headers, Thumbnail
 from archive.frames.utils import get_configuration_type_tuples
 from django.contrib.gis.geos import GEOSGeometry
 from django.db import transaction
@@ -87,7 +87,7 @@ class Meta:
         # }
 
     def create(self, validated_data):
-        version_data = validated_data.pop('version_set')
+        version_data = validated_data.pop('version_set') if 'version_set' in validated_data else {}
         header_data = validated_data.pop('headers')
         related_frames = validated_data.pop('related_frame_filenames')
         with transaction.atomic():
@@ -116,6 +116,23 @@ def create_related_frames(self, frame, data):
         frame.save()
 
 
+class ThumbnailSerializer(serializers.ModelSerializer):
+    size = serializers.ChoiceField(choices=settings.THUMBNAIL_SIZE_CHOICES, help_text='Size of the thumbnail')
+    url = serializers.CharField(read_only=True, help_text='Download URL for thumbnail')
+    basename = serializers.CharField(required=True, help_text='Filename of the thumbnail')
+    key = serializers.CharField(required=True, help_text='Key for the thumbnail in the file store')
+    extension = serializers.CharField(required=True, help_text='File extension of the thumbnail')
+    frame = FrameSerializer(read_only=True, help_text='Frame associated with this thumbnail')
+
+    class Meta:
+        model = Thumbnail
+        fields = ['frame', 'size', 'basename', 'url', 'key', 'extension']
+
+    def create(self, validated_data):
+        thumbnail, _ = Thumbnail.objects.update_or_create(defaults=validated_data, basename=validated_data['basename'])
+        return thumbnail
+
+
 class AggregateSerializer(serializers.Serializer):
     sites = serializers.ListField(child=serializers.CharField())
     telescopes = serializers.ListField(child=serializers.CharField())

archive/frames/signals/handlers.py

@@ -1,9 +1,13 @@
 from django.dispatch import receiver
 from django.db.models.signals import post_delete
 
-from archive.frames.models import Version
+from archive.frames.models import Version, Thumbnail
 
 
 @receiver(post_delete, sender=Version)
 def version_post_delete(sender, instance, *args, **kwargs):
     instance.delete_data()
+
+@receiver(post_delete, sender=Thumbnail)
+def thumbnail_post_delete(sender, instance, *args, **kwargs):
+    instance.delete_data()

archive/frames/tests/factories.py

@@ -4,7 +4,7 @@
 import datetime
 import json
 import os
-from archive.frames.models import Frame, Version, Headers
+from archive.frames.models import Frame, Thumbnail, Version, Headers
 from django.utils import timezone
 from pytz import UTC
 
@@ -186,3 +186,14 @@ def version_set(self, create, extracted, **kwargs):
 
 class PublicFrameFactory(FrameFactory):
     public_date = datetime.datetime(2000, 1, 1, tzinfo=UTC)
+
+
+class ThumbnailFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Thumbnail
+
+    size = factory.fuzzy.FuzzyChoice(['small', 'medium', 'large'])
+    basename = factory.fuzzy.FuzzyText(length=10)
+    extension = factory.fuzzy.FuzzyChoice(['.jpeg', '.jpg'])
+    key = factory.fuzzy.FuzzyText(length=32)
+    frame = factory.SubFactory('archive.frames.tests.factories.FrameFactory')