Publish v0.110

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## v0.110 - 2023-07-31
+* Multilingual templates
+* Support images in templates
+* Support creating templates from findings
+* UI: Move secondary toolbar actions to a dropdown menu
+* UI: Sticky Add button in finding and note list sidebars
+* Fix redirect after login for remoteuser default auth provider
+
+
 ## v0.102 - 2023-07-05
 * Fix serialization of project check messages
 

README.md

@@ -75,4 +75,4 @@ Get detailed installation instructions at [Installation](https://docs.sysreptor.
 
 ![Create finding from template](https://docs.sysreptor.com/images/create_finding_from_template.gif)
 
-![Export report as PDF](https://docs.sysreptor.com/images/export_project.gif)
+![Export report as PDF](https://docs.sysreptor.com/images/export_project.gif)

api/src/reportcreator_api/api_utils/backup_utils.py

@@ -12,7 +12,7 @@
 
 from reportcreator_api.archive import crypto
 from reportcreator_api.pentests.models import UploadedImage, UploadedAsset, UploadedProjectFile, ArchivedProject, \
-    UploadedUserNotebookFile, UploadedUserNotebookImage
+    UploadedUserNotebookFile, UploadedUserNotebookImage, UploadedTemplateImage
 
 
 def create_database_dump():
@@ -60,6 +60,7 @@ def create_backup():
 
     backup_files(z, UploadedImage, 'uploadedimages')
     backup_files(z, UploadedUserNotebookImage, 'uploadedimages')
+    backup_files(z, UploadedTemplateImage, 'uploadedimages')
     backup_files(z, UploadedAsset, 'uploadedassets')
     backup_files(z, UploadedProjectFile, 'uploadedfiles')
     backup_files(z, UploadedUserNotebookFile, 'uploadedfiles')

api/src/reportcreator_api/api_utils/healthchecks.py

@@ -1,5 +1,3 @@
-import functools
-import operator
 import uuid
 from django.utils.module_loading import import_string
 from django.core.cache import cache

api/src/reportcreator_api/archive/import_export/import_export.py

@@ -12,7 +12,8 @@
 from django.db.models import prefetch_related_objects, Prefetch
 from django.core.serializers.json import DjangoJSONEncoder
 
-from reportcreator_api.archive.import_export.serializers import FindingTemplateExportImportSerializer, PentestProjectExportImportSerializer, ProjectTypeExportImportSerializer
+from reportcreator_api.archive.import_export.serializers import PentestProjectExportImportSerializer, ProjectTypeExportImportSerializer, \
+    FindingTemplateImportSerializerV1, FindingTemplateExportImportSerializerV2
 from reportcreator_api.pentests.models import FindingTemplate, NotebookPage, PentestFinding, PentestProject, ProjectMemberInfo, ProjectType, ReportSection
 
 
@@ -111,8 +112,8 @@ def export_archive_iter(data, serializer_class: Type[serializers.Serializer], co
 
 
 @transaction.atomic()
-def import_archive(archive_file, serializer_class: Type[serializers.Serializer]):
-    context = {
+def import_archive(archive_file, serializer_classes: list[Type[serializers.Serializer]], context):
+    context = (context or {}) | {
         'archive': None,
         'storage_files': [],
     }
@@ -135,8 +136,23 @@ def import_archive(archive_file, serializer_class: Type[serializers.Serializer])
             # The actual work is performed in serializers
             imported_objects = []
             for m in to_import:
-                serializer = serializer_class(data=json.load(archive.extractfile(m)), context=context)
-                serializer.is_valid(raise_exception=True)
+                data = json.load(archive.extractfile(m))
+
+                serializer = None
+                error = None
+                for serializer_class in serializer_classes:
+                    try:
+                        serializer = serializer_class(data=data, context=context)
+                        serializer.is_valid(raise_exception=True)
+                        error = None
+                        break
+                    except Exception as ex:
+                        serializer = None
+                        # Use error of the first failing serializer_class
+                        if not error:
+                            error = ex
+                if error:
+                    raise error
                 obj = serializer.perform_import()
                 log.info(f'Imported object {obj=} {obj.id=}')
                 imported_objects.append(obj)
@@ -155,7 +171,7 @@ def import_archive(archive_file, serializer_class: Type[serializers.Serializer])
 
 
 def export_templates(data: Iterable[FindingTemplate]):
-    return export_archive_iter(data, serializer_class=FindingTemplateExportImportSerializer)
+    return export_archive_iter(data, serializer_class=FindingTemplateExportImportSerializerV2)
 
 def export_project_types(data: Iterable[ProjectType]):
     prefetch_related_objects(data, 'assets')
@@ -176,12 +192,12 @@ def export_projects(data: Iterable[PentestProject], export_all=False):
     })
 
 
-def import_templates(archive_file):
-    return import_archive(archive_file, serializer_class=FindingTemplateExportImportSerializer)
+def import_templates(archive_file, uploaded_by=None):
+    return import_archive(archive_file, serializer_classes=[FindingTemplateExportImportSerializerV2, FindingTemplateImportSerializerV1], context={'uploaded_by': uploaded_by})
 
-def import_project_types(archive_file):
-    return import_archive(archive_file, serializer_class=ProjectTypeExportImportSerializer)
+def import_project_types(archive_file, uploaded_by=None):
+    return import_archive(archive_file, serializer_classes=[ProjectTypeExportImportSerializer], context={'uploaded_by': uploaded_by})
 
-def import_projects(archive_file):
-    return import_archive(archive_file, serializer_class=PentestProjectExportImportSerializer)
+def import_projects(archive_file, uploaded_by=None):
+    return import_archive(archive_file, serializer_classes=[PentestProjectExportImportSerializer], context={'uploaded_by': uploaded_by})
 

api/src/reportcreator_api/archive/import_export/serializers.py

@@ -5,7 +5,8 @@
 from reportcreator_api.pentests.customfields.utils import HandleUndefinedFieldsOptions, ensure_defined_structure
 
 from reportcreator_api.pentests.models import FindingTemplate, NotebookPage, PentestFinding, PentestProject, ProjectType, ReportSection, \
-    SourceEnum, UploadedAsset, UploadedImage, UploadedFileBase, ProjectMemberInfo, UploadedProjectFile
+    SourceEnum, UploadedAsset, UploadedImage, UploadedFileBase, ProjectMemberInfo, UploadedProjectFile, Language, ReviewStatus, \
+    FindingTemplateTranslation, UploadedTemplateImage
 from reportcreator_api.pentests.serializers import ProjectMemberInfoSerializer
 from reportcreator_api.users.models import PentestUser
 from reportcreator_api.users.serializers import RelatedUserSerializer
@@ -113,26 +114,6 @@ def to_internal_value(self, data):
             raise serializers.SkipField()
 
 
-class FindingTemplateExportImportSerializer(ExportImportSerializer):
-    format = FormatField('templates/v1')
-
-    data = serializers.DictField(source='data_all')
-
-    class Meta:
-        model = FindingTemplate
-        fields = ['format', 'id', 'created', 'updated', 'tags', 'language', 'status', 'data']
-        extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False}}
-
-    def create(self, validated_data):
-        data = validated_data.pop('data_all', {})
-        template = FindingTemplate(**{
-            'source': SourceEnum.IMPORTED,
-        } | validated_data)
-        template.update_data(data)
-        template.save()
-        return template
-
-
 class FileListExportImportSerializer(serializers.ListSerializer):
     def export_files(self):
         for e in self.instance:
@@ -150,7 +131,8 @@ def create(self, validated_data):
                 'file': File(
                     file=self.extract_file(attrs['name']), 
                     name=attrs['name']), 
-                'linked_object': self.child.get_linked_object()
+                'linked_object': self.child.get_linked_object(),
+                'uploaded_by': self.context.get('uploaded_by'),
         }) for attrs in validated_data]
 
         child_model_class.objects.bulk_create(objs)
@@ -179,6 +161,104 @@ def export_files(self) -> Iterable[tuple[str, File]]:
         yield self.get_path_in_archive(self.instance.name), self.instance.file
 
 
+class FindingTemplateImportSerializerV1(ExportImportSerializer):
+    format = FormatField('templates/v1')
+
+    language = serializers.ChoiceField(choices=Language.choices, source='main_translation__language')
+    status = serializers.ChoiceField(choices=ReviewStatus.choices, source='main_translation__status')
+    data = serializers.DictField(source='main_translation__data')
+
+    class Meta:
+        model = FindingTemplate
+        fields = ['format', 'id', 'created', 'updated', 'tags', 'language', 'status', 'data']
+        extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False}}
+
+    def create(self, validated_data):
+        main_translation_data = {k[len('main_translation__'):]: validated_data.pop(k) for k in validated_data.copy().keys() if k.startswith('main_translation__')}
+        template = FindingTemplate.objects.create(**{
+            'source': SourceEnum.IMPORTED,
+        } | validated_data)
+        data = main_translation_data.pop('data', {})
+        main_translation = FindingTemplateTranslation(template=template, **main_translation_data)
+        main_translation.update_data(data)
+        main_translation.save()
+        template.main_translation = main_translation
+        template.save()
+        return template
+
+
+class FindingTemplateTranslationExportImportSerializer(ExportImportSerializer):
+    data = serializers.DictField(source='data_all')
+    is_main = serializers.BooleanField()
+
+    class Meta:
+        model = FindingTemplateTranslation
+        fields = ['id', 'created', 'updated', 'is_main', 'language', 'status', 'data']
+        extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False}}
+
+    def create(self, validated_data):
+        data = validated_data.pop('data_all', {})
+        instance = FindingTemplateTranslation(**validated_data)
+        instance.update_data(data)
+        instance.save()
+        return instance
+
+
+class UploadedTemplateImageExportImportSerializer(FileExportImportSerializer):
+    class Meta(FileExportImportSerializer.Meta):
+        model = UploadedTemplateImage
+
+    def get_linked_object(self):
+        return self.context['template']
+
+    def get_path_in_archive(self, name):
+        # Get ID of old project_type from archive
+        return str(self.context.get('template_id') or self.get_linked_object().id) + '-images/' + name
+
+
+class FindingTemplateExportImportSerializerV2(ExportImportSerializer):
+    format = FormatField('templates/v2')
+    translations = FindingTemplateTranslationExportImportSerializer(many=True, allow_empty=False)
+    images = UploadedTemplateImageExportImportSerializer(many=True, required=False)
+
+    class Meta:
+        model = FindingTemplate
+        fields = ['format', 'id', 'created', 'updated', 'tags', 'translations', 'images']
+        extra_kwargs = {'id': {'read_only': False}, 'created': {'read_only': False}}
+
+    def validate_translations(self, value):
+        if len(list(filter(lambda t: t.get('is_main'), value))) != 1:
+            raise serializers.ValidationError('No main translation given')
+        if len(set(map(lambda t: t.get('language'), value))) != len(value):
+            raise serializers.ValidationError('Duplicate template language detected')
+        return value
+
+    def export_files(self) -> Iterable[tuple[str, File]]:
+        self.context.update({'template': self.instance})
+        imgf = self.fields['images']
+        imgf.instance = list(imgf.get_attribute(self.instance).all())
+        yield from imgf.export_files()
+
+    def create(self, validated_data):
+        old_id = validated_data.pop('id')
+        images_data = validated_data.pop('images', [])
+        translations_data = validated_data.pop('translations')
+        instance = FindingTemplate.objects.create(**{
+            'source': SourceEnum.IMPORTED
+        } | validated_data)
+        self.context['template'] = instance
+        for t in translations_data:
+            is_main = t.pop('is_main', False)
+            translation_instance = self.fields['translations'].child.create(t | {'template': instance})
+            if is_main:
+                instance.main_translation = translation_instance
+        instance.save()
+
+        self.context.update({'template': instance, 'template_id': old_id})
+        self.fields['images'].create(images_data)
+        return instance
+
+
 class UploadedImageExportImportSerializer(FileExportImportSerializer):
     class Meta(FileExportImportSerializer.Meta):
         model = UploadedImage
@@ -271,7 +351,7 @@ def create(self, validated_data):
             value=data,
             definition=project.project_type.finding_fields_obj,
             handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE,
-            include_undefined=True)
+            include_unknown=True)
         )
         finding.save()
         return finding
@@ -389,7 +469,7 @@ def create(self, validated_data):
                 value=report_data,
                 definition=project_type.report_fields_obj,
                 handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE,
-                include_undefined=True
+                include_unknown=True
             ),    
         })
         project_type.linked_project = project

api/src/reportcreator_api/conf/settings.py

@@ -394,6 +394,8 @@
 
 # Generate HTTPS URIs in responses for requests behind a reverse proxy
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+USE_X_FORWARDED_HOST = config('USE_X_FORWARDED_HOST', cast=bool, default=False)
+USE_X_FORWARDED_PORT = config('USE_X_FORWARDED_PORT', cast=bool, default=False)
 
 
 # Monkey-Patch django to disable CSRF everywhere
@@ -522,9 +524,9 @@
 
 # Health checks
 HEALTH_CHECKS = {
-    'cache': 'reportcreator_api.api_utils.healthchecks.check_cache',
     'database': 'reportcreator_api.api_utils.healthchecks.check_database',
     'migrations': 'reportcreator_api.api_utils.healthchecks.check_migrations',
+    # 'cache': 'reportcreator_api.api_utils.healthchecks.check_cache',
 }
 
 # Notifications

api/src/reportcreator_api/conf/settings_test.py

@@ -7,7 +7,11 @@
     'uploaded_files': {'BACKEND': 'django.core.files.storage.InMemoryStorage'},
     'archived_files': {'BACKEND': 'django.core.files.storage.InMemoryStorage'},
 }
-
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+    },
+}
 
 REST_FRAMEWORK['DEFAULT_THROTTLE_CLASSES'] = []
 REST_FRAMEWORK['TEST_REQUEST_DEFAULT_FORMAT'] = 'json'

api/src/reportcreator_api/conf/urls.py

@@ -9,17 +9,21 @@
 from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
 
 from reportcreator_api.api_utils.views import SpellcheckWordView, UtilsViewSet, SpellcheckView, HealthcheckView
-from reportcreator_api.pentests.views import ArchivedProjectKeyPartViewSet, ArchivedProjectViewSet, FindingTemplateViewSet, \
+from reportcreator_api.pentests.views import ArchivedProjectKeyPartViewSet, ArchivedProjectViewSet, \
+    FindingTemplateViewSet, FindingTemplateTranslationViewSet, UploadedTemplateImageViewSet, \
     PentestFindingViewSet, PentestProjectViewSet, ProjectNotebookPageViewSet, \
     PentestProjectPreviewView, PentestProjectGenerateView, \
     ProjectTypeViewSet, ProjectTypePreviewView, \
     ReportSectionViewSet, UploadedAssetViewSet, UploadedImageViewSet, UploadedProjectFileViewSet, UploadedUserNotebookImageViewSet, \
-        UploadedUserNotebookFileViewSet, UserNotebookPageViewSet, UserPublicKeyViewSet
+    UploadedUserNotebookFileViewSet, UserNotebookPageViewSet, UserPublicKeyViewSet
 from reportcreator_api.users.views import APITokenViewSet, PentestUserViewSet, MFAMethodViewSet, AuthViewSet, AuthIdentityViewSet
 from reportcreator_api.notifications.views import NotificationViewSet
 
 
 router = DefaultRouter()
+# Make trailing slash in URL optional to support loading images and assets by fielname
+router.trailing_slash = '/?'
+
 router.register('pentestusers', PentestUserViewSet, basename='pentestuser')
 router.register('projecttypes', ProjectTypeViewSet, basename='projecttype')
 router.register('pentestprojects', PentestProjectViewSet, basename='pentestproject')
@@ -51,11 +55,9 @@
 archivedproject_router = NestedSimpleRouter(router, 'archivedprojects', lookup='archivedproject')
 archivedproject_router.register('keyparts', ArchivedProjectKeyPartViewSet, basename='archivedprojectkeypart')
 
-# Make trailing slash in URL optional to support loading images and assets by fielname
-router.trailing_slash = '/?'
-project_router.trailing_slash = '/?'
-projecttype_router.trailing_slash = '/?'
-archivedproject_router.trailing_slash = '/?'
+template_router = NestedSimpleRouter(router, 'findingtemplates', lookup='template')
+template_router.register('translations', FindingTemplateTranslationViewSet, basename='findingtemplatetranslation')
+template_router.register('images', UploadedTemplateImageViewSet, basename='uploadedtemplateimage')
 
 
 urlpatterns = [
@@ -68,6 +70,7 @@
         path('', include(project_router.urls)),
         path('', include(projecttype_router.urls)),
         path('', include(archivedproject_router.urls)),
+        path('', include(template_router.urls)),
 
         # Async views
         path('utils/spellcheck/', SpellcheckView.as_view(), name='utils-spellcheck'),

api/src/reportcreator_api/management/commands/cleanupfiles.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from reportcreator_api.pentests.models import ArchivedProject, UploadedAsset, UploadedImage, UploadedProjectFile, \
-    UploadedUserNotebookImage, UploadedUserNotebookFile
+    UploadedUserNotebookImage, UploadedUserNotebookFile, UploadedTemplateImage
 
 
 class Command(BaseCommand):
@@ -25,6 +25,9 @@ def handle(self, *args, **options):
         UploadedUserNotebookImage.objects \
             .filter(pk__in=[o.pk for o in UploadedUserNotebookImage.objects.iterator() if not self.file_exists(o.file)]) \
             .delete()
+        UploadedTemplateImage.objects \
+            .filter(pk__in=[o.pk for o in UploadedTemplateImage.objects.iterator() if not self.file_exists(o.file)]) \
+            .delete()
         UploadedProjectFile.objects \
             .filter(pk__in=[o.pk for o in UploadedProjectFile.objects.iterator() if not self.file_exists(o.file)]) \
             .delete()