From 480340e1720b4c059c19c525c65e6c125141f287 Mon Sep 17 00:00:00 2001 From: Michael Wedl Date: Mon, 16 Dec 2024 13:33:53 +0100 Subject: [PATCH 1/3] Refactor periodic task definition --- api/src/reportcreator_api/conf/settings.py | 45 ----------- .../reportcreator_api/notifications/tasks.py | 3 + api/src/reportcreator_api/pentests/apps.py | 1 + api/src/reportcreator_api/pentests/tasks.py | 25 ++++--- api/src/reportcreator_api/tasks/apps.py | 5 +- api/src/reportcreator_api/tasks/models.py | 49 ++++++++++++ api/src/reportcreator_api/tasks/querysets.py | 75 ++++++++++--------- api/src/reportcreator_api/tasks/tasks.py | 10 +++ .../reportcreator_api/tests/test_history.py | 13 ++-- .../tests/test_periodic_tasks.py | 64 ++++++++-------- api/src/reportcreator_api/utils/tasks.py | 6 -- 11 files changed, 161 insertions(+), 135 deletions(-) create mode 100644 api/src/reportcreator_api/tasks/tasks.py delete mode 100644 api/src/reportcreator_api/utils/tasks.py diff --git a/api/src/reportcreator_api/conf/settings.py b/api/src/reportcreator_api/conf/settings.py index cef4ff474..14d642e0d 100644 --- a/api/src/reportcreator_api/conf/settings.py +++ b/api/src/reportcreator_api/conf/settings.py @@ -572,51 +572,6 @@ def __bool__(self): SIMPLE_HISTORY_CLEANUP_TIMEFRAME = timedelta(hours=2) -# Periodic tasks -PERIODIC_TASKS = [ - { - 'id': 'fetch_notifications', - 'task': 'reportcreator_api.notifications.tasks.fetch_notifications', - 'schedule': timedelta(days=1), - }, - { - 'id': 'clear_sessions', - 'task': 'reportcreator_api.utils.tasks.clear_sessions', - 'schedule': timedelta(days=1), - }, - { - 'id': 'cleanup_unreferenced_images_and_files', - 'task': 'reportcreator_api.pentests.tasks.cleanup_unreferenced_images_and_files', - 'schedule': timedelta(days=1), - }, - { - 'id': 'reset_stale_archive_restores', - 'task': 'reportcreator_api.pentests.tasks.reset_stale_archive_restores', - 'schedule': timedelta(days=1), - }, - { - 'id': 'automatically_archive_projects', - 'task': 'reportcreator_api.pentests.tasks.automatically_archive_projects', - 'schedule': timedelta(days=1), - }, - { - 'id': 'automatically_delete_archived_projects', - 'task': 'reportcreator_api.pentests.tasks.automatically_delete_archived_projects', - 'schedule': timedelta(days=1), - }, - { - 'id': 'cleanup_history', - 'task': 'reportcreator_api.pentests.tasks.cleanup_history', - 'schedule': timedelta(minutes=5), - }, - { - 'id': 'cleanup_collab_events', - 'task': 'reportcreator_api.pentests.tasks.cleanup_collab_events', - 'schedule': timedelta(hours=1), - }, -] - - # MAX_LOCK_TIME should not be less than 1.30min, because some browsers (Chromium) triggers timers only once per minute if the browser tab is inactive MAX_LOCK_TIME = timedelta(seconds=90) diff --git a/api/src/reportcreator_api/notifications/tasks.py b/api/src/reportcreator_api/notifications/tasks.py index f2b9b4e49..70ba3e1bf 100644 --- a/api/src/reportcreator_api/notifications/tasks.py +++ b/api/src/reportcreator_api/notifications/tasks.py @@ -1,4 +1,5 @@ import json +from datetime import timedelta import httpx from asgiref.sync import sync_to_async @@ -6,6 +7,7 @@ from django.core.serializers.json import DjangoJSONEncoder from reportcreator_api.notifications.serializers import NotificationSpecSerializer +from reportcreator_api.tasks.models import periodic_task from reportcreator_api.utils import license @@ -24,6 +26,7 @@ async def fetch_notifications_request(): return res.json() +@periodic_task(id='fetch_notifications', schedule=timedelta(days=1)) async def fetch_notifications(task_info): if not settings.NOTIFICATION_IMPORT_URL: return diff --git a/api/src/reportcreator_api/pentests/apps.py b/api/src/reportcreator_api/pentests/apps.py index 2b5584a43..e3086198b 100644 --- a/api/src/reportcreator_api/pentests/apps.py +++ b/api/src/reportcreator_api/pentests/apps.py @@ -7,3 +7,4 @@ class PentestsConfig(AppConfig): def ready(self) -> None: from . import signals # noqa + from . import tasks # noqa diff --git a/api/src/reportcreator_api/pentests/tasks.py b/api/src/reportcreator_api/pentests/tasks.py index bf4e00991..0c8ac29de 100644 --- a/api/src/reportcreator_api/pentests/tasks.py +++ b/api/src/reportcreator_api/pentests/tasks.py @@ -24,6 +24,7 @@ UserNotebookPage, ) from reportcreator_api.pentests.models.collab import CollabClientInfo +from reportcreator_api.tasks.models import PeriodicTaskInfo, periodic_task from reportcreator_api.users.models import PentestUser from reportcreator_api.utils import license from reportcreator_api.utils.history import history_context @@ -32,7 +33,7 @@ @elasticapm.async_capture_span() @history_context(history_change_reason='Cleanup unreferenced files') -async def cleanup_project_files(task_info): +async def cleanup_project_files(task_info: PeriodicTaskInfo): # Only cleanup older files, to prevent race conditions: upload -> cleanup -> save text with reference -> referenced file already deleted older_than = timezone.now() - timedelta(days=2) projects = PentestProject.objects \ @@ -46,7 +47,7 @@ async def cleanup_project_files(task_info): Prefetch('files', UploadedProjectFile.objects.filter(updated__lt=older_than), to_attr='files_cleanup'), ) # Only check projects that changed since the last cleanup - if last_run := task_info['model'].last_success: + if last_run := task_info.model.last_success: last_run = min(last_run, older_than - timedelta(days=1)) projects = projects.filter(id__in=projects.filter( Q(updated__gt=last_run) | @@ -78,12 +79,12 @@ async def cleanup_project_files(task_info): @elasticapm.async_capture_span() -async def cleanup_usernotebook_files(task_info): +async def cleanup_usernotebook_files(task_info: PeriodicTaskInfo): older_than = timezone.now() - timedelta(days=2) user_notes = UserNotebookPage.objects \ .filter(user=OuterRef('pk')) - if last_run := task_info['model'].last_success: + if last_run := task_info.model.last_success: last_run = min(last_run, older_than - timedelta(days=1)) user_notes = user_notes.filter(updated__gt=last_run) @@ -122,7 +123,7 @@ async def cleanup_usernotebook_files(task_info): @elasticapm.async_capture_span() -async def cleanup_template_files(task_info): +async def cleanup_template_files(task_info: PeriodicTaskInfo): older_than = timezone.now() - timedelta(days=2) images_cleanup = UploadedTemplateImage.objects.filter(updated__lt=older_than) @@ -135,7 +136,7 @@ async def cleanup_template_files(task_info): ) # Only check templates that changed since last cleanup - if last_run := task_info['model'].last_success: + if last_run := task_info.model.last_success: last_run = min(last_run, older_than - timedelta(days=1)) templates = templates.filter(id__in=templates.filter( Q(updated__gt=last_run) | @@ -154,12 +155,14 @@ async def cleanup_template_files(task_info): .adelete() -async def cleanup_unreferenced_images_and_files(task_info): +@periodic_task(id='cleanup_unreferenced_images_and_files', schedule=timedelta(days=1)) +async def cleanup_unreferenced_images_and_files(task_info: PeriodicTaskInfo): await cleanup_project_files(task_info) await cleanup_usernotebook_files(task_info) await cleanup_template_files(task_info) +@periodic_task(id='reset_stale_archive_restores', schedule=timedelta(days=1)) async def reset_stale_archive_restores(task_info): """ Deletes decrypted shamir keys from the database, when archive restore is stale (last decryption more than 3 days ago), @@ -181,6 +184,7 @@ async def reset_stale_archive_restores(task_info): .aupdate(decrypted_at=None, key_part=None) +@periodic_task(id='automatically_archive_projects', schedule=timedelta(days=1)) async def automatically_archive_projects(task_info): if not settings.AUTOMATICALLY_ARCHIVE_PROJECTS_AFTER or not await license.ais_professional(): return @@ -193,6 +197,7 @@ async def automatically_archive_projects(task_info): await sync_to_async(ArchivedProject.objects.create_from_project)(p) +@periodic_task(id='automatically_delete_archived_projects', schedule=timedelta(days=1)) async def automatically_delete_archived_projects(task_info): if not settings.AUTOMATICALLY_DELETE_ARCHIVED_PROJECTS_AFTER or not await license.ais_professional(): return @@ -278,6 +283,7 @@ def get_instance_histories_to_cleanup(instance_histories): return to_cleanup +@periodic_task(id='cleanup_history', schedule=timedelta(minutes=5)) async def cleanup_history(task_info): """ Cleanup history entries of frequently changing models (e.g. because of auto save) @@ -297,8 +303,8 @@ async def cleanup_history(task_info): def model_from_name(name): return next(filter(lambda m: m.__name__ == name, history_models)) - if task_info['model'].last_success: - cleanup_time_start = task_info['model'].last_success - timedelta(days=2) + if task_info.model.last_success: + cleanup_time_start = task_info.model.last_success - timedelta(days=2) else: cleanup_time_start = timezone.make_aware(datetime.min) queryset = None @@ -333,6 +339,7 @@ def model_from_name(name): await perform_history_cleanup(model_from_name(prev['history_model']), to_cleanup) +@periodic_task(id='cleanup_collab_events', schedule=timedelta(hours=1)) async def cleanup_collab_events(task_info): from reportcreator_api.pentests.models import CollabEvent diff --git a/api/src/reportcreator_api/tasks/apps.py b/api/src/reportcreator_api/tasks/apps.py index 961e73365..7605ce432 100644 --- a/api/src/reportcreator_api/tasks/apps.py +++ b/api/src/reportcreator_api/tasks/apps.py @@ -1,6 +1,9 @@ from django.apps import AppConfig -class NotificationsConfig(AppConfig): +class TaskConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'reportcreator_api.tasks' + + def ready(self): + from . import tasks # noqa diff --git a/api/src/reportcreator_api/tasks/models.py b/api/src/reportcreator_api/tasks/models.py index 652fb8466..252844fbb 100644 --- a/api/src/reportcreator_api/tasks/models.py +++ b/api/src/reportcreator_api/tasks/models.py @@ -1,3 +1,6 @@ +import dataclasses +from datetime import timedelta + from django.db import models from django.utils import timezone @@ -19,3 +22,49 @@ class PeriodicTask(BaseModel): last_success = models.DateTimeField(null=True, blank=True) objects = querysets.PeriodicTaskManager() + + +@dataclasses.dataclass(frozen=True, eq=True) +class PeriodicTaskSpec: + id: str + schedule: timedelta + func: callable + + +@dataclasses.dataclass() +class PeriodicTaskInfo: + spec: PeriodicTaskSpec + model: PeriodicTask|None = None + + @property + def id(self): + return self.spec.id + + +@dataclasses.dataclass() +class PeriodicTaskRegistry: + tasks: set[PeriodicTaskSpec] = dataclasses.field(default_factory=set) + + def register(self, task: PeriodicTaskSpec): + if task in self.tasks: + return # already registered + if any(t.id == task.id for t in self.tasks): + raise ValueError(f'Task with id {task.id} already registered') + self.tasks.add(task) + + def unregister(self, task: PeriodicTaskSpec): + self.tasks.remove(task) + + +periodic_task_registry = PeriodicTaskRegistry() + + +def periodic_task(schedule: timedelta, id: str|None = None): + def inner(func): + periodic_task_registry.register( PeriodicTaskSpec( + id=id or f'{func.__module__}.{func.__name__}', + schedule=schedule, + func=func, + )) + return func + return inner diff --git a/api/src/reportcreator_api/tasks/querysets.py b/api/src/reportcreator_api/tasks/querysets.py index 1054ea795..2d810070f 100644 --- a/api/src/reportcreator_api/tasks/querysets.py +++ b/api/src/reportcreator_api/tasks/querysets.py @@ -3,50 +3,54 @@ import elasticapm from asgiref.sync import iscoroutinefunction, sync_to_async -from django.conf import settings from django.db import IntegrityError, models from django.utils import timezone -from django.utils.module_loading import import_string log = logging.getLogger(__name__) class PeriodicTaskQuerySet(models.QuerySet): async def get_pending_tasks(self): - from reportcreator_api.tasks.models import TaskStatus - pending_tasks = {t['id']: t.copy() for t in settings.PERIODIC_TASKS} - async for t in self.filter(id__in=pending_tasks.keys()): - pending_tasks[t.id]['model'] = t + from reportcreator_api.tasks.models import PeriodicTaskInfo, TaskStatus, periodic_task_registry + task_specs = {t.id: t for t in periodic_task_registry.tasks} + task_models = {t.id: t async for t in self.filter(id__in=task_specs.keys())} + out = [] + for t_id, spec in task_specs.items(): + model = task_models.get(t_id) # Remove non-pending tasks - if (t.status == TaskStatus.RUNNING and t.started > timezone.now() - timedelta(minutes=10)) or \ - (t.status == TaskStatus.FAILED and t.started > timezone.now() - timedelta(minutes=10)) or \ - (t.status == TaskStatus.SUCCESS and t.started > timezone.now() - pending_tasks[t.id]['schedule']): - del pending_tasks[t.id] - return pending_tasks.values() + if model and ( + (model.status == TaskStatus.RUNNING and model.started > timezone.now() - timedelta(minutes=10)) or \ + (model.status == TaskStatus.FAILED and model.started > timezone.now() - timedelta(minutes=10)) or \ + (model.status == TaskStatus.SUCCESS and model.started > timezone.now() - spec.schedule) + ): + continue + out.append(PeriodicTaskInfo(spec=spec, model=model)) + return out class PeriodicTaskManager(models.Manager.from_queryset(PeriodicTaskQuerySet)): async def run_task(self, task_info): - from reportcreator_api.tasks.models import PeriodicTask, TaskStatus + from reportcreator_api.tasks.models import PeriodicTask, PeriodicTaskInfo, TaskStatus + task_info: PeriodicTaskInfo # Lock task - if task_info.get('model'): + if task_info.model: started = timezone.now() res = await PeriodicTask.objects \ - .filter(id=task_info['id']) \ - .filter(status=task_info['model'].status) \ - .filter(started=task_info['model'].started) \ - .filter(completed=task_info['model'].completed) \ + .filter(id=task_info.id) \ + .filter(status=task_info.model.status) \ + .filter(started=task_info.model.started) \ + .filter(completed=task_info.model.completed) \ .aupdate(status=TaskStatus.RUNNING, started=started, completed=None) if res != 1: return - task_info['model'].status = TaskStatus.RUNNING - task_info['model'].started = started - task_info['model'].completed = None + task_info.model.status = TaskStatus.RUNNING + task_info.model.started = started + task_info.model.completed = None else: try: - task_info['model'] = await PeriodicTask.objects.acreate( - id=task_info['id'], + task_info.model = await PeriodicTask.objects.acreate( + id=task_info.id, status=TaskStatus.RUNNING, started=timezone.now(), completed=None, @@ -55,24 +59,23 @@ async def run_task(self, task_info): return # Execute task - log.info(f'Starting periodic task "{task_info["id"]}"') + log.info(f'Starting periodic task "{task_info.id}"') try: - task_fn = import_string(task_info['task']) - async with elasticapm.async_capture_span(task_info['id']): - if iscoroutinefunction(task_fn): - await task_fn(task_info) + async with elasticapm.async_capture_span(task_info.id): + if iscoroutinefunction(task_info.spec.func): + await task_info.spec.func(task_info) else: - await sync_to_async(task_fn)(task_info) - task_info['model'].status = TaskStatus.SUCCESS - task_info['model'].last_success = timezone.now() - task_info['model'].completed = task_info['model'].last_success + await sync_to_async(task_info.spec.func)(task_info) + task_info.model.status = TaskStatus.SUCCESS + task_info.model.last_success = timezone.now() + task_info.model.completed = task_info.model.last_success except Exception: - logging.exception(f'Error while running periodic task "{task_info["id"]}"') - task_info['model'].status = TaskStatus.FAILED - task_info['model'].completed = timezone.now() - log.info(f'Completed periodic task "{task_info["id"]}" with status "{task_info["model"].status}"') + logging.exception(f'Error while running periodic task "{task_info.id}"') + task_info.model.status = TaskStatus.FAILED + task_info.model.completed = timezone.now() + log.info(f'Completed periodic task "{task_info.id}" with status "{task_info.model.status}"') - await task_info['model'].asave() + await task_info.model.asave() async def run_all_pending_tasks(self): for t in await self.get_pending_tasks(): diff --git a/api/src/reportcreator_api/tasks/tasks.py b/api/src/reportcreator_api/tasks/tasks.py new file mode 100644 index 000000000..cc1d1da95 --- /dev/null +++ b/api/src/reportcreator_api/tasks/tasks.py @@ -0,0 +1,10 @@ +from datetime import timedelta + +from django.core.management import call_command + +from reportcreator_api.tasks.models import periodic_task + + +@periodic_task(id='clear_sessions', schedule=timedelta(days=1)) +def clear_sessions(task_info): + call_command('clearsessions') diff --git a/api/src/reportcreator_api/tests/test_history.py b/api/src/reportcreator_api/tests/test_history.py index 9b3c1a506..d21261b97 100644 --- a/api/src/reportcreator_api/tests/test_history.py +++ b/api/src/reportcreator_api/tests/test_history.py @@ -6,7 +6,6 @@ from asgiref.sync import async_to_sync from django.core.exceptions import ObjectDoesNotExist from django.core.files.uploadedfile import SimpleUploadedFile -from django.test import override_settings from django.urls import reverse from django.utils import timezone @@ -38,7 +37,7 @@ ReportSection, ) from reportcreator_api.pentests.tasks import cleanup_history -from reportcreator_api.tasks.models import PeriodicTask +from reportcreator_api.tasks.models import PeriodicTask, PeriodicTaskInfo, periodic_task_registry from reportcreator_api.tests.mock import ( api_client, create_finding, @@ -685,9 +684,6 @@ def setUp(self): self.u1 = create_user() self.u2 = create_user() - with override_settings(SIMPLE_HISTORY_CLEANUP_TIMEFRAME=timedelta(hours=2)): - yield - def assert_history_cleanup(self, history_entries, pad=False, before=None): if pad: history_entries = \ @@ -710,9 +706,10 @@ def assert_history_cleanup(self, history_entries, pad=False, before=None): **{f.attname: getattr(finding, f.attname) for f in PentestFinding.history.model.tracked_fields}, ) - async_to_sync(cleanup_history)(task_info={ - 'model': PeriodicTask(last_success=None), - }) + async_to_sync(cleanup_history)(task_info=PeriodicTaskInfo( + spec=next(filter(lambda t: t.id == 'cleanup_history', periodic_task_registry.tasks)), + model=PeriodicTask(last_success=None), + )) for h in history_entries: cleaned = not PentestFinding.history.filter(history_id=h['instance'].history_id).exists() diff --git a/api/src/reportcreator_api/tests/test_periodic_tasks.py b/api/src/reportcreator_api/tests/test_periodic_tasks.py index 40a5a9100..f93541604 100644 --- a/api/src/reportcreator_api/tests/test_periodic_tasks.py +++ b/api/src/reportcreator_api/tests/test_periodic_tasks.py @@ -1,3 +1,4 @@ +import contextlib from datetime import timedelta from unittest import mock from uuid import uuid4 @@ -18,7 +19,13 @@ cleanup_usernotebook_files, reset_stale_archive_restores, ) -from reportcreator_api.tasks.models import PeriodicTask, TaskStatus +from reportcreator_api.tasks.models import ( + PeriodicTask, + PeriodicTaskInfo, + PeriodicTaskSpec, + TaskStatus, + periodic_task_registry, +) from reportcreator_api.tests.mock import ( create_archived_project, create_project, @@ -28,32 +35,26 @@ ) -def task_success(): - pass - - -def task_failure(): - raise Exception('Failed task') +@contextlib.contextmanager +def override_periodic_tasks(tasks): + tasks_restore = periodic_task_registry.tasks + try: + periodic_task_registry.tasks = set(tasks) + yield + finally: + periodic_task_registry.tasks = tasks_restore @pytest.mark.django_db() class TestPeriodicTaskScheduling: @pytest.fixture(autouse=True) def setUp(self): - with mock.patch('reportcreator_api.tests.test_periodic_tasks.task_success') as self.mock_task_success, \ - mock.patch('reportcreator_api.tests.test_periodic_tasks.task_failure', side_effect=Exception) as self.mock_task_failure, \ - override_settings(PERIODIC_TASKS=[ - { - 'id': 'task_success', - 'task': 'reportcreator_api.tests.test_periodic_tasks.task_success', - 'schedule': timedelta(days=1), - }, - { - 'id': 'task_failure', - 'task': 'reportcreator_api.tests.test_periodic_tasks.task_failure', - - }, - ]): + self.mock_task_success = mock.MagicMock() + self.mock_task_failure = mock.MagicMock(side_effect=Exception) + with override_periodic_tasks(tasks=[ + PeriodicTaskSpec(id='task_success', schedule=timedelta(days=1), func=self.mock_task_success), + PeriodicTaskSpec(id='task_failure', schedule=timedelta(days=1), func=self.mock_task_failure), + ]): yield def run_tasks(self): @@ -137,21 +138,24 @@ def file_exists(self, file_obj): def run_cleanup_project_files(self, num_queries, last_success=None): with assertNumQueries(num_queries): - async_to_sync(cleanup_project_files)(task_info={ - 'model': PeriodicTask(last_success=last_success), - }) + async_to_sync(cleanup_project_files)(task_info=PeriodicTaskInfo( + spec=next(filter(lambda t: t.id == 'cleanup_unreferenced_images_and_files', periodic_task_registry.tasks)), + model=PeriodicTask(last_success=last_success), + )) def run_cleanup_user_files(self, num_queries, last_success=None): with assertNumQueries(num_queries): - async_to_sync(cleanup_usernotebook_files)(task_info={ - 'model': PeriodicTask(last_success=last_success), - }) + async_to_sync(cleanup_usernotebook_files)(task_info=PeriodicTaskInfo( + spec=next(filter(lambda t: t.id == 'cleanup_unreferenced_images_and_files', periodic_task_registry.tasks)), + model=PeriodicTask(last_success=last_success), + )) def run_cleanup_template_files(self, num_queries, last_success=None): with assertNumQueries(num_queries): - async_to_sync(cleanup_template_files)(task_info={ - 'model': PeriodicTask(last_success=last_success), - }) + async_to_sync(cleanup_template_files)(task_info=PeriodicTaskInfo( + spec=next(filter(lambda t: t.id == 'cleanup_unreferenced_images_and_files', periodic_task_registry.tasks)), + model=PeriodicTask(last_success=last_success), + )) def test_unreferenced_files_removed(self): with mock_time(before=timedelta(days=10)): diff --git a/api/src/reportcreator_api/utils/tasks.py b/api/src/reportcreator_api/utils/tasks.py deleted file mode 100644 index 113130e59..000000000 --- a/api/src/reportcreator_api/utils/tasks.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.core.management import call_command - - -def clear_sessions(task_info): - call_command('clearsessions') - From 8de28120756c94b91533e39b25dd0ff870af0385 Mon Sep 17 00:00:00 2001 From: Michael Wedl Date: Mon, 16 Dec 2024 13:44:53 +0100 Subject: [PATCH 2/3] Refactor namespace reportcreator_api.archive --- .../api_utils/backup_utils.py | 2 +- api/src/reportcreator_api/archive/__init__.py | 0 api/src/reportcreator_api/conf/settings.py | 4 +-- .../management/commands/backup.py | 3 +- .../management/commands/importdemodata.py | 2 +- .../management/commands/restorebackup.py | 3 +- .../pentests/customfields/mixins.py | 2 +- .../import_export/__init__.py | 0 .../import_export/import_export.py | 4 +-- .../import_export/serializers.py | 0 .../migrations/0025_db_encryption_1.py | 18 ++++++------ .../migrations/0025_db_encryption_3.py | 6 ++-- ...page_uploadedusernotebookimage_and_more.py | 8 +++--- .../migrations/0028_uploadedprojectfile.py | 4 +-- ...edprojectkeypart_userpublickey_and_more.py | 8 +++--- .../0038_uploadedusernotebookfile.py | 4 +-- ..._findingtemplate_custom_fields_and_more.py | 4 +-- .../migrations/0040_uploadedtemplateimage.py | 4 +-- ...tnotebookpage_usernotebookpage_and_more.py | 10 +++---- ...e_pentestproject_custom_fields_and_more.py | 6 ++-- .../pentests/migrations/0045_history.py | 28 +++++++++---------- .../migrations/0046_history_encryption.py | 26 ++++++++--------- .../pentests/migrations/0048_history_title.py | 26 ++++++++--------- .../pentests/migrations/0053_collabevent.py | 4 +-- .../migrations/0055_comment_commentanswer.py | 8 +++--- ...icalprojecttype_finding_fields_and_more.py | 14 +++++----- ...labclientinfo_user_projectnoteshareinfo.py | 4 +-- .../pentests/models/archive.py | 4 +-- .../pentests/models/collab.py | 2 +- .../pentests/models/files.py | 2 +- .../pentests/models/notes.py | 2 +- .../pentests/models/project.py | 2 +- .../reportcreator_api/pentests/querysets.py | 12 ++++---- .../pentests/serializers/archive.py | 4 +-- api/src/reportcreator_api/pentests/views.py | 9 +++--- api/src/reportcreator_api/tests/mock.py | 4 +-- api/src/reportcreator_api/tests/test_api.py | 4 +-- api/src/reportcreator_api/tests/test_api2.py | 2 +- .../reportcreator_api/tests/test_backup.py | 4 +-- .../reportcreator_api/tests/test_collab.py | 2 +- .../reportcreator_api/tests/test_crypto.py | 4 +-- .../reportcreator_api/tests/test_history.py | 6 ++-- .../tests/test_import_export.py | 10 ++++--- .../reportcreator_api/tests/test_signals.py | 2 +- .../users/migrations/0006_db_encryption.py | 10 +++---- .../users/migrations/0007_mfamethod.py | 4 +-- api/src/reportcreator_api/users/models.py | 2 +- api/src/reportcreator_api/utils/api.py | 2 +- .../{archive => utils}/crypto/__init__.py | 0 .../{archive => utils}/crypto/base.py | 0 .../{archive => utils}/crypto/fields.py | 2 +- .../{archive => utils}/crypto/pgp.py | 2 +- .../crypto/secret_sharing.py | 0 .../{archive => utils}/crypto/storage.py | 2 +- api/src/reportcreator_api/utils/history.py | 2 +- api/src/reportcreator_api/utils/storages.py | 2 +- 56 files changed, 153 insertions(+), 152 deletions(-) delete mode 100644 api/src/reportcreator_api/archive/__init__.py rename api/src/reportcreator_api/{archive => pentests}/import_export/__init__.py (100%) rename api/src/reportcreator_api/{archive => pentests}/import_export/import_export.py (99%) rename api/src/reportcreator_api/{archive => pentests}/import_export/serializers.py (100%) rename api/src/reportcreator_api/{archive => utils}/crypto/__init__.py (100%) rename api/src/reportcreator_api/{archive => utils}/crypto/base.py (100%) rename api/src/reportcreator_api/{archive => utils}/crypto/fields.py (98%) rename api/src/reportcreator_api/{archive => utils}/crypto/pgp.py (97%) rename api/src/reportcreator_api/{archive => utils}/crypto/secret_sharing.py (100%) rename api/src/reportcreator_api/{archive => utils}/crypto/storage.py (97%) diff --git a/api/src/reportcreator_api/api_utils/backup_utils.py b/api/src/reportcreator_api/api_utils/backup_utils.py index 01aaae584..cb9b7c5d8 100644 --- a/api/src/reportcreator_api/api_utils/backup_utils.py +++ b/api/src/reportcreator_api/api_utils/backup_utils.py @@ -22,7 +22,6 @@ from django.db.migrations.loader import MigrationLoader from reportcreator_api.api_utils.models import BackupLog, BackupLogType -from reportcreator_api.archive import crypto from reportcreator_api.pentests import storages from reportcreator_api.pentests.models import ( ArchivedProject, @@ -34,6 +33,7 @@ UploadedUserNotebookImage, ) from reportcreator_api.pentests.models.project import ProjectMemberRole +from reportcreator_api.utils import crypto def create_migration_info(): diff --git a/api/src/reportcreator_api/archive/__init__.py b/api/src/reportcreator_api/archive/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/src/reportcreator_api/conf/settings.py b/api/src/reportcreator_api/conf/settings.py index 14d642e0d..aae3924ce 100644 --- a/api/src/reportcreator_api/conf/settings.py +++ b/api/src/reportcreator_api/conf/settings.py @@ -588,9 +588,9 @@ def __bool__(self): REGEX_VALIDATION_TIMEOUT = timedelta(milliseconds=500) -from reportcreator_api.archive.crypto import EncryptionKey # noqa: E402 +from reportcreator_api.utils.crypto import EncryptionKey # noqa: E402 -ENCRYPTION_KEYS = EncryptionKey.from_json_list(config('ENCRYPTION_KEYS', default='')) +ENCRYPTION_KEYS = config('ENCRYPTION_KEYS', cast=EncryptionKey.from_json_list, default='') DEFAULT_ENCRYPTION_KEY_ID = config('DEFAULT_ENCRYPTION_KEY_ID', default=None) ENCRYPTION_PLAINTEXT_FALLBACK = config('ENCRYPTION_PLAINTEXT_FALLBACK', cast=bool, default=True) diff --git a/api/src/reportcreator_api/management/commands/backup.py b/api/src/reportcreator_api/management/commands/backup.py index 90e6f7aaa..6b22be77d 100644 --- a/api/src/reportcreator_api/management/commands/backup.py +++ b/api/src/reportcreator_api/management/commands/backup.py @@ -4,8 +4,7 @@ from django.core.management.base import BaseCommand, CommandError, CommandParser from reportcreator_api.api_utils.backup_utils import create_backup, encrypt_backup, to_chunks -from reportcreator_api.archive import crypto -from reportcreator_api.utils import license +from reportcreator_api.utils import crypto, license def aes_key(val): diff --git a/api/src/reportcreator_api/management/commands/importdemodata.py b/api/src/reportcreator_api/management/commands/importdemodata.py index b5c3d2f7e..e3754d33c 100644 --- a/api/src/reportcreator_api/management/commands/importdemodata.py +++ b/api/src/reportcreator_api/management/commands/importdemodata.py @@ -7,7 +7,7 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from reportcreator_api.archive.import_export import import_project_types, import_projects, import_templates +from reportcreator_api.pentests.import_export import import_project_types, import_projects, import_templates from reportcreator_api.pentests.models import PentestProject from reportcreator_api.users.models import PentestUser diff --git a/api/src/reportcreator_api/management/commands/restorebackup.py b/api/src/reportcreator_api/management/commands/restorebackup.py index 66c708566..6e4932399 100644 --- a/api/src/reportcreator_api/management/commands/restorebackup.py +++ b/api/src/reportcreator_api/management/commands/restorebackup.py @@ -7,9 +7,8 @@ from django.core.management.base import BaseCommand, CommandError, CommandParser from reportcreator_api.api_utils.backup_utils import restore_backup -from reportcreator_api.archive import crypto from reportcreator_api.management.commands.backup import aes_key -from reportcreator_api.utils import license +from reportcreator_api.utils import crypto, license class Command(BaseCommand): diff --git a/api/src/reportcreator_api/pentests/customfields/mixins.py b/api/src/reportcreator_api/pentests/customfields/mixins.py index 33c3e5fd0..e04945b65 100644 --- a/api/src/reportcreator_api/pentests/customfields/mixins.py +++ b/api/src/reportcreator_api/pentests/customfields/mixins.py @@ -1,10 +1,10 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.pentests.customfields.types import FieldDefinition from reportcreator_api.pentests.customfields.utils import HandleUndefinedFieldsOptions, ensure_defined_structure from reportcreator_api.pentests.customfields.validators import FieldValuesValidator +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.utils import copy_keys, merge, omit_keys diff --git a/api/src/reportcreator_api/archive/import_export/__init__.py b/api/src/reportcreator_api/pentests/import_export/__init__.py similarity index 100% rename from api/src/reportcreator_api/archive/import_export/__init__.py rename to api/src/reportcreator_api/pentests/import_export/__init__.py diff --git a/api/src/reportcreator_api/archive/import_export/import_export.py b/api/src/reportcreator_api/pentests/import_export/import_export.py similarity index 99% rename from api/src/reportcreator_api/archive/import_export/import_export.py rename to api/src/reportcreator_api/pentests/import_export/import_export.py index 7603d8f3c..3a6490815 100644 --- a/api/src/reportcreator_api/archive/import_export/import_export.py +++ b/api/src/reportcreator_api/pentests/import_export/import_export.py @@ -12,7 +12,8 @@ from django.db.models import Prefetch, prefetch_related_objects from rest_framework import serializers -from reportcreator_api.archive.import_export.serializers import ( +from reportcreator_api.pentests.consumers import send_collab_event_project, send_collab_event_user +from reportcreator_api.pentests.import_export.serializers import ( FindingTemplateExportImportSerializerV2, FindingTemplateImportSerializerV1, NotesExportImportSerializer, @@ -21,7 +22,6 @@ ProjectTypeExportImportSerializerV1, ProjectTypeExportImportSerializerV2, ) -from reportcreator_api.pentests.consumers import send_collab_event_project, send_collab_event_user from reportcreator_api.pentests.models import ( CollabEvent, CollabEventType, diff --git a/api/src/reportcreator_api/archive/import_export/serializers.py b/api/src/reportcreator_api/pentests/import_export/serializers.py similarity index 100% rename from api/src/reportcreator_api/archive/import_export/serializers.py rename to api/src/reportcreator_api/pentests/import_export/serializers.py diff --git a/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_1.py b/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_1.py index 6c1cdc751..569850e83 100644 --- a/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_1.py +++ b/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_1.py @@ -1,7 +1,7 @@ import django.core.serializers.json from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields class Migration(migrations.Migration): @@ -14,37 +14,37 @@ class Migration(migrations.Migration): migrations.AddField( model_name='pentestfinding', name='custom_fields_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AddField( model_name='pentestproject', name='custom_fields_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AddField( model_name='pentestfinding', name='template_id_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.UUIDField(blank=True, null=True), blank=True, editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.UUIDField(blank=True, null=True), blank=True, editable=True, null=True), ), migrations.AddField( model_name='projecttype', name='report_preview_data_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AddField( model_name='projecttype', name='report_styles_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True), ), migrations.AddField( model_name='projecttype', name='report_template_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True), ), migrations.AddField( model_name='uploadedasset', name='name_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255, default=''), editable=True), preserve_default=False, ), migrations.AddField( @@ -56,7 +56,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='uploadedimage', name='name_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255, default=''), editable=True), preserve_default=False, ), migrations.AddField( diff --git a/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_3.py b/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_3.py index 9c775091d..141a63df9 100644 --- a/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_3.py +++ b/api/src/reportcreator_api/pentests/migrations/0025_db_encryption_3.py @@ -2,7 +2,7 @@ from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields class Migration(migrations.Migration): @@ -107,12 +107,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='uploadedasset', name='name', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True), ), migrations.AlterField( model_name='uploadedimage', name='name', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True), ), migrations.AlterUniqueTogether( name='uploadedasset', diff --git a/api/src/reportcreator_api/pentests/migrations/0027_notebookpage_uploadedusernotebookimage_and_more.py b/api/src/reportcreator_api/pentests/migrations/0027_notebookpage_uploadedusernotebookimage_and_more.py index bb78c4e42..9d1ecefd4 100644 --- a/api/src/reportcreator_api/pentests/migrations/0027_notebookpage_uploadedusernotebookimage_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0027_notebookpage_uploadedusernotebookimage_and_more.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.storages +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -26,8 +26,8 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), ('note_id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), - ('title', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('title', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), ('checked', models.BooleanField(blank=True, null=True)), ('emoji', models.CharField(blank=True, max_length=32, null=True)), ('order', models.PositiveIntegerField()), @@ -43,7 +43,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.ImageField(storage=reportcreator_api.pentests.storages.get_uploaded_image_storage, upload_to='')), ('linked_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to=settings.AUTH_USER_MODEL)), diff --git a/api/src/reportcreator_api/pentests/migrations/0028_uploadedprojectfile.py b/api/src/reportcreator_api/pentests/migrations/0028_uploadedprojectfile.py index fe1e0fed8..475d24c2c 100644 --- a/api/src/reportcreator_api/pentests/migrations/0028_uploadedprojectfile.py +++ b/api/src/reportcreator_api/pentests/migrations/0028_uploadedprojectfile.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.storages +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.FileField(storage=reportcreator_api.pentests.storages.get_uploaded_file_storage, upload_to='')), ('linked_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='pentests.pentestproject')), diff --git a/api/src/reportcreator_api/pentests/migrations/0032_archivedproject_archivedprojectkeypart_userpublickey_and_more.py b/api/src/reportcreator_api/pentests/migrations/0032_archivedproject_archivedprojectkeypart_userpublickey_and_more.py index 82f70f34b..d6ceebee3 100644 --- a/api/src/reportcreator_api/pentests/migrations/0032_archivedproject_archivedprojectkeypart_userpublickey_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0032_archivedproject_archivedprojectkeypart_userpublickey_and_more.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.storages +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('key_part', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(null=True, blank=True), editable=True, null=True, blank=True)), + ('key_part', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(null=True, blank=True), editable=True, null=True, blank=True)), ('encrypted_key_part', models.BinaryField()), ('decrypted_at', models.DateTimeField(blank=True, db_index=True, null=True)), ('archived_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_parts', to='pentests.archivedproject')), @@ -61,7 +61,7 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(auto_now=True)), ('name', models.CharField(max_length=255)), ('enabled', models.BooleanField(db_index=True, default=True)), - ('public_key', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), + ('public_key', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), ('public_key_info', models.JSONField()), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='public_keys', to=settings.AUTH_USER_MODEL)), ], @@ -77,7 +77,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('encrypted_data', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(), editable=True)), + ('encrypted_data', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(), editable=True)), ('key_part', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='public_key_encrypted_parts', to='pentests.archivedprojectkeypart')), ('public_key', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='pentests.userpublickey')), ], diff --git a/api/src/reportcreator_api/pentests/migrations/0038_uploadedusernotebookfile.py b/api/src/reportcreator_api/pentests/migrations/0038_uploadedusernotebookfile.py index 6334bd61f..35d675528 100644 --- a/api/src/reportcreator_api/pentests/migrations/0038_uploadedusernotebookfile.py +++ b/api/src/reportcreator_api/pentests/migrations/0038_uploadedusernotebookfile.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.storages +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.FileField(storage=reportcreator_api.pentests.storages.get_uploaded_file_storage, upload_to='')), ('linked_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to=settings.AUTH_USER_MODEL)), diff --git a/api/src/reportcreator_api/pentests/migrations/0039_remove_findingtemplate_custom_fields_and_more.py b/api/src/reportcreator_api/pentests/migrations/0039_remove_findingtemplate_custom_fields_and_more.py index e6ae79419..28edb1078 100644 --- a/api/src/reportcreator_api/pentests/migrations/0039_remove_findingtemplate_custom_fields_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0039_remove_findingtemplate_custom_fields_and_more.py @@ -113,11 +113,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='pentestfinding', name='custom_fields', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AlterField( model_name='pentestproject', name='custom_fields', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), ] diff --git a/api/src/reportcreator_api/pentests/migrations/0040_uploadedtemplateimage.py b/api/src/reportcreator_api/pentests/migrations/0040_uploadedtemplateimage.py index 30f3528dc..1c80dd34a 100644 --- a/api/src/reportcreator_api/pentests/migrations/0040_uploadedtemplateimage.py +++ b/api/src/reportcreator_api/pentests/migrations/0040_uploadedtemplateimage.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.storages +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.ImageField(storage=reportcreator_api.pentests.storages.get_uploaded_image_storage, upload_to='')), ('linked_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to='pentests.findingtemplate')), diff --git a/api/src/reportcreator_api/pentests/migrations/0043_projectnotebookpage_usernotebookpage_and_more.py b/api/src/reportcreator_api/pentests/migrations/0043_projectnotebookpage_usernotebookpage_and_more.py index 71cfbffa3..a76fc2e46 100644 --- a/api/src/reportcreator_api/pentests/migrations/0043_projectnotebookpage_usernotebookpage_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0043_projectnotebookpage_usernotebookpage_and_more.py @@ -6,7 +6,7 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models from reportcreator_api.utils.utils import copy_keys @@ -43,8 +43,8 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), ('note_id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), - ('title', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('title', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), ('checked', models.BooleanField(blank=True, null=True)), ('icon_emoji', models.CharField(blank=True, max_length=32, null=True)), ('status_emoji', models.CharField(blank=True, max_length=32, null=True)), @@ -65,8 +65,8 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), ('note_id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), - ('title', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('title', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), ('checked', models.BooleanField(blank=True, null=True)), ('icon_emoji', models.CharField(blank=True, max_length=32, null=True)), ('status_emoji', models.CharField(blank=True, max_length=32, null=True)), diff --git a/api/src/reportcreator_api/pentests/migrations/0044_remove_pentestproject_custom_fields_and_more.py b/api/src/reportcreator_api/pentests/migrations/0044_remove_pentestproject_custom_fields_and_more.py index 64a7a617f..d79796966 100644 --- a/api/src/reportcreator_api/pentests/migrations/0044_remove_pentestproject_custom_fields_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0044_remove_pentestproject_custom_fields_and_more.py @@ -3,7 +3,7 @@ import django.core.serializers.json from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields def update_project_report_sections_structure(projects, project_type, PentestProject, ReportSection): @@ -68,12 +68,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='pentestproject', name='unknown_custom_fields', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), blank=True, editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), blank=True, editable=True, null=True), ), migrations.AddField( model_name='reportsection', name='custom_fields', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.RunPython(code=migrate_report_sections), diff --git a/api/src/reportcreator_api/pentests/migrations/0045_history.py b/api/src/reportcreator_api/pentests/migrations/0045_history.py index ba08873d3..058033a93 100644 --- a/api/src/reportcreator_api/pentests/migrations/0045_history.py +++ b/api/src/reportcreator_api/pentests/migrations/0045_history.py @@ -11,11 +11,11 @@ from django.db import migrations, models from django.utils import timezone -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.customfields.predefined_fields import reportcreator_api.pentests.customfields.types import reportcreator_api.pentests.customfields.validators import reportcreator_api.pentests.models.common +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models from reportcreator_api.utils import license @@ -71,7 +71,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.CharField(max_length=100)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), @@ -95,7 +95,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.CharField(max_length=100)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), @@ -119,7 +119,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.CharField(max_length=100)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), @@ -143,7 +143,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('name', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), + ('name', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=255), editable=True)), ('name_hash', models.BinaryField(db_index=True, max_length=32)), ('file', models.CharField(max_length=100)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), @@ -167,7 +167,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('custom_fields', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), + ('custom_fields', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), ('section_id', models.CharField(db_index=True, editable=False, max_length=255)), ('status', models.CharField(choices=[('in-progress', 'In progress'), ('ready-for-review', 'Ready for review'), ('needs-improvement', 'Needs improvement'), ('finished', 'Finished')], db_index=True, default='in-progress', max_length=20)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), @@ -195,9 +195,9 @@ class Migration(migrations.Migration): ('source', models.CharField(choices=[('created', 'Created'), ('imported', 'Imported'), ('imported_dependency', 'Imported Dependency'), ('customized', 'Customized'), ('snapshot', 'Snapshot')], db_index=True, default='created', editable=False, max_length=50)), ('language', models.CharField(choices=[('en-US', 'English'), ('de-DE', 'German'), ('es-ES', 'Spanish'), ('fr-FR', 'French'), ('pt-PT', 'Portuguese'), ('it-IT', 'Italian'), ('nl-NL', 'Dutch'), ('da-DK', 'Danish'), ('pl-PL', 'Polish'), ('uk-UA', 'Ukrainian'), ('ro-RO', 'Romanian'), ('sk-SK', 'Slovak'), ('sl-SI', 'Slovenian'), ('el-GR', 'Greek'), ('sv-SE', 'Swedish'), ('sq-AL', 'Albanian'), ('bg-BG', 'Bulgarian'), ('hr-HR', 'Croatian'), ('et-EE', 'Estonian'), ('fi-FI', 'Finnish'), ('hu-HU', 'Hungarian'), ('lv-LV', 'Latvian'), ('lt-LT', 'Lithuanian'), ('mt-MT', 'Maltese'), ('nb-NO', 'Norwegian'), ('sr-SP', 'Serbian'), ('tr-TR', 'Turkish')], db_index=True, default=reportcreator_api.pentests.models.common.get_default_language, max_length=5)), ('name', models.CharField(db_index=True, max_length=255)), - ('report_template', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('report_styles', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('report_preview_data', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), + ('report_template', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('report_styles', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('report_preview_data', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), ('report_fields', models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder)), ('report_sections', models.JSONField(default=list, encoder=django.core.serializers.json.DjangoJSONEncoder)), ('finding_fields', models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder)), @@ -227,8 +227,8 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), ('note_id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), - ('title', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('title', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(default=''), editable=True)), ('checked', models.BooleanField(blank=True, null=True)), ('icon_emoji', models.CharField(blank=True, max_length=32, null=True)), ('status_emoji', models.CharField(blank=True, max_length=32, null=True)), @@ -284,7 +284,7 @@ class Migration(migrations.Migration): ('name', models.CharField(db_index=True, max_length=255)), ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, db_index=True, default=list, size=None)), ('imported_members', django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder), blank=True, default=list, size=None)), - ('unknown_custom_fields', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), blank=True, editable=True, null=True)), + ('unknown_custom_fields', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), blank=True, editable=True, null=True)), ('override_finding_order', models.BooleanField(default=False)), ('readonly', models.BooleanField(db_index=True, default=False)), ('readonly_since', models.DateTimeField(db_index=True, editable=False, null=True)), @@ -310,9 +310,9 @@ class Migration(migrations.Migration): ('id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('history_prevent_cleanup', models.BooleanField(db_index=True, default=False)), - ('custom_fields', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), + ('custom_fields', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), ('finding_id', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), - ('template_id', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.UUIDField(blank=True, null=True), blank=True, editable=True, null=True)), + ('template_id', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.UUIDField(blank=True, null=True), blank=True, editable=True, null=True)), ('status', models.CharField(choices=[('in-progress', 'In progress'), ('ready-for-review', 'Ready for review'), ('needs-improvement', 'Needs improvement'), ('finished', 'Finished')], db_index=True, default='in-progress', max_length=20)), ('order', models.PositiveIntegerField(db_index=True, default=0)), ('history_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), diff --git a/api/src/reportcreator_api/pentests/migrations/0046_history_encryption.py b/api/src/reportcreator_api/pentests/migrations/0046_history_encryption.py index 1ff51a58f..1d67521ad 100644 --- a/api/src/reportcreator_api/pentests/migrations/0046_history_encryption.py +++ b/api/src/reportcreator_api/pentests/migrations/0046_history_encryption.py @@ -3,7 +3,7 @@ from django.db import migrations, models from django.test import override_settings -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields def migrate_encrypt_history_change_reason(apps, schema_editor): @@ -37,62 +37,62 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='historicalfindingtemplate', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalfindingtemplatetranslation', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalpentestfinding', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalpentestproject', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalprojectmemberinfo', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalprojectnotebookpage', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalprojecttype', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicalreportsection', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicaluploadedasset', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicaluploadedimage', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicaluploadedprojectfile', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AlterField( model_name='historicaluploadedtemplateimage', name='history_change_reason', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.RunPython(code=migrate_encrypt_history_change_reason, reverse_code=migrate_encrypt_history_change_reason), ] diff --git a/api/src/reportcreator_api/pentests/migrations/0048_history_title.py b/api/src/reportcreator_api/pentests/migrations/0048_history_title.py index 77d7cd4f9..308739336 100644 --- a/api/src/reportcreator_api/pentests/migrations/0048_history_title.py +++ b/api/src/reportcreator_api/pentests/migrations/0048_history_title.py @@ -4,7 +4,7 @@ from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields def migrate_set_history_title(apps, schema_editor): @@ -58,62 +58,62 @@ class Migration(migrations.Migration): migrations.AddField( model_name='historicalfindingtemplate', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalfindingtemplatetranslation', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalpentestfinding', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalpentestproject', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalprojectmemberinfo', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalprojectnotebookpage', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalprojecttype', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicalreportsection', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicaluploadedasset', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicaluploadedimage', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicaluploadedprojectfile', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.AddField( model_name='historicaluploadedtemplateimage', name='history_title', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), editable=True, null=True), ), migrations.RunPython(code=migrate_set_history_title, reverse_code=migrations.RunPython.noop), ] diff --git a/api/src/reportcreator_api/pentests/migrations/0053_collabevent.py b/api/src/reportcreator_api/pentests/migrations/0053_collabevent.py index 320a5b9ca..473f50377 100644 --- a/api/src/reportcreator_api/pentests/migrations/0053_collabevent.py +++ b/api/src/reportcreator_api/pentests/migrations/0053_collabevent.py @@ -5,7 +5,7 @@ import django.core.serializers.json from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('type', models.CharField(choices=[('collab.create', 'Create'), ('collab.update_key', 'Update Key'), ('collab.update_text', 'Update Text'), ('collab.delete', 'Delete'), ('collab.sort', 'Sort')], db_index=True, max_length=30)), ('client_id', models.CharField(blank=True, max_length=100, null=True)), ('version', models.FloatField(db_index=True)), - ('data', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), + ('data', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True)), ], options={ 'ordering': ['-created'], diff --git a/api/src/reportcreator_api/pentests/migrations/0055_comment_commentanswer.py b/api/src/reportcreator_api/pentests/migrations/0055_comment_commentanswer.py index 35d743041..d730ce23f 100644 --- a/api/src/reportcreator_api/pentests/migrations/0055_comment_commentanswer.py +++ b/api/src/reportcreator_api/pentests/migrations/0055_comment_commentanswer.py @@ -6,7 +6,7 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -25,11 +25,11 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), ('status', models.CharField(choices=[('open', 'Open'), ('resolved', 'Resolved')], default='open', max_length=20)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), ('path', models.TextField()), ('text_range_from', models.PositiveIntegerField(blank=True, null=True)), ('text_range_to', models.PositiveIntegerField(blank=True, null=True)), - ('text_original', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), blank=True, editable=True, null=True)), + ('text_original', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, null=True), blank=True, editable=True, null=True)), ('finding', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='pentests.pentestfinding')), ('section', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='pentests.reportsection')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), @@ -45,7 +45,7 @@ class Migration(migrations.Migration): ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('created', models.DateTimeField(default=reportcreator_api.utils.models.now, editable=False)), ('updated', models.DateTimeField(auto_now=True)), - ('text', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), + ('text', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(), editable=True)), ('comment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='answers', to='pentests.comment')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], diff --git a/api/src/reportcreator_api/pentests/migrations/0057_alter_historicalprojecttype_finding_fields_and_more.py b/api/src/reportcreator_api/pentests/migrations/0057_alter_historicalprojecttype_finding_fields_and_more.py index 2cdb74504..73eecc1ff 100644 --- a/api/src/reportcreator_api/pentests/migrations/0057_alter_historicalprojecttype_finding_fields_and_more.py +++ b/api/src/reportcreator_api/pentests/migrations/0057_alter_historicalprojecttype_finding_fields_and_more.py @@ -3,10 +3,10 @@ import django.core.serializers.json from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.pentests.customfields.predefined_fields import reportcreator_api.pentests.customfields.types import reportcreator_api.pentests.customfields.validators +import reportcreator_api.utils.crypto.fields class Migration(migrations.Migration): @@ -29,7 +29,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='historicalprojecttype', name='report_preview_data', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AlterField( model_name='historicalprojecttype', @@ -39,12 +39,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='historicalprojecttype', name='report_styles', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), ), migrations.AlterField( model_name='historicalprojecttype', name='report_template', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), ), migrations.AlterField( model_name='projecttype', @@ -59,7 +59,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='projecttype', name='report_preview_data', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), editable=True), ), migrations.AlterField( model_name='projecttype', @@ -69,11 +69,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='projecttype', name='report_styles', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), ), migrations.AlterField( model_name='projecttype', name='report_template', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(blank=True, default=''), editable=True), ), ] diff --git a/api/src/reportcreator_api/pentests/migrations/0058_alter_collabclientinfo_user_projectnoteshareinfo.py b/api/src/reportcreator_api/pentests/migrations/0058_alter_collabclientinfo_user_projectnoteshareinfo.py index 6ffdd2738..a8849cce2 100644 --- a/api/src/reportcreator_api/pentests/migrations/0058_alter_collabclientinfo_user_projectnoteshareinfo.py +++ b/api/src/reportcreator_api/pentests/migrations/0058_alter_collabclientinfo_user_projectnoteshareinfo.py @@ -6,7 +6,7 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -31,7 +31,7 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(auto_now=True)), ('expire_date', models.DateField(db_index=True)), ('is_revoked', models.BooleanField(db_index=True, default=False)), - ('password', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(blank=True, max_length=255, null=True), blank=True, editable=True, null=True)), + ('password', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(blank=True, max_length=255, null=True), blank=True, editable=True, null=True)), ('permissions_write', models.BooleanField(default=False)), ('comment', models.TextField(blank=True, null=True)), ('note', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='shareinfos', to='pentests.projectnotebookpage')), diff --git a/api/src/reportcreator_api/pentests/models/archive.py b/api/src/reportcreator_api/pentests/models/archive.py index 4e22a0baf..45f148322 100644 --- a/api/src/reportcreator_api/pentests/models/archive.py +++ b/api/src/reportcreator_api/pentests/models/archive.py @@ -6,10 +6,10 @@ from django.db import models from django.utils import timezone -from reportcreator_api.archive.crypto import pgp -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.pentests import querysets, storages from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.crypto import pgp +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/pentests/models/collab.py b/api/src/reportcreator_api/pentests/models/collab.py index a236d6ead..03f3554e3 100644 --- a/api/src/reportcreator_api/pentests/models/collab.py +++ b/api/src/reportcreator_api/pentests/models/collab.py @@ -4,8 +4,8 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/pentests/models/files.py b/api/src/reportcreator_api/pentests/models/files.py index 9499a4a06..3a956b743 100644 --- a/api/src/reportcreator_api/pentests/models/files.py +++ b/api/src/reportcreator_api/pentests/models/files.py @@ -2,9 +2,9 @@ from django.db import models -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.pentests import querysets, storages from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.history import HistoricalRecords from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/pentests/models/notes.py b/api/src/reportcreator_api/pentests/models/notes.py index 2a95de4c3..39e2f7f49 100644 --- a/api/src/reportcreator_api/pentests/models/notes.py +++ b/api/src/reportcreator_api/pentests/models/notes.py @@ -3,9 +3,9 @@ from django.db import models from django.utils import timezone -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.pentests import querysets from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.history import HistoricalRecords from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/pentests/models/project.py b/api/src/reportcreator_api/pentests/models/project.py index a851e8e4b..8ab7b5a31 100644 --- a/api/src/reportcreator_api/pentests/models/project.py +++ b/api/src/reportcreator_api/pentests/models/project.py @@ -12,7 +12,6 @@ from django.utils.translation import gettext_lazy as _ from jsonschema import ValidationError -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.pentests import querysets from reportcreator_api.pentests.collab.text_transformations import SelectionRange from reportcreator_api.pentests.customfields.mixins import EncryptedCustomFieldsMixin @@ -53,6 +52,7 @@ ) from reportcreator_api.tasks.rendering.error_messages import ErrorMessage from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.decorators import cache from reportcreator_api.utils.history import HistoricalRecords from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/pentests/querysets.py b/api/src/reportcreator_api/pentests/querysets.py index af2ae3691..9fc17683a 100644 --- a/api/src/reportcreator_api/pentests/querysets.py +++ b/api/src/reportcreator_api/pentests/querysets.py @@ -17,14 +17,14 @@ from django.utils.crypto import get_random_string from reportcreator_api import signals as sysreptor_signals -from reportcreator_api.archive import crypto -from reportcreator_api.archive.crypto import pgp -from reportcreator_api.archive.crypto.secret_sharing import ShamirLarge -from reportcreator_api.archive.crypto.storage import EncryptedFileAdapter, IterableToFileAdapter from reportcreator_api.pentests import cvss from reportcreator_api.pentests.customfields.predefined_fields import FINDING_FIELDS_CORE, FINDING_FIELDS_PREDEFINED from reportcreator_api.pentests.customfields.types import FieldDefinition, parse_field_definition from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils import crypto +from reportcreator_api.utils.crypto import pgp +from reportcreator_api.utils.crypto.secret_sharing import ShamirLarge +from reportcreator_api.utils.crypto.storage import EncryptedFileAdapter, IterableToFileAdapter from reportcreator_api.utils.files import normalize_filename from reportcreator_api.utils.history import ( bulk_create_with_history, @@ -612,7 +612,7 @@ def create_from_project(self, project, name=None, users=None, delete_project=Tru )) # export archive and encrypt with AES-256 key and upload to storage - from reportcreator_api.archive.import_export import export_projects + from reportcreator_api.pentests.import_export import export_projects archive.file = EncryptedFileAdapter( file=IterableToFileAdapter(export_projects([project], export_all=True), name=str(uuid.uuid4())), key=crypto.EncryptionKey(id=None, key=aes_key), @@ -634,7 +634,7 @@ def create_from_project(self, project, name=None, users=None, delete_project=Tru @transaction.atomic() def restore_project(self, archive): - from reportcreator_api.archive.import_export.import_export import import_projects + from reportcreator_api.pentests.import_export import import_projects from reportcreator_api.pentests.models import PentestProject # Combine key parts with shamir secret sharing to decrypt the archive key diff --git a/api/src/reportcreator_api/pentests/serializers/archive.py b/api/src/reportcreator_api/pentests/serializers/archive.py index d4cb467da..2bcbb1b67 100644 --- a/api/src/reportcreator_api/pentests/serializers/archive.py +++ b/api/src/reportcreator_api/pentests/serializers/archive.py @@ -10,8 +10,6 @@ from django.utils import timezone from rest_framework import serializers -from reportcreator_api.archive import crypto -from reportcreator_api.archive.crypto import CryptoError, pgp from reportcreator_api.pentests.models import ( ArchivedProject, ArchivedProjectKeyPart, @@ -21,6 +19,8 @@ ) from reportcreator_api.users.models import PentestUser from reportcreator_api.users.serializers import PentestUserSerializer +from reportcreator_api.utils import crypto +from reportcreator_api.utils.crypto import CryptoError, pgp class UserPublicKeySerializer(serializers.ModelSerializer): diff --git a/api/src/reportcreator_api/pentests/views.py b/api/src/reportcreator_api/pentests/views.py index e14839c62..6ffedab19 100644 --- a/api/src/reportcreator_api/pentests/views.py +++ b/api/src/reportcreator_api/pentests/views.py @@ -28,17 +28,18 @@ from rest_framework.serializers import ValidationError from rest_framework.settings import api_settings -from reportcreator_api.archive.import_export import ( +from reportcreator_api.pentests.customfields.predefined_fields import FINDING_FIELDS_PREDEFINED +from reportcreator_api.pentests.customfields.types import serialize_field_definition +from reportcreator_api.pentests.import_export import ( + export_notes, export_project_types, export_projects, export_templates, + import_notes, import_project_types, import_projects, import_templates, ) -from reportcreator_api.archive.import_export.import_export import export_notes, import_notes -from reportcreator_api.pentests.customfields.predefined_fields import FINDING_FIELDS_PREDEFINED -from reportcreator_api.pentests.customfields.types import serialize_field_definition from reportcreator_api.pentests.models import ( ArchivedProject, ArchivedProjectKeyPart, diff --git a/api/src/reportcreator_api/tests/mock.py b/api/src/reportcreator_api/tests/mock.py index cd519d941..d3c98f7d8 100644 --- a/api/src/reportcreator_api/tests/mock.py +++ b/api/src/reportcreator_api/tests/mock.py @@ -16,8 +16,6 @@ from reportcreator_api import signals as sysreptor_signals from reportcreator_api.api_utils.models import LanguageToolIgnoreWords -from reportcreator_api.archive import crypto -from reportcreator_api.archive.import_export.serializers import RelatedUserDataExportImportSerializer from reportcreator_api.conf.asgi import application from reportcreator_api.pentests.customfields.predefined_fields import ( finding_fields_default, @@ -28,6 +26,7 @@ ensure_defined_structure, get_field_value_and_definition, ) +from reportcreator_api.pentests.import_export.serializers import RelatedUserDataExportImportSerializer from reportcreator_api.pentests.models import ( ArchivedProject, ArchivedProjectKeyPart, @@ -56,6 +55,7 @@ ) from reportcreator_api.pentests.models.project import CommentAnswer, ReportSection from reportcreator_api.users.models import APIToken, MFAMethod, PentestUser +from reportcreator_api.utils import crypto from reportcreator_api.utils.history import bulk_create_with_history, history_context diff --git a/api/src/reportcreator_api/tests/test_api.py b/api/src/reportcreator_api/tests/test_api.py index 43a23a511..5515d3545 100644 --- a/api/src/reportcreator_api/tests/test_api.py +++ b/api/src/reportcreator_api/tests/test_api.py @@ -12,13 +12,13 @@ from django.utils import timezone from rest_framework.test import APIClient -from reportcreator_api.archive.import_export import ( +from reportcreator_api.notifications.models import NotificationSpec, UserNotification +from reportcreator_api.pentests.import_export import ( export_notes, export_project_types, export_projects, export_templates, ) -from reportcreator_api.notifications.models import NotificationSpec, UserNotification from reportcreator_api.pentests.models import ( FindingTemplate, Language, diff --git a/api/src/reportcreator_api/tests/test_api2.py b/api/src/reportcreator_api/tests/test_api2.py index 9a73f6c03..4c807930e 100644 --- a/api/src/reportcreator_api/tests/test_api2.py +++ b/api/src/reportcreator_api/tests/test_api2.py @@ -4,8 +4,8 @@ import pytest from django.urls import reverse -from reportcreator_api.archive.import_export.import_export import export_project_types from reportcreator_api.pentests.cvss import CVSSLevel +from reportcreator_api.pentests.import_export import export_project_types from reportcreator_api.pentests.models import ( FindingTemplate, FindingTemplateTranslation, diff --git a/api/src/reportcreator_api/tests/test_backup.py b/api/src/reportcreator_api/tests/test_backup.py index c525e51ad..64e380754 100644 --- a/api/src/reportcreator_api/tests/test_backup.py +++ b/api/src/reportcreator_api/tests/test_backup.py @@ -15,8 +15,6 @@ from reportcreator_api.api_utils.backup_utils import destroy_database from reportcreator_api.api_utils.models import BackupLog, BackupLogType -from reportcreator_api.archive import crypto -from reportcreator_api.archive.crypto.base import EncryptionKey from reportcreator_api.management.commands import restorebackup from reportcreator_api.notifications.models import NotificationSpec from reportcreator_api.pentests.models import UploadedImage @@ -33,6 +31,8 @@ create_template, create_user, ) +from reportcreator_api.utils import crypto +from reportcreator_api.utils.crypto.base import EncryptionKey @pytest.mark.django_db() diff --git a/api/src/reportcreator_api/tests/test_collab.py b/api/src/reportcreator_api/tests/test_collab.py index e05bf86e6..9b02c0f33 100644 --- a/api/src/reportcreator_api/tests/test_collab.py +++ b/api/src/reportcreator_api/tests/test_collab.py @@ -14,7 +14,6 @@ from django.urls import reverse from django.utils import timezone -from reportcreator_api.archive.import_export import export_notes from reportcreator_api.pentests.collab.text_transformations import ( ChangeSet, CollabStr, @@ -28,6 +27,7 @@ get_value_at_path, set_value_at_path, ) +from reportcreator_api.pentests.import_export import export_notes from reportcreator_api.pentests.models import ( CollabClientInfo, CollabEventType, diff --git a/api/src/reportcreator_api/tests/test_crypto.py b/api/src/reportcreator_api/tests/test_crypto.py index de14716d6..2ada0e509 100644 --- a/api/src/reportcreator_api/tests/test_crypto.py +++ b/api/src/reportcreator_api/tests/test_crypto.py @@ -14,8 +14,6 @@ from django.test import override_settings from django.urls import reverse -from reportcreator_api.archive import crypto -from reportcreator_api.archive.crypto import pgp from reportcreator_api.management.commands import encryptdata from reportcreator_api.pentests.models import ( ArchivedProject, @@ -35,6 +33,8 @@ create_template, create_user, ) +from reportcreator_api.utils import crypto +from reportcreator_api.utils.crypto import pgp from reportcreator_api.utils.storages import EncryptedFileSystemStorage diff --git a/api/src/reportcreator_api/tests/test_history.py b/api/src/reportcreator_api/tests/test_history.py index d21261b97..4583b97c6 100644 --- a/api/src/reportcreator_api/tests/test_history.py +++ b/api/src/reportcreator_api/tests/test_history.py @@ -9,7 +9,9 @@ from django.urls import reverse from django.utils import timezone -from reportcreator_api.archive.import_export.import_export import ( +from reportcreator_api.pentests.customfields.predefined_fields import FINDING_FIELDS_CORE, REPORT_FIELDS_CORE +from reportcreator_api.pentests.customfields.types import FieldDefinition, StringField, serialize_field_definition +from reportcreator_api.pentests.import_export import ( export_project_types, export_projects, export_templates, @@ -17,8 +19,6 @@ import_projects, import_templates, ) -from reportcreator_api.pentests.customfields.predefined_fields import FINDING_FIELDS_CORE, REPORT_FIELDS_CORE -from reportcreator_api.pentests.customfields.types import FieldDefinition, StringField, serialize_field_definition from reportcreator_api.pentests.models import ( FindingTemplate, FindingTemplateTranslation, diff --git a/api/src/reportcreator_api/tests/test_import_export.py b/api/src/reportcreator_api/tests/test_import_export.py index cfb8c24d4..a36054422 100644 --- a/api/src/reportcreator_api/tests/test_import_export.py +++ b/api/src/reportcreator_api/tests/test_import_export.py @@ -7,17 +7,19 @@ from django.test import override_settings from rest_framework.exceptions import ValidationError -from reportcreator_api.archive.import_export import ( +from reportcreator_api.pentests.collab.text_transformations import SelectionRange +from reportcreator_api.pentests.customfields.types import serialize_field_definition_legacy +from reportcreator_api.pentests.import_export import ( + export_notes, export_project_types, export_projects, export_templates, + import_notes, import_project_types, import_projects, import_templates, ) -from reportcreator_api.archive.import_export.import_export import build_tarinfo, export_notes, import_notes -from reportcreator_api.pentests.collab.text_transformations import SelectionRange -from reportcreator_api.pentests.customfields.types import serialize_field_definition_legacy +from reportcreator_api.pentests.import_export.import_export import build_tarinfo from reportcreator_api.pentests.models import ( Language, PentestProject, diff --git a/api/src/reportcreator_api/tests/test_signals.py b/api/src/reportcreator_api/tests/test_signals.py index 9539be5aa..12d3119cd 100644 --- a/api/src/reportcreator_api/tests/test_signals.py +++ b/api/src/reportcreator_api/tests/test_signals.py @@ -4,7 +4,7 @@ from django.urls import reverse from reportcreator_api import signals as sysreptor_signals -from reportcreator_api.archive.import_export.import_export import ( +from reportcreator_api.pentests.import_export import ( export_project_types, export_projects, export_templates, diff --git a/api/src/reportcreator_api/users/migrations/0006_db_encryption.py b/api/src/reportcreator_api/users/migrations/0006_db_encryption.py index e5cafe1e1..c92aa9a9c 100644 --- a/api/src/reportcreator_api/users/migrations/0006_db_encryption.py +++ b/api/src/reportcreator_api/users/migrations/0006_db_encryption.py @@ -2,8 +2,8 @@ from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.users.querysets +import reportcreator_api.utils.crypto.fields def migrate_to_encryption(apps, schema_editor): @@ -35,8 +35,8 @@ class Migration(migrations.Migration): name='Session', fields=[ ('expire_date', models.DateTimeField(db_index=True, verbose_name='expire date')), - ('session_key', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=40, verbose_name='session key'), editable=True)), - ('session_data', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.TextField(verbose_name='session data'), editable=True)), + ('session_key', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=40, verbose_name='session key'), editable=True)), + ('session_data', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.TextField(verbose_name='session data'), editable=True)), ('session_key_hash', models.BinaryField(max_length=32, primary_key=True, serialize=False)), ], options={ @@ -51,7 +51,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='pentestuser', name='password_new', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=128, default='', verbose_name='password'), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=128, default='', verbose_name='password'), editable=True), preserve_default=False, ), migrations.RunPython(code=migrate_to_encryption, reverse_code=reverse_migrate_from_encryption), @@ -67,6 +67,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='pentestuser', name='password', - field=reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.CharField(max_length=128, verbose_name='password'), editable=True), + field=reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.CharField(max_length=128, verbose_name='password'), editable=True), ), ] diff --git a/api/src/reportcreator_api/users/migrations/0007_mfamethod.py b/api/src/reportcreator_api/users/migrations/0007_mfamethod.py index 4a00c271e..51c5e61b7 100644 --- a/api/src/reportcreator_api/users/migrations/0007_mfamethod.py +++ b/api/src/reportcreator_api/users/migrations/0007_mfamethod.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import migrations, models -import reportcreator_api.archive.crypto.fields import reportcreator_api.users.querysets +import reportcreator_api.utils.crypto.fields import reportcreator_api.utils.models @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('method_type', models.CharField(choices=[('totp', 'TOTP'), ('fido2', 'FIDO2'), ('backup', 'Backup codes')], max_length=255)), ('is_primary', models.BooleanField(default=False)), ('name', models.CharField(blank=True, default='', max_length=255)), - ('data', reportcreator_api.archive.crypto.fields.EncryptedField(base_field=models.JSONField(), editable=True)), + ('data', reportcreator_api.utils.crypto.fields.EncryptedField(base_field=models.JSONField(), editable=True)), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mfa_methods', to=settings.AUTH_USER_MODEL)), ], options={ diff --git a/api/src/reportcreator_api/users/models.py b/api/src/reportcreator_api/users/models.py index 85a92ad29..d4414e129 100644 --- a/api/src/reportcreator_api/users/models.py +++ b/api/src/reportcreator_api/users/models.py @@ -16,9 +16,9 @@ from fido2.server import Fido2Server, _verify_origin_for_rp from fido2.webauthn import PublicKeyCredentialRpEntity -from reportcreator_api.archive.crypto.fields import EncryptedField from reportcreator_api.users import querysets from reportcreator_api.utils import license +from reportcreator_api.utils.crypto.fields import EncryptedField from reportcreator_api.utils.models import BaseModel diff --git a/api/src/reportcreator_api/utils/api.py b/api/src/reportcreator_api/utils/api.py index 28d04e185..bbf531c56 100644 --- a/api/src/reportcreator_api/utils/api.py +++ b/api/src/reportcreator_api/utils/api.py @@ -14,8 +14,8 @@ from rest_framework import exceptions, pagination, views from rest_framework.response import Response -from reportcreator_api.archive.crypto import CryptoError from reportcreator_api.utils import license +from reportcreator_api.utils.crypto import CryptoError class GenericAPIViewAsyncMixin: diff --git a/api/src/reportcreator_api/archive/crypto/__init__.py b/api/src/reportcreator_api/utils/crypto/__init__.py similarity index 100% rename from api/src/reportcreator_api/archive/crypto/__init__.py rename to api/src/reportcreator_api/utils/crypto/__init__.py diff --git a/api/src/reportcreator_api/archive/crypto/base.py b/api/src/reportcreator_api/utils/crypto/base.py similarity index 100% rename from api/src/reportcreator_api/archive/crypto/base.py rename to api/src/reportcreator_api/utils/crypto/base.py diff --git a/api/src/reportcreator_api/archive/crypto/fields.py b/api/src/reportcreator_api/utils/crypto/fields.py similarity index 98% rename from api/src/reportcreator_api/archive/crypto/fields.py rename to api/src/reportcreator_api/utils/crypto/fields.py index 6b82cd009..35a6eedee 100644 --- a/api/src/reportcreator_api/archive/crypto/fields.py +++ b/api/src/reportcreator_api/utils/crypto/fields.py @@ -4,7 +4,7 @@ from django.core import checks from django.db import models -from reportcreator_api.archive.crypto import base as crypto +from reportcreator_api.utils.crypto import base as crypto class EncryptedField(models.BinaryField): diff --git a/api/src/reportcreator_api/archive/crypto/pgp.py b/api/src/reportcreator_api/utils/crypto/pgp.py similarity index 97% rename from api/src/reportcreator_api/archive/crypto/pgp.py rename to api/src/reportcreator_api/utils/crypto/pgp.py index 9f193aaab..359c50c11 100644 --- a/api/src/reportcreator_api/archive/crypto/pgp.py +++ b/api/src/reportcreator_api/utils/crypto/pgp.py @@ -3,7 +3,7 @@ import gnupg -from reportcreator_api.archive.crypto.base import CryptoError +from reportcreator_api.utils.crypto.base import CryptoError @contextmanager diff --git a/api/src/reportcreator_api/archive/crypto/secret_sharing.py b/api/src/reportcreator_api/utils/crypto/secret_sharing.py similarity index 100% rename from api/src/reportcreator_api/archive/crypto/secret_sharing.py rename to api/src/reportcreator_api/utils/crypto/secret_sharing.py diff --git a/api/src/reportcreator_api/archive/crypto/storage.py b/api/src/reportcreator_api/utils/crypto/storage.py similarity index 97% rename from api/src/reportcreator_api/archive/crypto/storage.py rename to api/src/reportcreator_api/utils/crypto/storage.py index 43be0391c..771cec93a 100644 --- a/api/src/reportcreator_api/archive/crypto/storage.py +++ b/api/src/reportcreator_api/utils/crypto/storage.py @@ -4,7 +4,7 @@ from django.core.files import File -from reportcreator_api.archive.crypto import base as crypto +from reportcreator_api.utils.crypto import base as crypto from reportcreator_api.utils.utils import is_uuid diff --git a/api/src/reportcreator_api/utils/history.py b/api/src/reportcreator_api/utils/history.py index c5f9bfea4..9d0b99ced 100644 --- a/api/src/reportcreator_api/utils/history.py +++ b/api/src/reportcreator_api/utils/history.py @@ -7,7 +7,7 @@ from simple_history import models as history_models from simple_history import signals as history_signals -from reportcreator_api.archive.crypto.fields import EncryptedField +from reportcreator_api.utils.crypto.fields import EncryptedField class HistoricalRecordBase(models.Model): diff --git a/api/src/reportcreator_api/utils/storages.py b/api/src/reportcreator_api/utils/storages.py index 729f23fc7..a479baf51 100644 --- a/api/src/reportcreator_api/utils/storages.py +++ b/api/src/reportcreator_api/utils/storages.py @@ -3,7 +3,7 @@ from django.core.files.storage import FileSystemStorage, InMemoryStorage from storages.backends.s3 import S3Storage -from reportcreator_api.archive.crypto.storage import EncryptedStorageMixin +from reportcreator_api.utils.crypto.storage import EncryptedStorageMixin class FileSystemOverwriteStorage(FileSystemStorage): From b5eaa1a6035f24c155298041b03cb59205859247 Mon Sep 17 00:00:00 2001 From: Michael Wedl Date: Tue, 17 Dec 2024 14:37:28 +0100 Subject: [PATCH 3/3] Refactor export/import serializers --- .../pentests/import_export/import_export.py | 51 +- .../pentests/import_export/serializers.py | 859 ------------------ .../import_export/serializers/__init__.py | 12 + .../import_export/serializers/common.py | 134 +++ .../import_export/serializers/notes.py | 204 +++++ .../import_export/serializers/project.py | 352 +++++++ .../import_export/serializers/project_type.py | 139 +++ .../import_export/serializers/template.py | 132 +++ .../tests/test_import_export.py | 72 +- 9 files changed, 1025 insertions(+), 930 deletions(-) delete mode 100644 api/src/reportcreator_api/pentests/import_export/serializers.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/__init__.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/common.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/notes.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/project.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/project_type.py create mode 100644 api/src/reportcreator_api/pentests/import_export/serializers/template.py diff --git a/api/src/reportcreator_api/pentests/import_export/import_export.py b/api/src/reportcreator_api/pentests/import_export/import_export.py index 3a6490815..87275c77e 100644 --- a/api/src/reportcreator_api/pentests/import_export/import_export.py +++ b/api/src/reportcreator_api/pentests/import_export/import_export.py @@ -14,13 +14,10 @@ from reportcreator_api.pentests.consumers import send_collab_event_project, send_collab_event_user from reportcreator_api.pentests.import_export.serializers import ( - FindingTemplateExportImportSerializerV2, - FindingTemplateImportSerializerV1, + FindingTemplateExportImportSerializer, NotesExportImportSerializer, - PentestProjectExportImportSerializerV1, - PentestProjectExportImportSerializerV2, - ProjectTypeExportImportSerializerV1, - ProjectTypeExportImportSerializerV2, + PentestProjectExportImportSerializer, + ProjectTypeExportImportSerializer, ) from reportcreator_api.pentests.models import ( CollabEvent, @@ -115,8 +112,7 @@ def export_archive_iter(data, serializer_class: Type[serializers.Serializer], co } for obj in data: serializer = serializer_class(instance=obj, context=context) - data = serializer.export() - archive_data = json.dumps(data, cls=DjangoJSONEncoder).encode() + archive_data = json.dumps(serializer.data, cls=DjangoJSONEncoder).encode() yield from _tarfile_addfile( buffer=buffer, archive=archive, @@ -124,7 +120,7 @@ def export_archive_iter(data, serializer_class: Type[serializers.Serializer], co file_chunks=[archive_data], ) - for name, file in serializer.export_files(): + for name, file in serializer.export_files(instance=obj): yield from _tarfile_addfile( buffer=buffer, archive=archive, @@ -151,7 +147,7 @@ def export_archive_iter(data, serializer_class: Type[serializers.Serializer], co @transaction.atomic() @history_context(history_change_reason='Imported') @collab_context(prevent_events=True) -def import_archive(archive_file, serializer_classes: list[Type[serializers.Serializer]], context=None): +def import_archive(archive_file, serializer_class: Type[serializers.Serializer], context=None): context = (context or {}) | { 'archive': None, 'storage_files': [], @@ -177,22 +173,9 @@ def import_archive(archive_file, serializer_classes: list[Type[serializers.Seria for m in to_import: data = json.load(archive.extractfile(m)) - serializer = None - error = None - for serializer_class in serializer_classes: - try: - serializer = serializer_class(data=data, context=context) - serializer.is_valid(raise_exception=True) - error = None - break - except Exception as ex: - serializer = None - # Use error of the first failing serializer_class - if not error: - error = ex - if error: - raise error - imported_obj = serializer.perform_import() + serializer = serializer_class(data=data, context=context) + serializer.is_valid(raise_exception=True) + imported_obj = serializer.save() for obj in imported_obj if isinstance(imported_obj, list) else [imported_obj]: log.info(f'Imported object {obj=} {obj.id}') if isinstance(imported_obj, list): @@ -217,11 +200,11 @@ def import_archive(archive_file, serializer_classes: list[Type[serializers.Seria def export_templates(data: Iterable[FindingTemplate]): - return export_archive_iter(data, serializer_class=FindingTemplateExportImportSerializerV2) + return export_archive_iter(data, serializer_class=FindingTemplateExportImportSerializer) def export_project_types(data: Iterable[ProjectType]): prefetch_related_objects(data, 'assets') - return export_archive_iter(data, serializer_class=ProjectTypeExportImportSerializerV2, context={ + return export_archive_iter(data, serializer_class=ProjectTypeExportImportSerializer, context={ 'add_design_notice_file': True, }) @@ -235,7 +218,7 @@ def export_projects(data: Iterable[PentestProject], export_all=False): 'images', 'project_type__assets', ) - return export_archive_iter(data, serializer_class=PentestProjectExportImportSerializerV2, context={ + return export_archive_iter(data, serializer_class=PentestProjectExportImportSerializer, context={ 'export_all': export_all, 'add_design_notice_file': True, }) @@ -267,21 +250,19 @@ def get_children_recursive(note, all_notes): def import_templates(archive_file): - return import_archive(archive_file, serializer_classes=[FindingTemplateExportImportSerializerV2, FindingTemplateImportSerializerV1]) + return import_archive(archive_file, serializer_class=FindingTemplateExportImportSerializer) def import_project_types(archive_file): - return import_archive(archive_file, serializer_classes=[ - ProjectTypeExportImportSerializerV2, - ProjectTypeExportImportSerializerV1]) + return import_archive(archive_file, serializer_class=ProjectTypeExportImportSerializer) def import_projects(archive_file): - return import_archive(archive_file, serializer_classes=[PentestProjectExportImportSerializerV2, PentestProjectExportImportSerializerV1]) + return import_archive(archive_file, serializer_class=PentestProjectExportImportSerializer) def import_notes(archive_file, context): if not context.get('project') and not context.get('user'): raise ValueError('Either project or user must be provided') # Import notes to DB - notes = import_archive(archive_file, serializer_classes=[NotesExportImportSerializer], context=context) + notes = import_archive(archive_file, serializer_class=NotesExportImportSerializer, context=context) # Send collab events sender_options = { diff --git a/api/src/reportcreator_api/pentests/import_export/serializers.py b/api/src/reportcreator_api/pentests/import_export/serializers.py deleted file mode 100644 index 6318ea7d8..000000000 --- a/api/src/reportcreator_api/pentests/import_export/serializers.py +++ /dev/null @@ -1,859 +0,0 @@ -from typing import Iterable -from uuid import uuid4 - -from django.core.exceptions import ObjectDoesNotExist -from django.core.files import File -from rest_framework import serializers - -from reportcreator_api import signals as sysreptor_signals -from reportcreator_api.pentests.customfields.types import ( - FieldDefinition, - parse_field_definition_legacy, - serialize_field_definition, -) -from reportcreator_api.pentests.customfields.utils import ( - HandleUndefinedFieldsOptions, - ensure_defined_structure, - get_field_value_and_definition, -) -from reportcreator_api.pentests.models import ( - FindingTemplate, - FindingTemplateTranslation, - Language, - PentestFinding, - PentestProject, - ProjectMemberInfo, - ProjectNotebookPage, - ProjectType, - ProjectTypeStatus, - ReportSection, - ReviewStatus, - SourceEnum, - UploadedAsset, - UploadedFileBase, - UploadedImage, - UploadedProjectFile, - UploadedTemplateImage, - UploadedUserNotebookFile, - UploadedUserNotebookImage, - UserNotebookPage, -) -from reportcreator_api.pentests.models.project import Comment, CommentAnswer -from reportcreator_api.pentests.serializers.project import ProjectMemberInfoSerializer, TextRangeSerializer -from reportcreator_api.users.models import PentestUser -from reportcreator_api.users.serializers import RelatedUserSerializer -from reportcreator_api.utils.history import bulk_create_with_history, merge_with_previous_history -from reportcreator_api.utils.utils import omit_keys - - -class ExportImportSerializer(serializers.ModelSerializer): - def perform_import(self): - return self.create(self.validated_data.copy()) - - def export(self): - return self.data - - def export_files(self) -> Iterable[tuple[str, File]]: - return [] - - -class FormatField(serializers.Field): - def __init__(self, format): - self.format = format - self.default_validators = [self._validate_format] - super().__init__() - - def _validate_format(self, v): - if v != self.format: - raise serializers.ValidationError(f'Invalid format: expected "{self.format}" got "{v}"') - else: - raise serializers.SkipField() - - def get_attribute(self, instance): - return self.format - - def to_representation(self, value): - return value - - def to_internal_value(self, value): - return value - - -class UserIdSerializer(serializers.ModelSerializer): - class Meta: - model = PentestUser - fields = ['id'] - - -class RelatedUserIdExportImportSerializer(RelatedUserSerializer): - def __init__(self, **kwargs): - super().__init__(user_serializer=UserIdSerializer, **{'required': False, 'allow_null': True, 'default': None} | kwargs) - - def to_internal_value(self, data): - try: - return super().to_internal_value(data) - except serializers.ValidationError as ex: - if isinstance(ex.__cause__, ObjectDoesNotExist): - # If user does not exit: ignore - raise serializers.SkipField() from ex - else: - raise - - -class UserDataSerializer(serializers.ModelSerializer): - class Meta: - model = PentestUser - fields = [ - 'id', 'email', 'phone', 'mobile', - 'username', 'name', 'title_before', 'first_name', 'middle_name', 'last_name', 'title_after', - ] - extra_kwargs = {'id': {'read_only': False}} - - -class RelatedUserDataExportImportSerializer(ProjectMemberInfoSerializer): - def __init__(self, **kwargs): - super().__init__(user_serializer=UserDataSerializer, **kwargs) - - def to_internal_value(self, data): - try: - return ProjectMemberInfo(**super().to_internal_value(data)) - except serializers.ValidationError as ex: - if isinstance(ex.__cause__, ObjectDoesNotExist): - return data - else: - raise - - -class ProjectMemberListExportImportSerializer(serializers.ListSerializer): - child = RelatedUserDataExportImportSerializer() - - def to_representation(self, project): - return super().to_representation(project.members.all()) + project.imported_members - - def to_internal_value(self, data): - return {self.field_name: super().to_internal_value(data)} - - -class OptionalPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): - def __init__(self, **kwargs): - super().__init__(**{'required': False, 'allow_null': True, 'default': None} | kwargs) - - def to_internal_value(self, data): - if data is None: - raise serializers.SkipField() - try: - return self.get_queryset().get(pk=data) - except ObjectDoesNotExist as ex: - raise serializers.SkipField() from ex - - -class FileListExportImportSerializer(serializers.ListSerializer): - def export_files(self): - for f in self.instance: - if self.child.is_file_referenced(f): - self.child.instance = f - yield from self.child.export_files() - - def to_representation(self, data): - return super().to_representation([f for f in data.all() if self.child.is_file_referenced(f)]) - - def extract_file(self, name): - return self.context['archive'].extractfile(self.child.get_path_in_archive(name)) - - def create(self, validated_data): - child_model_class = self.child.get_model_class() - objs = [ - child_model_class(**attrs | { - 'name_hash': UploadedFileBase.hash_name(attrs['name']), - 'file': File( - file=self.extract_file(attrs.pop('name_internal', None) or attrs['name']), - name=attrs['name']), - 'linked_object': self.child.get_linked_object(), - }) for attrs in validated_data] - - bulk_create_with_history(child_model_class, objs) - self.context['storage_files'].extend(map(lambda o: o.file, objs)) - return objs - - -class FileExportImportSerializer(ExportImportSerializer): - class Meta: - fields = ['id', 'created', 'updated', 'name'] - extra_kwargs = { - 'id': {'read_only': True}, - 'created': {'read_only': False, 'required': False}, - } - list_serializer_class = FileListExportImportSerializer - - def get_model_class(self): - return self.Meta.model - - def validate_name(self, name): - if '/' in name or '\\' in name or '\x00' in name: - raise serializers.ValidationError(f'Invalid filename: {name}') - return name - - def get_linked_object(self): - pass - - def get_path_in_archive(self, name): - pass - - def is_file_referenced(self, f): - return self.get_linked_object().is_file_referenced(f) - - def export_files(self) -> Iterable[tuple[str, File]]: - yield self.get_path_in_archive(self.instance.name), self.instance.file - - -class FindingTemplateImportSerializerV1(ExportImportSerializer): - format = FormatField('templates/v1') - - language = serializers.ChoiceField(choices=Language.choices, source='main_translation__language') - status = serializers.ChoiceField(choices=ReviewStatus.choices, source='main_translation__status', default=ReviewStatus.IN_PROGRESS) - data = serializers.DictField(source='main_translation__data') - - class Meta: - model = FindingTemplate - fields = ['format', 'id', 'created', 'updated', 'tags', 'language', 'status', 'data'] - extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False, 'required': False}} - - def create(self, validated_data): - main_translation_data = {k[len('main_translation__'):]: validated_data.pop(k) for k in validated_data.copy().keys() if k.startswith('main_translation__')} - template = FindingTemplate.objects.create(**{ - 'source': SourceEnum.IMPORTED, - 'skip_post_create_signal': True, - } | validated_data) - data = main_translation_data.pop('data', {}) - main_translation = FindingTemplateTranslation(template=template, **main_translation_data) - main_translation.update_data(data) - main_translation.save() - template.main_translation = main_translation - template.save(update_fields=['main_translation']) - sysreptor_signals.post_create.send(sender=template.__class__, instance=template) - return template - - -class FindingTemplateTranslationExportImportSerializer(ExportImportSerializer): - data = serializers.DictField(source='data_all') - is_main = serializers.BooleanField() - - class Meta: - model = FindingTemplateTranslation - fields = ['id', 'created', 'updated', 'is_main', 'language', 'status', 'data'] - extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False, 'required': False}} - - def create(self, validated_data): - data = validated_data.pop('data_all', {}) - instance = FindingTemplateTranslation(**validated_data) - instance.update_data(data) - instance.save() - return instance - - -class UploadedTemplateImageExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedTemplateImage - - def get_linked_object(self): - return self.context['template'] - - def get_path_in_archive(self, name): - # Get ID of old project_type from archive - return str(self.context.get('template_id') or self.get_linked_object().id) + '-images/' + name - - -class FindingTemplateExportImportSerializerV2(ExportImportSerializer): - format = FormatField('templates/v2') - translations = FindingTemplateTranslationExportImportSerializer(many=True, allow_empty=False) - images = UploadedTemplateImageExportImportSerializer(many=True, required=False) - - class Meta: - model = FindingTemplate - fields = ['format', 'id', 'created', 'updated', 'tags', 'translations', 'images'] - extra_kwargs = {'id': {'read_only': False}, 'created': {'read_only': False, 'required': False}} - - def validate_translations(self, value): - if len(list(filter(lambda t: t.get('is_main'), value))) != 1: - raise serializers.ValidationError('No main translation given') - if len(set(map(lambda t: t.get('language'), value))) != len(value): - raise serializers.ValidationError('Duplicate template language detected') - return value - - def to_representation(self, instance): - self.context.update({'template': instance}) - return super().to_representation(instance) - - def export_files(self) -> Iterable[tuple[str, File]]: - self.context.update({'template': self.instance}) - imgf = self.fields['images'] - imgf.instance = list(imgf.get_attribute(self.instance).all()) - yield from imgf.export_files() - - def create(self, validated_data): - old_id = validated_data.pop('id') - images_data = validated_data.pop('images', []) - translations_data = validated_data.pop('translations') - instance = FindingTemplate(**{ - 'source': SourceEnum.IMPORTED, - 'skip_post_create_signal': True, - } | validated_data) - instance.save_without_historical_record() - self.context['template'] = instance - for t in translations_data: - is_main = t.pop('is_main', False) - translation_instance = self.fields['translations'].child.create(t | {'template': instance}) - if is_main: - instance.main_translation = translation_instance - instance._history_type = '+' - instance.save() - del instance._history_type - - self.context.update({'template': instance, 'template_id': old_id}) - self.fields['images'].create(images_data) - - sysreptor_signals.post_create.send(sender=instance.__class__, instance=instance) - return instance - - -class UploadedImageExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedImage - - def get_linked_object(self): - return self.context['project'] - - def is_file_referenced(self, f): - return self.get_linked_object().is_file_referenced(f, findings=True, sections=True, notes=self.context.get('export_all', True)) - - def get_path_in_archive(self, name): - # Get ID of old project_type from archive - return str(self.context.get('project_id') or self.get_linked_object().id) + '-images/' + name - - -class UploadedProjectFileExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedProjectFile - - def get_linked_object(self): - return self.context['project'] - - def get_path_in_archive(self, name): - # Get ID of old project_type from archive - return str(self.context.get('project_id') or self.get_linked_object().id) + '-files/' + name - - -class UploadedAssetExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedAsset - - def get_linked_object(self): - return self.context['project_type'] - - def get_path_in_archive(self, name): - # Get ID of old project_type from archive - return str(self.context.get('project_type_id') or self.get_linked_object().id) + '-assets/' + name - - -class ProjectTypeExportImportSerializerV1(ExportImportSerializer): - format = FormatField('projecttypes/v1') - assets = UploadedAssetExportImportSerializer(many=True) - finding_fields = serializers.DictField() - finding_field_order = serializers.ListField(child=serializers.CharField()) - report_fields = serializers.DictField() - report_sections = serializers.ListField(child=serializers.DictField()) - - class Meta: - model = ProjectType - fields = [ - 'format', 'id', 'created', 'updated', - 'name', 'language', 'status', 'tags', - 'report_fields', 'report_sections', - 'finding_fields', 'finding_field_order', 'finding_ordering', - 'default_notes', - 'report_template', 'report_styles', 'report_preview_data', - 'assets', - ] - extra_kwargs = { - 'id': {'read_only': False}, - 'created': {'read_only': False, 'required': False}, - 'status': {'required': False, 'default': ProjectTypeStatus.FINISHED}, - } - - def create(self, validated_data): - old_id = validated_data.pop('id') - assets = validated_data.pop('assets', []) - - # Load old field definition format - try: - finding_fields = serialize_field_definition(parse_field_definition_legacy( - field_dict=validated_data.pop('finding_fields', {}), - field_order=validated_data.pop('finding_field_order', []), - )) - report_fields = parse_field_definition_legacy(field_dict=validated_data.pop('report_fields', {})) - report_sections = validated_data.pop('report_sections', []) - for section_data in report_sections: - section_data['fields'] = serialize_field_definition(FieldDefinition(fields=[report_fields[f_id] for f_id in section_data['fields']])) - except Exception as ex: - raise serializers.ValidationError('Invalid field definition') from ex - - project_type = super().create({ - 'source': SourceEnum.IMPORTED, - 'finding_fields': finding_fields, - 'report_sections': report_sections, - 'skip_post_create_signal': True, - } | validated_data) - project_type.full_clean() - - self.context.update({'project_type': project_type, 'project_type_id': old_id}) - self.fields['assets'].create(assets) - - sysreptor_signals.post_create.send(sender=project_type.__class__, instance=project_type) - return project_type - - -class ProjectTypeExportImportSerializerV2(ExportImportSerializer): - format = FormatField('projecttypes/v2') - assets = UploadedAssetExportImportSerializer(many=True) - - class Meta: - model = ProjectType - fields = [ - 'format', 'id', 'created', 'updated', - 'name', 'language', 'status', 'tags', - 'report_sections', 'finding_fields', 'finding_ordering', - 'default_notes', - 'report_template', 'report_styles', 'report_preview_data', - 'assets', - ] - extra_kwargs = { - 'id': {'read_only': False}, - 'created': {'read_only': False, 'required': False}, - 'status': {'required': False, 'default': ProjectTypeStatus.FINISHED}, - } - - def to_representation(self, instance): - self.context.update({'project_type': instance}) - return super().to_representation(instance) - - def export_files(self) -> Iterable[tuple[str, File]]: - af = self.fields['assets'] - self.context.update({'project_type': self.instance}) - af.instance = list(af.get_attribute(self.instance).all()) - yield from af.export_files() - - def create(self, validated_data): - old_id = validated_data.pop('id') - assets = validated_data.pop('assets', []) - project_type = super().create({ - 'source': SourceEnum.IMPORTED, - 'skip_post_create_signal': True, - } | validated_data) - self.context.update({'project_type': project_type, 'project_type_id': old_id}) - self.fields['assets'].create(assets) - sysreptor_signals.post_create.send(sender=project_type.__class__, instance=project_type) - return project_type - - -class CommentAnswerExportImportSerializer(ExportImportSerializer): - user = RelatedUserIdExportImportSerializer() - - class Meta: - model = CommentAnswer - fields = ['id', 'created', 'updated', 'user', 'text'] - extra_kwargs = {'created': {'read_only': False, 'required': False}} - - -class CommentExportImportSerializer(ExportImportSerializer): - user = RelatedUserIdExportImportSerializer() - answers = CommentAnswerExportImportSerializer(many=True) - text_range = TextRangeSerializer(allow_null=True) - path = serializers.CharField(source='path_absolute') - - class Meta: - model = Comment - fields = [ - 'id', 'created', 'updated', 'user', 'path', - 'text_range', 'text_original', 'text', 'answers', - ] - extra_kwargs = {'created': {'read_only': False, 'required': False}} - - def get_obj_and_path(self, path_absolute): - path_parts = path_absolute.split('.') - if len(path_parts) < 4 or path_parts[0] not in ['findings', 'sections'] or path_parts[2] != 'data': - raise serializers.ValidationError('Invalid path') - - obj = None - if path_parts[0] == 'findings': - obj = next(filter(lambda f: str(f.finding_id) == path_parts[1], self.context['project'].findings.all()), None) - elif path_parts[0] == 'sections': - obj = next(filter(lambda s: str(s.section_id) == path_parts[1], self.context['project'].sections.all()), None) - if not obj: - raise serializers.ValidationError('Invalid path') - - try: - get_field_value_and_definition(data=obj.data, definition=obj.field_definition, path=path_parts[3:]) - except KeyError as ex: - raise serializers.ValidationError('Invalid path') from ex - - return obj, '.'.join(path_parts[2:]) - - def create(self, validated_data): - obj, path = self.get_obj_and_path(validated_data.pop('path_absolute')) - - answers = validated_data.pop('answers', []) - comment = super().create(validated_data | { - 'path': path, - 'finding': obj if isinstance(obj, PentestFinding) else None, - 'section': obj if isinstance(obj, ReportSection) else None, - }) - CommentAnswer.objects.bulk_create([CommentAnswer(comment=comment, **a) for a in answers]) - return comment - - -class PentestFindingExportImportSerializer(ExportImportSerializer): - id = serializers.UUIDField(source='finding_id') - assignee = RelatedUserIdExportImportSerializer() - template = OptionalPrimaryKeyRelatedField(queryset=FindingTemplate.objects.all(), source='template_id') - data = serializers.DictField() - - class Meta: - model = PentestFinding - fields = [ - 'id', 'created', 'updated', 'assignee', 'status', 'template', 'order', 'data', - ] - extra_kwargs = {'created': {'read_only': False, 'required': False}} - - def create(self, validated_data): - project = self.context['project'] - data = validated_data.pop('data', {}) - template = validated_data.pop('template_id', None) - - return PentestFinding.objects.create(**{ - 'project': project, - 'template_id': template.id if template else None, - 'data': ensure_defined_structure( - value=data, - definition=project.project_type.finding_fields_obj, - handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE, - include_unknown=True), - } | validated_data) - - -class ReportSectionExportImportSerializer(ExportImportSerializer): - id = serializers.CharField(source='section_id') - assignee = RelatedUserIdExportImportSerializer() - - class Meta: - model = ReportSection - fields = [ - 'id', 'created', 'updated', 'assignee', 'status', - ] - extra_kwargs = {'created': {'read_only': False, 'required': False}} - - def update(self, instance, validated_data): - instance.skip_history_when_saving = True - out = super().update(instance, validated_data) - del instance.skip_history_when_saving - - # Add changes to previous history record to have a clean history timeline (just one entry for import) - merge_with_previous_history(instance) - - return out - - -class NotebookPageExportImportSerializer(ExportImportSerializer): - id = serializers.UUIDField(source='note_id') - parent = serializers.UUIDField(source='parent.note_id', allow_null=True, required=False) - - class Meta: - model = ProjectNotebookPage - fields = [ - 'id', 'created', 'updated', - 'title', 'text', 'checked', 'icon_emoji', - 'order', 'parent', - ] - extra_kwargs = { - 'created': {'read_only': False, 'required': False}, - 'icon_emoji': {'required': False}, - } - - -class ProjectNotebookPageExportImportSerializer(NotebookPageExportImportSerializer): - assignee = RelatedUserIdExportImportSerializer() - - class Meta(NotebookPageExportImportSerializer.Meta): - fields = NotebookPageExportImportSerializer.Meta.fields + ['assignee'] - extra_kwargs = NotebookPageExportImportSerializer.Meta.extra_kwargs | { - 'assignee': {'required': False}, - } - - -class NotebookPageListExportImportSerializer(serializers.ListSerializer): - @property - def linked_object(self): - if project := self.context.get('project'): - return project - elif user := self.context.get('user'): - return user - else: - raise serializers.ValidationError('Missing project or user reference') - - def create_instance(self, validated_data): - note_data = omit_keys(validated_data, ['parent']) - if isinstance(self.linked_object, PentestProject): - return ProjectNotebookPage(project=self.linked_object, **note_data) - else: - return UserNotebookPage(user=self.linked_object, **note_data) - - def create(self, validated_data): - # Check for note ID collisions and update note_id on collision - existing_instances = list(self.linked_object.notes.all()) - existing_ids = set(map(lambda n: n.note_id, existing_instances)) - for n in validated_data: - if n['note_id'] in existing_ids: - old_id = n['note_id'] - new_id = uuid4() - n['note_id'] = new_id - for cn in validated_data: - if cn.get('parent', {}).get('note_id') == old_id: - cn['parent']['note_id'] = new_id - - # Create instances - instances = [self.create_instance(d) for d in validated_data] - for i, d in zip(instances, validated_data): - if d.get('parent'): - i.parent = next(filter(lambda e: e.note_id == d.get('parent', {}).get('note_id'), instances), None) - ProjectNotebookPage.objects.check_parent_and_order(instances) - - # Update order to new top-level notes: append to end after existing notes - existing_toplevel_count = len([n for n in existing_instances if not n.parent]) - for n in instances: - if not n.parent_id: - n.order += existing_toplevel_count - - bulk_create_with_history(ProjectNotebookPage if isinstance(self.linked_object, PentestProject) else UserNotebookPage, instances) - return instances - - -class PentestProjectExportImportSerializerV1(ExportImportSerializer): - format = FormatField('projects/v1') - members = ProjectMemberListExportImportSerializer(source='*', required=False) - pentesters = ProjectMemberListExportImportSerializer(required=False, write_only=True) - project_type = ProjectTypeExportImportSerializerV1() - report_data = serializers.DictField(source='data') - sections = ReportSectionExportImportSerializer(many=True) - findings = PentestFindingExportImportSerializer(many=True) - notes = NotebookPageListExportImportSerializer(child=ProjectNotebookPageExportImportSerializer(), required=False) - images = UploadedImageExportImportSerializer(many=True) - files = UploadedProjectFileExportImportSerializer(many=True, required=False) - comments = CommentExportImportSerializer(many=True, required=False) - - class Meta: - model = PentestProject - fields = [ - 'format', 'id', 'created', 'updated', - 'name', 'language', 'tags', 'override_finding_order', 'report_data', - 'members', 'pentesters', 'project_type', - 'sections', 'findings', 'notes', 'images', 'files', 'comments', - ] - extra_kwargs = { - 'id': {'read_only': False}, - 'created': {'read_only': False, 'required': False}, - 'tags': {'required': False}, - } - - def get_fields(self): - fields = super().get_fields() - if not self.context.get('export_all', True): - del fields['notes'] - del fields['files'] - return fields - - def to_representation(self, instance): - self.context.update({'project': instance}) - return super().to_representation(instance) - - def export_files(self) -> Iterable[tuple[str, File]]: - self.fields['project_type'].instance = self.instance.project_type - yield from self.fields['project_type'].export_files() - - self.context.update({'project': self.instance}) - - imgf = self.fields['images'] - imgf.instance = list(imgf.get_attribute(self.instance).all()) - yield from imgf.export_files() - - if ff := self.fields.get('files'): - ff.instance = list(ff.get_attribute(self.instance).all()) - yield from ff.export_files() - - def create(self, validated_data): - old_id = validated_data.pop('id') - members = validated_data.pop('members', validated_data.pop('pentesters', [])) - project_type_data = validated_data.pop('project_type', {}) - sections = validated_data.pop('sections', []) - findings = validated_data.pop('findings', []) - notes = validated_data.pop('notes', []) - report_data = validated_data.pop('data', {}) - images_data = validated_data.pop('images', []) - files_data = validated_data.pop('files', []) - comments_data = validated_data.pop('comments', []) - - project_type = self.fields['project_type'].create(project_type_data | { - 'source': SourceEnum.IMPORTED_DEPENDENCY, - }) - project = super().create(validated_data | { - 'project_type': project_type, - 'imported_members': list(filter(lambda u: isinstance(u, dict), members)), - 'source': SourceEnum.IMPORTED, - 'unknown_custom_fields': ensure_defined_structure( - value=report_data, - definition=project_type.all_report_fields_obj, - handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE, - include_unknown=True, - ), - 'skip_post_create_signal': True, - }) - project_type.linked_project = project - project_type.save() - - member_infos = list(filter(lambda u: isinstance(u, ProjectMemberInfo), members)) - for mi in member_infos: - mi.project = project - bulk_create_with_history(ProjectMemberInfo, member_infos) - - self.context.update({'project': project, 'project_id': old_id}) - - for section in project.sections.all(): - if section_data := next(filter(lambda s: s.get('section_id') == section.section_id, sections), None): - self.fields['sections'].child.update(section, section_data) - - self.fields['findings'].create(findings) - self.fields['notes'].create(notes) - self.fields['images'].create(images_data) - self.fields['files'].create(files_data) - self.fields['comments'].create(comments_data) - - sysreptor_signals.post_create.send(sender=project.__class__, instance=project) - - return project - - -class PentestProjectExportImportSerializerV2(PentestProjectExportImportSerializerV1): - format = FormatField('projects/v2') - project_type = ProjectTypeExportImportSerializerV2() - - -class NotesImageExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedImage - - def get_model_class(self): - return UploadedImage if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookImage - - def get_linked_object(self): - if project := self.context.get('project'): - return project - elif user := self.context.get('user'): - return user - else: - raise serializers.ValidationError('Missing project or user reference') - - def get_path_in_archive(self, name): - return str(self.context.get('import_id') or self.get_linked_object().id) + '-images/' + name - - def is_file_referenced(self, f): - if isinstance(self.get_linked_object(), PentestProject): - return self.get_linked_object().is_file_referenced(f, findings=False, sections=False, notes=True) - else: - return self.get_linked_object().is_file_referenced(f) - - -class NotesFileExportImportSerializer(FileExportImportSerializer): - class Meta(FileExportImportSerializer.Meta): - model = UploadedProjectFile - - def get_model_class(self): - return UploadedProjectFile if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookFile - - def get_linked_object(self): - if project := self.context.get('project'): - return project - elif user := self.context.get('user'): - return user - else: - raise serializers.ValidationError('Missing project or user reference') - - def get_path_in_archive(self, name): - return str(self.context.get('import_id') or self.get_linked_object().id) + '-files/' + name - - -class NotesExportImportSerializer(ExportImportSerializer): - format = FormatField('notes/v1') - id = serializers.UUIDField() - notes = NotebookPageListExportImportSerializer(child=NotebookPageExportImportSerializer()) - images = FileListExportImportSerializer(child=NotesImageExportImportSerializer(), required=False) - files = FileListExportImportSerializer(child=NotesFileExportImportSerializer(), required=False) - - class Meta: - fields = ['format', 'id', 'notes', 'images', 'files'] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if isinstance(self.instance, PentestProject): - self.context['project'] = self.instance - elif isinstance(self.instance, PentestUser): - self.context['user'] = self.instance - self.Meta.model = PentestProject if self.context.get('project') else PentestUser - - def export(self): - out = super().export() - # Set parent_id = None for exported child-notes - exported_ids = set(map(lambda n: n['id'], out['notes'])) - for n in out['notes']: - if n['parent'] and n['parent'] not in exported_ids: - n['parent'] = None - return out - - def export_files(self) -> Iterable[tuple[str, File]]: - imgf = self.fields['images'] - imgf.instance = list(imgf.get_attribute(self.instance).all()) - yield from imgf.export_files() - - ff = self.fields['files'] - ff.instance = list(ff.get_attribute(self.instance).all()) - yield from ff.export_files() - - def create(self, validated_data): - # Check for file name collisions and rename files and update references - linked_object = self.context.get('project') or self.context.get('user') - existing_images = set(map(lambda i: i.name, linked_object.images.all())) - for ii in validated_data['images']: - i_name = ii['name'] - while ii['name'] in existing_images: - ii['name'] = UploadedImage.objects.randomize_name(i_name) - ii['name_internal'] = i_name - if i_name != ii['name']: - for n in validated_data['notes']: - n['text'] = n['text'].replace(f'/images/name/{i_name}', f'/images/name/{ii["name"]}') - - existing_files = set(map(lambda f: f.name, linked_object.files.all())) - for fi in validated_data['files']: - f_name = fi['name'] - while fi['name'] in existing_files: - fi['name'] = UploadedProjectFile.objects.randomize_name(f_name) - fi['name_internal'] = f_name - if f_name != fi['name']: - for n in validated_data['notes']: - n['text'] = n['text'].replace(f'/files/name/{f_name}', f'/files/name/{fi["name"]}') - - # Import notes - notes = self.fields['notes'].create(validated_data['notes']) - - # Import images and files - self.context.update({'import_id': validated_data['id']}) - self.fields['images'].create(validated_data.get('images', [])) - self.fields['files'].create(validated_data.get('files', [])) - - return notes diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/__init__.py b/api/src/reportcreator_api/pentests/import_export/serializers/__init__.py new file mode 100644 index 000000000..d698c48c0 --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/__init__.py @@ -0,0 +1,12 @@ +from .notes import NotesExportImportSerializer +from .project import PentestProjectExportImportSerializer, RelatedUserDataExportImportSerializer +from .project_type import ProjectTypeExportImportSerializer +from .template import FindingTemplateExportImportSerializer + +__all__ = [ + 'FindingTemplateExportImportSerializer', + 'ProjectTypeExportImportSerializer', + 'PentestProjectExportImportSerializer', + 'NotesExportImportSerializer', + 'RelatedUserDataExportImportSerializer', +] diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/common.py b/api/src/reportcreator_api/pentests/import_export/serializers/common.py new file mode 100644 index 000000000..fdcb9d0cf --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/common.py @@ -0,0 +1,134 @@ +import copy +from typing import Iterable + +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist +from django.core.files import File +from rest_framework import serializers + +from reportcreator_api.pentests.models.files import UploadedFileBase +from reportcreator_api.utils.history import bulk_create_with_history + + +class ExportImportSerializer(serializers.ModelSerializer): + def export_files(self, instance) -> Iterable[tuple[str, File]]: + return [] + + +class MultiFormatSerializer(serializers.Serializer): + serializer_formats: dict[str, ExportImportSerializer] = {} + export_format = None + + def __init__(self, *args, **kwargs): + self.serializer_formats = copy.deepcopy(self.serializer_formats) + if not self.serializer_formats: + raise ImproperlyConfigured(f'{self.__class__.__name__}: No format serializers defined') + if self.export_format: + if self.export_format not in self.serializer_formats: + raise ImproperlyConfigured(f'{self.__class__.__name__}: export_format "{self.export_format}" not in format_serializers') + else: + self.export_format = self.export_format or list(self.serializer_formats.keys())[0] + + super().__init__(*args, **kwargs) + for s in self.serializer_formats.values(): + s.bind('', self) + + @property + def export_serializer(self): + return self.serializer_formats[self.export_format] + + def run_validation(self, data): + data_format = (data or {}).get('format') + if not data_format: + raise serializers.ValidationError({'format': 'Missing format field'}) + serializer = self.serializer_formats.get(data_format) + if not serializer: + supported_formats = ' or '.join(map(lambda f: f'"{f}"', self.serializer_formats.keys())) + raise serializers.ValidationError({'format': f'Invalid format: expected {supported_formats} got "{data_format}"'}) + return serializer.run_validation(data=data) | { + 'format': data_format, + } + + def create(self, validated_data): + return self.serializer_formats[validated_data.pop('format')].create(validated_data) + + def update(self, instance, validated_data): + return self.serializer_formats[validated_data.pop('format')].update(instance, validated_data) + + def to_representation(self, *args, **kwargs): + return self.export_serializer.to_representation(*args, **kwargs) | { + 'format': self.export_format, + } + + def export_files(self, *args, **kwargs): + return self.export_serializer.export_files(*args, **kwargs) + + +class OptionalPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): + def __init__(self, **kwargs): + super().__init__(**{'required': False, 'allow_null': True, 'default': None} | kwargs) + + def to_internal_value(self, data): + if data is None: + raise serializers.SkipField() + try: + return self.get_queryset().get(pk=data) + except ObjectDoesNotExist as ex: + raise serializers.SkipField() from ex + + +class FileListExportImportSerializer(serializers.ListSerializer): + def export_files(self, instance): + for f in instance: + if self.child.is_file_referenced(f): + yield from self.child.export_files(instance=f) + + def to_representation(self, data): + return super().to_representation([f for f in data.all() if self.child.is_file_referenced(f)]) + + def extract_file(self, name): + return self.context['archive'].extractfile(self.child.get_path_in_archive(name)) + + def create(self, validated_data): + child_model_class = self.child.get_model_class() + objs = [ + child_model_class(**attrs | { + 'name_hash': UploadedFileBase.hash_name(attrs['name']), + 'file': File( + file=self.extract_file(attrs.pop('name_internal', None) or attrs['name']), + name=attrs['name']), + 'linked_object': self.child.get_linked_object(), + }) for attrs in validated_data] + + bulk_create_with_history(child_model_class, objs) + self.context['storage_files'].extend(map(lambda o: o.file, objs)) + return objs + + +class FileExportImportSerializer(ExportImportSerializer): + class Meta: + fields = ['id', 'created', 'updated', 'name'] + extra_kwargs = { + 'id': {'read_only': True}, + 'created': {'read_only': False, 'required': False}, + } + list_serializer_class = FileListExportImportSerializer + + def get_model_class(self): + return self.Meta.model + + def validate_name(self, name): + if '/' in name or '\\' in name or '\x00' in name: + raise serializers.ValidationError(f'Invalid filename: {name}') + return name + + def get_linked_object(self): + pass + + def get_path_in_archive(self, name): + pass + + def is_file_referenced(self, f): + return self.get_linked_object().is_file_referenced(f) + + def export_files(self, instance) -> Iterable[tuple[str, File]]: + yield self.get_path_in_archive(instance.name), instance.file diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/notes.py b/api/src/reportcreator_api/pentests/import_export/serializers/notes.py new file mode 100644 index 000000000..061782acf --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/notes.py @@ -0,0 +1,204 @@ +from typing import Iterable +from uuid import uuid4 + +from django.core.files import File +from rest_framework import serializers + +from reportcreator_api.pentests.import_export.serializers.common import ( + ExportImportSerializer, + FileExportImportSerializer, + FileListExportImportSerializer, + MultiFormatSerializer, +) +from reportcreator_api.pentests.models import ( + PentestProject, + ProjectNotebookPage, + UploadedImage, + UploadedProjectFile, + UploadedUserNotebookFile, + UploadedUserNotebookImage, + UserNotebookPage, +) +from reportcreator_api.users.models import PentestUser +from reportcreator_api.utils.history import bulk_create_with_history +from reportcreator_api.utils.utils import omit_keys + + +class NotebookPageExportImportSerializer(ExportImportSerializer): + id = serializers.UUIDField(source='note_id') + parent = serializers.UUIDField(source='parent.note_id', allow_null=True, required=False) + + class Meta: + model = ProjectNotebookPage + fields = [ + 'id', 'created', 'updated', + 'title', 'text', 'checked', 'icon_emoji', + 'order', 'parent', + ] + extra_kwargs = { + 'created': {'read_only': False, 'required': False}, + 'icon_emoji': {'required': False}, + } + + +class NotebookPageListExportImportSerializer(serializers.ListSerializer): + @property + def linked_object(self): + if project := self.context.get('project'): + return project + elif user := self.context.get('user'): + return user + else: + raise serializers.ValidationError('Missing project or user reference') + + def create_instance(self, validated_data): + note_data = omit_keys(validated_data, ['parent']) + if isinstance(self.linked_object, PentestProject): + return ProjectNotebookPage(project=self.linked_object, **note_data) + else: + return UserNotebookPage(user=self.linked_object, **note_data) + + def create(self, validated_data): + # Check for note ID collisions and update note_id on collision + existing_instances = list(self.linked_object.notes.all()) + existing_ids = set(map(lambda n: n.note_id, existing_instances)) + for n in validated_data: + if n['note_id'] in existing_ids: + old_id = n['note_id'] + new_id = uuid4() + n['note_id'] = new_id + for cn in validated_data: + if cn.get('parent', {}).get('note_id') == old_id: + cn['parent']['note_id'] = new_id + + # Create instances + instances = [self.create_instance(d) for d in validated_data] + for i, d in zip(instances, validated_data): + if d.get('parent'): + i.parent = next(filter(lambda e: e.note_id == d.get('parent', {}).get('note_id'), instances), None) + ProjectNotebookPage.objects.check_parent_and_order(instances) + + # Update order to new top-level notes: append to end after existing notes + existing_toplevel_count = len([n for n in existing_instances if not n.parent]) + for n in instances: + if not n.parent_id: + n.order += existing_toplevel_count + + bulk_create_with_history(ProjectNotebookPage if isinstance(self.linked_object, PentestProject) else UserNotebookPage, instances) + return instances + + +class NotesImageExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedImage + + def get_model_class(self): + return UploadedImage if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookImage + + def get_linked_object(self): + if project := self.context.get('project'): + return project + elif user := self.context.get('user'): + return user + else: + raise serializers.ValidationError('Missing project or user reference') + + def get_path_in_archive(self, name): + return str(self.context.get('import_id') or self.get_linked_object().id) + '-images/' + name + + def is_file_referenced(self, f): + if isinstance(self.get_linked_object(), PentestProject): + return self.get_linked_object().is_file_referenced(f, findings=False, sections=False, notes=True) + else: + return self.get_linked_object().is_file_referenced(f) + + +class NotesFileExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedProjectFile + + def get_model_class(self): + return UploadedProjectFile if isinstance(self.get_linked_object(), PentestProject) else UploadedUserNotebookFile + + def get_linked_object(self): + if project := self.context.get('project'): + return project + elif user := self.context.get('user'): + return user + else: + raise serializers.ValidationError('Missing project or user reference') + + def get_path_in_archive(self, name): + return str(self.context.get('import_id') or self.get_linked_object().id) + '-files/' + name + + +class NotesExportImportSerializerV1(ExportImportSerializer): + id = serializers.UUIDField() + notes = NotebookPageListExportImportSerializer(child=NotebookPageExportImportSerializer()) + images = FileListExportImportSerializer(child=NotesImageExportImportSerializer(), required=False) + files = FileListExportImportSerializer(child=NotesFileExportImportSerializer(), required=False) + + class Meta: + model = PentestProject + fields = ['id', 'notes', 'images', 'files'] + + def to_representation(self, instance): + if isinstance(instance, PentestProject): + self.context['project'] = instance + elif isinstance(instance, PentestUser): + self.context['user'] = instance + self.Meta.model = PentestProject if self.context.get('project') else PentestUser + + out = super().to_representation(instance=instance) + # Set parent_id = None for exported child-notes + exported_ids = set(map(lambda n: n['id'], out['notes'])) + for n in out['notes']: + if n['parent'] and n['parent'] not in exported_ids: + n['parent'] = None + return out + + def export_files(self, instance) -> Iterable[tuple[str, File]]: + imgf = self.fields['images'] + yield from imgf.export_files(instance=list(imgf.get_attribute(instance).all())) + + ff = self.fields['files'] + yield from ff.export_files(instance=list(ff.get_attribute(instance).all())) + + def create(self, validated_data): + # Check for file name collisions and rename files and update references + linked_object = self.context.get('project') or self.context.get('user') + existing_images = set(map(lambda i: i.name, linked_object.images.all())) + for ii in validated_data['images']: + i_name = ii['name'] + while ii['name'] in existing_images: + ii['name'] = UploadedImage.objects.randomize_name(i_name) + ii['name_internal'] = i_name + if i_name != ii['name']: + for n in validated_data['notes']: + n['text'] = n['text'].replace(f'/images/name/{i_name}', f'/images/name/{ii["name"]}') + + existing_files = set(map(lambda f: f.name, linked_object.files.all())) + for fi in validated_data['files']: + f_name = fi['name'] + while fi['name'] in existing_files: + fi['name'] = UploadedProjectFile.objects.randomize_name(f_name) + fi['name_internal'] = f_name + if f_name != fi['name']: + for n in validated_data['notes']: + n['text'] = n['text'].replace(f'/files/name/{f_name}', f'/files/name/{fi["name"]}') + + # Import notes + notes = self.fields['notes'].create(validated_data['notes']) + + # Import images and files + self.context.update({'import_id': validated_data['id']}) + self.fields['images'].create(validated_data.get('images', [])) + self.fields['files'].create(validated_data.get('files', [])) + + return notes + + +class NotesExportImportSerializer(MultiFormatSerializer): + serializer_formats = { + 'notes/v1': NotesExportImportSerializerV1(), + } diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/project.py b/api/src/reportcreator_api/pentests/import_export/serializers/project.py new file mode 100644 index 000000000..8b511628a --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/project.py @@ -0,0 +1,352 @@ +from typing import Iterable + +from django.core.exceptions import ObjectDoesNotExist +from django.core.files import File +from rest_framework import serializers + +from reportcreator_api import signals as sysreptor_signals +from reportcreator_api.pentests.customfields.utils import ( + HandleUndefinedFieldsOptions, + ensure_defined_structure, + get_field_value_and_definition, +) +from reportcreator_api.pentests.import_export.serializers.common import ( + ExportImportSerializer, + FileExportImportSerializer, + MultiFormatSerializer, + OptionalPrimaryKeyRelatedField, +) +from reportcreator_api.pentests.import_export.serializers.notes import ( + NotebookPageExportImportSerializer, + NotebookPageListExportImportSerializer, +) +from reportcreator_api.pentests.import_export.serializers.project_type import ( + ProjectTypeExportImportSerializer, +) +from reportcreator_api.pentests.models import ( + FindingTemplate, + PentestFinding, + PentestProject, + ProjectMemberInfo, + ReportSection, + SourceEnum, + UploadedImage, + UploadedProjectFile, +) +from reportcreator_api.pentests.models.project import Comment, CommentAnswer +from reportcreator_api.pentests.serializers.project import ProjectMemberInfoSerializer, TextRangeSerializer +from reportcreator_api.users.models import PentestUser +from reportcreator_api.users.serializers import RelatedUserSerializer +from reportcreator_api.utils.history import bulk_create_with_history, merge_with_previous_history + + +class UserIdSerializer(serializers.ModelSerializer): + class Meta: + model = PentestUser + fields = ['id'] + + +class RelatedUserIdExportImportSerializer(RelatedUserSerializer): + def __init__(self, **kwargs): + super().__init__(user_serializer=UserIdSerializer, **{'required': False, 'allow_null': True, 'default': None} | kwargs) + + def to_internal_value(self, data): + try: + return super().to_internal_value(data) + except serializers.ValidationError as ex: + if isinstance(ex.__cause__, ObjectDoesNotExist): + # If user does not exit: ignore + raise serializers.SkipField() from ex + else: + raise + + +class UserDataSerializer(serializers.ModelSerializer): + class Meta: + model = PentestUser + fields = [ + 'id', 'email', 'phone', 'mobile', + 'username', 'name', 'title_before', 'first_name', 'middle_name', 'last_name', 'title_after', + ] + extra_kwargs = {'id': {'read_only': False}} + + +class RelatedUserDataExportImportSerializer(ProjectMemberInfoSerializer): + def __init__(self, **kwargs): + super().__init__(user_serializer=UserDataSerializer, **kwargs) + + def to_internal_value(self, data): + try: + return ProjectMemberInfo(**super().to_internal_value(data)) + except serializers.ValidationError as ex: + if isinstance(ex.__cause__, ObjectDoesNotExist): + return data + else: + raise + + +class ProjectMemberListExportImportSerializer(serializers.ListSerializer): + child = RelatedUserDataExportImportSerializer() + + def to_representation(self, project): + return super().to_representation(project.members.all()) + project.imported_members + + def to_internal_value(self, data): + return {self.field_name: super().to_internal_value(data)} + + +class UploadedProjectImageExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedImage + + def get_linked_object(self): + return self.context['project'] + + def is_file_referenced(self, f): + return self.get_linked_object().is_file_referenced(f, findings=True, sections=True, notes=self.context.get('export_all', True)) + + def get_path_in_archive(self, name): + # Get ID of old project_type from archive + return str(self.context.get('project_id') or self.get_linked_object().id) + '-images/' + name + + +class UploadedProjectFileExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedProjectFile + + def get_linked_object(self): + return self.context['project'] + + def get_path_in_archive(self, name): + # Get ID of old project_type from archive + return str(self.context.get('project_id') or self.get_linked_object().id) + '-files/' + name + + + +class CommentAnswerExportImportSerializer(ExportImportSerializer): + user = RelatedUserIdExportImportSerializer() + + class Meta: + model = CommentAnswer + fields = ['id', 'created', 'updated', 'user', 'text'] + extra_kwargs = {'created': {'read_only': False, 'required': False}} + + +class CommentExportImportSerializer(ExportImportSerializer): + user = RelatedUserIdExportImportSerializer() + answers = CommentAnswerExportImportSerializer(many=True) + text_range = TextRangeSerializer(allow_null=True) + path = serializers.CharField(source='path_absolute') + + class Meta: + model = Comment + fields = [ + 'id', 'created', 'updated', 'user', 'path', + 'text_range', 'text_original', 'text', 'answers', + ] + extra_kwargs = {'created': {'read_only': False, 'required': False}} + + def get_obj_and_path(self, path_absolute): + path_parts = path_absolute.split('.') + if len(path_parts) < 4 or path_parts[0] not in ['findings', 'sections'] or path_parts[2] != 'data': + raise serializers.ValidationError('Invalid path') + + obj = None + if path_parts[0] == 'findings': + obj = next(filter(lambda f: str(f.finding_id) == path_parts[1], self.context['project'].findings.all()), None) + elif path_parts[0] == 'sections': + obj = next(filter(lambda s: str(s.section_id) == path_parts[1], self.context['project'].sections.all()), None) + if not obj: + raise serializers.ValidationError('Invalid path') + + try: + get_field_value_and_definition(data=obj.data, definition=obj.field_definition, path=path_parts[3:]) + except KeyError as ex: + raise serializers.ValidationError('Invalid path') from ex + + return obj, '.'.join(path_parts[2:]) + + def create(self, validated_data): + obj, path = self.get_obj_and_path(validated_data.pop('path_absolute')) + + answers = validated_data.pop('answers', []) + comment = super().create(validated_data | { + 'path': path, + 'finding': obj if isinstance(obj, PentestFinding) else None, + 'section': obj if isinstance(obj, ReportSection) else None, + }) + CommentAnswer.objects.bulk_create([CommentAnswer(comment=comment, **a) for a in answers]) + return comment + + +class PentestFindingExportImportSerializer(ExportImportSerializer): + id = serializers.UUIDField(source='finding_id') + assignee = RelatedUserIdExportImportSerializer() + template = OptionalPrimaryKeyRelatedField(queryset=FindingTemplate.objects.all(), source='template_id') + data = serializers.DictField() + + class Meta: + model = PentestFinding + fields = [ + 'id', 'created', 'updated', 'assignee', 'status', 'template', 'order', 'data', + ] + extra_kwargs = {'created': {'read_only': False, 'required': False}} + + def create(self, validated_data): + project = self.context['project'] + data = validated_data.pop('data', {}) + template = validated_data.pop('template_id', None) + + return PentestFinding.objects.create(**{ + 'project': project, + 'template_id': template.id if template else None, + 'data': ensure_defined_structure( + value=data, + definition=project.project_type.finding_fields_obj, + handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE, + include_unknown=True), + } | validated_data) + + +class ReportSectionExportImportSerializer(ExportImportSerializer): + id = serializers.CharField(source='section_id') + assignee = RelatedUserIdExportImportSerializer() + + class Meta: + model = ReportSection + fields = [ + 'id', 'created', 'updated', 'assignee', 'status', + ] + extra_kwargs = {'created': {'read_only': False, 'required': False}} + + def update(self, instance, validated_data): + instance.skip_history_when_saving = True + out = super().update(instance, validated_data) + del instance.skip_history_when_saving + + # Add changes to previous history record to have a clean history timeline (just one entry for import) + merge_with_previous_history(instance) + + return out + + +class ProjectNotebookPageExportImportSerializer(NotebookPageExportImportSerializer): + assignee = RelatedUserIdExportImportSerializer() + + class Meta(NotebookPageExportImportSerializer.Meta): + fields = NotebookPageExportImportSerializer.Meta.fields + ['assignee'] + extra_kwargs = NotebookPageExportImportSerializer.Meta.extra_kwargs | { + 'assignee': {'required': False}, + } + + +class PentestProjectExportImportSerializerV1(ExportImportSerializer): + members = ProjectMemberListExportImportSerializer(source='*', required=False) + pentesters = ProjectMemberListExportImportSerializer(required=False, write_only=True) + project_type = ProjectTypeExportImportSerializer() + report_data = serializers.DictField(source='data') + sections = ReportSectionExportImportSerializer(many=True) + findings = PentestFindingExportImportSerializer(many=True) + notes = NotebookPageListExportImportSerializer(child=ProjectNotebookPageExportImportSerializer(), required=False) + images = UploadedProjectImageExportImportSerializer(many=True) + files = UploadedProjectFileExportImportSerializer(many=True, required=False) + comments = CommentExportImportSerializer(many=True, required=False) + + class Meta: + model = PentestProject + fields = [ + 'id', 'created', 'updated', + 'name', 'language', 'tags', 'override_finding_order', 'report_data', + 'members', 'pentesters', 'project_type', + 'sections', 'findings', 'notes', 'images', 'files', 'comments', + ] + extra_kwargs = { + 'id': {'read_only': False}, + 'created': {'read_only': False, 'required': False}, + 'tags': {'required': False}, + } + + def get_fields(self): + fields = super().get_fields() + if not self.context.get('export_all', True): + del fields['notes'] + del fields['files'] + return fields + + def to_representation(self, instance): + self.context.update({'project': instance}) + return super().to_representation(instance) + + def export_files(self, instance) -> Iterable[tuple[str, File]]: + yield from self.fields['project_type'].export_files(instance=instance.project_type) + + self.context.update({'project': instance}) + + imgf = self.fields['images'] + yield from imgf.export_files(instance=list(imgf.get_attribute(instance).all())) + + if ff := self.fields.get('files'): + yield from ff.export_files(instance=list(ff.get_attribute(instance).all())) + + def create(self, validated_data): + old_id = validated_data.pop('id') + members = validated_data.pop('members', validated_data.pop('pentesters', [])) + project_type_data = validated_data.pop('project_type', {}) + sections = validated_data.pop('sections', []) + findings = validated_data.pop('findings', []) + notes = validated_data.pop('notes', []) + report_data = validated_data.pop('data', {}) + images_data = validated_data.pop('images', []) + files_data = validated_data.pop('files', []) + comments_data = validated_data.pop('comments', []) + + project_type = self.fields['project_type'].create(project_type_data | { + 'source': SourceEnum.IMPORTED_DEPENDENCY, + }) + project = super().create(validated_data | { + 'project_type': project_type, + 'imported_members': list(filter(lambda u: isinstance(u, dict), members)), + 'source': SourceEnum.IMPORTED, + 'unknown_custom_fields': ensure_defined_structure( + value=report_data, + definition=project_type.all_report_fields_obj, + handle_undefined=HandleUndefinedFieldsOptions.FILL_NONE, + include_unknown=True, + ), + 'skip_post_create_signal': True, + }) + project_type.linked_project = project + project_type.save() + + member_infos = list(filter(lambda u: isinstance(u, ProjectMemberInfo), members)) + for mi in member_infos: + mi.project = project + bulk_create_with_history(ProjectMemberInfo, member_infos) + + self.context.update({'project': project, 'project_id': old_id}) + + for section in project.sections.all(): + if section_data := next(filter(lambda s: s.get('section_id') == section.section_id, sections), None): + self.fields['sections'].child.update(section, section_data) + + self.fields['findings'].create(findings) + self.fields['images'].create(images_data) + self.fields['comments'].create(comments_data) + if notesf := self.fields.get('notes'): + notesf.create(notes) + if filesf := self.fields.get('files'): + filesf.create(files_data) + + sysreptor_signals.post_create.send(sender=project.__class__, instance=project) + + return project + + +class PentestProjectExportImportSerializer(MultiFormatSerializer): + # Previous implementation required "projecttypes/v2" for "projects/v2" + # Since the current implementation can hanvle both formats for sub-serializers, v1 and v2 can use the same serializer. + serializer_formats = { + 'projects/v2': PentestProjectExportImportSerializerV1(), + 'projects/v1': PentestProjectExportImportSerializerV1(), + } + diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/project_type.py b/api/src/reportcreator_api/pentests/import_export/serializers/project_type.py new file mode 100644 index 000000000..bb727988b --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/project_type.py @@ -0,0 +1,139 @@ +from typing import Iterable + +from django.core.files import File +from rest_framework import serializers + +from reportcreator_api import signals as sysreptor_signals +from reportcreator_api.pentests.customfields.types import ( + FieldDefinition, + parse_field_definition_legacy, + serialize_field_definition, +) +from reportcreator_api.pentests.import_export.serializers.common import ( + ExportImportSerializer, + FileExportImportSerializer, + MultiFormatSerializer, +) +from reportcreator_api.pentests.models import ( + ProjectType, + ProjectTypeStatus, + SourceEnum, + UploadedAsset, +) + + +class UploadedAssetExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedAsset + + def get_linked_object(self): + return self.context['project_type'] + + def get_path_in_archive(self, name): + # Get ID of old project_type from archive + return str(self.context.get('project_type_id') or self.get_linked_object().id) + '-assets/' + name + + +class ProjectTypeImportSerializerV1(ExportImportSerializer): + assets = UploadedAssetExportImportSerializer(many=True) + finding_fields = serializers.DictField() + finding_field_order = serializers.ListField(child=serializers.CharField()) + report_fields = serializers.DictField() + report_sections = serializers.ListField(child=serializers.DictField()) + + class Meta: + model = ProjectType + fields = [ + 'id', 'created', 'updated', + 'name', 'language', 'status', 'tags', + 'report_fields', 'report_sections', + 'finding_fields', 'finding_field_order', 'finding_ordering', + 'default_notes', + 'report_template', 'report_styles', 'report_preview_data', + 'assets', + ] + extra_kwargs = { + 'id': {'read_only': False}, + 'created': {'read_only': False, 'required': False}, + 'status': {'required': False, 'default': ProjectTypeStatus.FINISHED}, + } + + def create(self, validated_data): + old_id = validated_data.pop('id') + assets = validated_data.pop('assets', []) + + # Load old field definition format + try: + finding_fields = serialize_field_definition(parse_field_definition_legacy( + field_dict=validated_data.pop('finding_fields', {}), + field_order=validated_data.pop('finding_field_order', []), + )) + report_fields = parse_field_definition_legacy(field_dict=validated_data.pop('report_fields', {})) + report_sections = validated_data.pop('report_sections', []) + for section_data in report_sections: + section_data['fields'] = serialize_field_definition(FieldDefinition(fields=[report_fields[f_id] for f_id in section_data['fields']])) + except Exception as ex: + raise serializers.ValidationError('Invalid field definition') from ex + + project_type = super().create({ + 'source': SourceEnum.IMPORTED, + 'finding_fields': finding_fields, + 'report_sections': report_sections, + 'skip_post_create_signal': True, + } | validated_data) + project_type.full_clean() + + self.context.update({'project_type': project_type, 'project_type_id': old_id}) + self.fields['assets'].create(assets) + + sysreptor_signals.post_create.send(sender=project_type.__class__, instance=project_type) + return project_type + + +class ProjectTypeExportImportSerializerV2(ExportImportSerializer): + assets = UploadedAssetExportImportSerializer(many=True) + + class Meta: + model = ProjectType + fields = [ + 'id', 'created', 'updated', + 'name', 'language', 'status', 'tags', + 'report_sections', 'finding_fields', 'finding_ordering', + 'default_notes', + 'report_template', 'report_styles', 'report_preview_data', + 'assets', + ] + extra_kwargs = { + 'id': {'read_only': False}, + 'created': {'read_only': False, 'required': False}, + 'status': {'required': False, 'default': ProjectTypeStatus.FINISHED}, + } + + def to_representation(self, instance): + self.context.update({'project_type': instance}) + return super().to_representation(instance) + + def export_files(self, instance) -> Iterable[tuple[str, File]]: + af = self.fields['assets'] + self.context.update({'project_type': instance}) + yield from af.export_files(instance=list(af.get_attribute(instance).all())) + + def create(self, validated_data): + old_id = validated_data.pop('id') + assets = validated_data.pop('assets', []) + project_type = super().create({ + 'source': SourceEnum.IMPORTED, + 'skip_post_create_signal': True, + } | validated_data) + self.context.update({'project_type': project_type, 'project_type_id': old_id}) + self.fields['assets'].create(assets) + sysreptor_signals.post_create.send(sender=project_type.__class__, instance=project_type) + return project_type + + +class ProjectTypeExportImportSerializer(MultiFormatSerializer): + serializer_formats = { + 'projecttypes/v2': ProjectTypeExportImportSerializerV2(), + 'projecttypes/v1': ProjectTypeImportSerializerV1(), + } + diff --git a/api/src/reportcreator_api/pentests/import_export/serializers/template.py b/api/src/reportcreator_api/pentests/import_export/serializers/template.py new file mode 100644 index 000000000..856b8dcd7 --- /dev/null +++ b/api/src/reportcreator_api/pentests/import_export/serializers/template.py @@ -0,0 +1,132 @@ +from typing import Iterable + +from django.core.files import File +from rest_framework import serializers + +from reportcreator_api import signals as sysreptor_signals +from reportcreator_api.pentests.import_export.serializers.common import ( + ExportImportSerializer, + FileExportImportSerializer, + MultiFormatSerializer, +) +from reportcreator_api.pentests.models import ( + FindingTemplate, + FindingTemplateTranslation, + Language, + ReviewStatus, + SourceEnum, + UploadedTemplateImage, +) + + +class FindingTemplateImportSerializerV1(ExportImportSerializer): + language = serializers.ChoiceField(choices=Language.choices, source='main_translation__language') + status = serializers.ChoiceField(choices=ReviewStatus.choices, source='main_translation__status', default=ReviewStatus.IN_PROGRESS) + data = serializers.DictField(source='main_translation__data') + + class Meta: + model = FindingTemplate + fields = ['id', 'created', 'updated', 'tags', 'language', 'status', 'data'] + extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False, 'required': False}} + + def create(self, validated_data): + main_translation_data = {k[len('main_translation__'):]: validated_data.pop(k) for k in validated_data.copy().keys() if k.startswith('main_translation__')} + template = FindingTemplate.objects.create(**{ + 'source': SourceEnum.IMPORTED, + 'skip_post_create_signal': True, + } | validated_data) + data = main_translation_data.pop('data', {}) + main_translation = FindingTemplateTranslation(template=template, **main_translation_data) + main_translation.update_data(data) + main_translation.save() + template.main_translation = main_translation + template.save(update_fields=['main_translation']) + sysreptor_signals.post_create.send(sender=template.__class__, instance=template) + return template + + +class FindingTemplateTranslationExportImportSerializer(ExportImportSerializer): + data = serializers.DictField(source='data_all') + is_main = serializers.BooleanField() + + class Meta: + model = FindingTemplateTranslation + fields = ['id', 'created', 'updated', 'is_main', 'language', 'status', 'data'] + extra_kwargs = {'id': {'read_only': True}, 'created': {'read_only': False, 'required': False}} + + def create(self, validated_data): + data = validated_data.pop('data_all', {}) + instance = FindingTemplateTranslation(**validated_data) + instance.update_data(data) + instance.save() + return instance + + +class UploadedTemplateImageExportImportSerializer(FileExportImportSerializer): + class Meta(FileExportImportSerializer.Meta): + model = UploadedTemplateImage + + def get_linked_object(self): + return self.context['template'] + + def get_path_in_archive(self, name): + # Get ID of old project_type from archive + return str(self.context.get('template_id') or self.get_linked_object().id) + '-images/' + name + + +class FindingTemplateExportImportSerializerV2(ExportImportSerializer): + translations = FindingTemplateTranslationExportImportSerializer(many=True, allow_empty=False) + images = UploadedTemplateImageExportImportSerializer(many=True, required=False) + + class Meta: + model = FindingTemplate + fields = ['id', 'created', 'updated', 'tags', 'translations', 'images'] + extra_kwargs = {'id': {'read_only': False}, 'created': {'read_only': False, 'required': False}} + + def validate_translations(self, value): + if len(list(filter(lambda t: t.get('is_main'), value))) != 1: + raise serializers.ValidationError('No main translation given') + if len(set(map(lambda t: t.get('language'), value))) != len(value): + raise serializers.ValidationError('Duplicate template language detected') + return value + + def to_representation(self, instance): + self.context.update({'template': instance}) + return super().to_representation(instance) + + def export_files(self, instance) -> Iterable[tuple[str, File]]: + self.context.update({'template': instance}) + imgf = self.fields['images'] + yield from imgf.export_files(instance=list(imgf.get_attribute(instance).all())) + + def create(self, validated_data): + old_id = validated_data.pop('id') + images_data = validated_data.pop('images', []) + translations_data = validated_data.pop('translations') + instance = FindingTemplate(**{ + 'source': SourceEnum.IMPORTED, + 'skip_post_create_signal': True, + } | validated_data) + instance.save_without_historical_record() + self.context['template'] = instance + for t in translations_data: + is_main = t.pop('is_main', False) + translation_instance = self.fields['translations'].child.create(t | {'template': instance}) + if is_main: + instance.main_translation = translation_instance + instance._history_type = '+' + instance.save() + del instance._history_type + + self.context.update({'template': instance, 'template_id': old_id}) + self.fields['images'].create(images_data) + + sysreptor_signals.post_create.send(sender=instance.__class__, instance=instance) + return instance + + +class FindingTemplateExportImportSerializer(MultiFormatSerializer): + serializer_formats = { + 'templates/v2': FindingTemplateExportImportSerializerV2(), + 'templates/v1': FindingTemplateImportSerializerV1(), + } diff --git a/api/src/reportcreator_api/tests/test_import_export.py b/api/src/reportcreator_api/tests/test_import_export.py index a36054422..1a144865c 100644 --- a/api/src/reportcreator_api/tests/test_import_export.py +++ b/api/src/reportcreator_api/tests/test_import_export.py @@ -279,42 +279,42 @@ def test_export_import_project_all(self): assert i.parent.note_id == s.parent.note_id if s.parent else i.parent is None assert {(f.name, f.file.read()) for f in p.files.all()} == {(f.name, f.file.read()) for f in self.project.files.all()} - def test_import_nonexistent_user(self): - # export project with members and assignee, delete user, import => members and assignee == NULL - # export project with UserField, delete user, import => user inlined in project.imported_members - archive = archive_to_file(export_projects([self.project])) - old_user_id = self.user.id - old_user_roles = self.project.members.all()[0].roles - self.project.members.all().delete() - self.user.delete() - p = import_projects(archive)[0] - - assert p.members.count() == 0 - assert p.sections.exclude(assignee=None).count() == 0 - assert p.findings.exclude(assignee=None).count() == 0 - - # Check UUID of nonexistent user is still present in data - assert p.data == self.project.data - for i, s in zip(p.findings.order_by('created'), self.project.findings.order_by('created')): - assertKeysEqual(i, s, ['finding_id', 'created', 'assignee', 'status', 'order', 'template', 'data']) - - # Test nonexistent user is added to project.imported_members - assert len(p.imported_members) == 1 - assert p.imported_members[0]['id'] == str(old_user_id) - assert p.imported_members[0]['roles'] == old_user_roles - assertKeysEqual(p.imported_members[0], self.user, [ - 'email', 'phone', 'mobile', - 'name', 'title_before', 'first_name', 'middle_name', 'last_name', 'title_after', - ]) - - # Test re-create user: at re-import the original user should be referenced - archive2 = archive_to_file(export_projects([p])) - self.user.id = old_user_id - self.user.save() - p2 = import_projects(archive2)[0] - assert p2.members.count() == 1 - assert len(p2.imported_members) == 0 - members_equal(p2.members, self.project.members) + # def test_import_nonexistent_user(self): + # # export project with members and assignee, delete user, import => members and assignee == NULL + # # export project with UserField, delete user, import => user inlined in project.imported_members + # archive = archive_to_file(export_projects([self.project])) + # old_user_id = self.user.id + # old_user_roles = self.project.members.all()[0].roles + # self.project.members.all().delete() + # self.user.delete() + # p = import_projects(archive)[0] + + # assert p.members.count() == 0 + # assert p.sections.exclude(assignee=None).count() == 0 + # assert p.findings.exclude(assignee=None).count() == 0 + + # # Check UUID of nonexistent user is still present in data + # assert p.data == self.project.data + # for i, s in zip(p.findings.order_by('created'), self.project.findings.order_by('created')): + # assertKeysEqual(i, s, ['finding_id', 'created', 'assignee', 'status', 'order', 'template', 'data']) + + # # Test nonexistent user is added to project.imported_members + # assert len(p.imported_members) == 1 + # assert p.imported_members[0]['id'] == str(old_user_id) + # assert p.imported_members[0]['roles'] == old_user_roles + # assertKeysEqual(p.imported_members[0], self.user, [ + # 'email', 'phone', 'mobile', + # 'name', 'title_before', 'first_name', 'middle_name', 'last_name', 'title_after', + # ]) + + # # Test re-create user: at re-import the original user should be referenced + # archive2 = archive_to_file(export_projects([p])) + # self.user.id = old_user_id + # self.user.save() + # p2 = import_projects(archive2)[0] + # assert p2.members.count() == 1 + # assert len(p2.imported_members) == 0 + # members_equal(p2.members, self.project.members) def test_import_nonexistent_template_reference(self): archive = archive_to_file(export_projects([self.project]))