Merge branch 'feature/florian/e2e-tests' into 'main'

End-to-End Testing

Closes #1

See merge request reportcreator/reportcreator!778

.gitlab-ci.yml

@@ -150,7 +150,6 @@ test-api:
     - chmod 777 api/test-reports
     - docker run -e DATABASE_HOST=${POSTGRES_PORT_5432_TCP_ADDR} -e DATABASE_NAME=${POSTGRES_DB} -e DATABASE_USER=${POSTGRES_USER} -e DATABASE_PASSWORD=${POSTGRES_PASSWORD} -e ENABLED_PLUGINS='*' --mount=type=bind,source=$PWD/api/test-reports,target=/app/api/test-reports $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA pytest -n auto --junitxml=test-reports/junit.xml --cov=reportcreator_api --cov-report=term --cov-report=xml:test-reports/coverage.xml
 
-
 test-frontend:
   stage: test
   needs: [build-test-frontend]
@@ -165,6 +164,45 @@ test-frontend:
     - mkdir packages/frontend/test-reports
     - docker run --mount=type=bind,source=$PWD/packages/frontend/test-reports,target=/app/packages/frontend/test-reports $CI_REGISTRY_IMAGE/frontend-test:$CI_COMMIT_SHORT_SHA npm run test
 
+integration-test-frontend:
+  stage: test
+  image: docker:27.3.1
+  extends: .depends_docker
+  artifacts:
+    when: always
+    paths:
+      - packages2/frontend/test-reports
+    reports:
+      junit: packages2/frontend/test-reports/junit.xml
+  services:
+    - docker:27.3.1-dind
+  variables:
+    FF_NETWORK_PER_BUILD: "true"
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ""
+    POSTGRES_DB: postgres
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: postgres
+    DATABASE_HOST: postgres
+    POSTGRES_HOST_AUTH_METHOD: trust
+    SYSREPTOR_INTEGRATION_USER: reptor
+  script:
+  # Setup
+    - docker network create mynet
+    - docker pull postgres:15 -q
+    - docker pull mcr.microsoft.com/playwright:v1.49.0-noble -q
+    - docker pull $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA -q
+    - docker pull $CI_REGISTRY_IMAGE/frontend-test:$CI_COMMIT_SHORT_SHA -q
+    - docker cp $(docker create --name temp $CI_REGISTRY_IMAGE/frontend-test:$CI_COMMIT_SHORT_SHA):/app/packages/ packages2 && docker rm temp
+  # Run API + DB
+    - docker run --name postgres --rm --network=mynet --detach -e POSTGRES_DB=$POSTGRES_DB -e POSTGRES_USER=$POSTGRES_USER -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD postgres:15
+    - docker run --name=api --rm --network=mynet --detach -e ENABLED_PLUGINS=* -e LICENSE=$SYSREPTOR_INTEGRATION_LICENSE -e SYSREPTOR_INTEGRATION_USER=$SYSREPTOR_INTEGRATION_USER -e DATABASE_HOST=postgres -e DATABASE_NAME=${POSTGRES_DB} -e DATABASE_USER=${POSTGRES_USER} -e DATABASE_PASSWORD=${POSTGRES_PASSWORD} -e DJANGO_SUPERUSER_PASSWORD=$SYSREPTOR_INTEGRATION_PASSWORD $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA
+  # Let the api migrations run...
+    - sleep 15
+  # Add Superuser
+    - docker exec api python3 manage.py createsuperuser --noinput --username=$SYSREPTOR_INTEGRATION_USER
+  # Run E2E Tests
+    - docker run --name=playwright --rm --network=mynet --mount=type=bind,source=$PWD/packages2/,target=/app/packages/ -w /app/packages/frontend/ -e FRONTEND_ADMIN_PASSWORD=$SYSREPTOR_INTEGRATION_PASSWORD -e FRONTEND_ADMIN_USER=$SYSREPTOR_INTEGRATION_USER -e LICENSE=$SYSREPTOR_INTEGRATION_LICENSE mcr.microsoft.com/playwright:v1.49.0-noble bash -c 'npx playwright test'
 build-release:
   stage: build-release
   extends: .depends_docker
@@ -193,7 +231,6 @@ build-release:
         docker buildx build --provenance false $BASE_IMAGE_BUILD_ARG --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg VERSION="$VERSION_NUMBER" --cache-from $CI_REGISTRY_IMAGE/api-test:$CI_COMMIT_SHORT_SHA --cache-from $CI_REGISTRY_IMAGE/frontend-test:$CI_COMMIT_SHORT_SHA --target=api --platform linux/amd64,linux/arm64 --push --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
       fi
 
-
 release-gitlab-release:
   stage: release
   needs: [build-release]
@@ -205,7 +242,6 @@ release-gitlab-release:
   release:
     tag_name: "$CI_COMMIT_TAG"
     description: "$CI_COMMIT_TAG"
-
 
 release-prod:
   stage: release

Dockerfile

@@ -2,10 +2,10 @@
 ARG TESTED_API_IMAGE=undefined_test_image_used_in_ci
 ARG PROD_API_IMAGE=undefined_prod_image_used_in_ci
 
-
-
 FROM --platform=$BUILDPLATFORM node:20-alpine3.19 AS frontend-dev
 ENV NODE_OPTIONS="--max-old-space-size=4096"
+# Install curl
+RUN apk add --no-cache curl
 WORKDIR /app/packages/
 
 
@@ -167,6 +167,7 @@ ENV VERSION=dev \
 
 # Start server
 EXPOSE 8000
+COPY --chown=1000:1000 api/start.sh /app/api/
 CMD ["/bin/bash", "/app/api/start.sh"]
 
 
@@ -188,7 +189,6 @@ RUN mkdir -p /app/api/sysreptor_plugins/ && chmod 777 /app/api/sysreptor_plugins
 # Copy generated template rendering script
 COPY --from=rendering --chown=user:user /app/packages/rendering/dist /app/packages/rendering/dist/
 
-
 FROM --platform=$BUILDPLATFORM api-test AS api-statics
 # Generate static frontend files
 # Post-process django files (for admin, API browser) and post-process them (e.g. add unique file hash)
@@ -226,7 +226,5 @@ RUN dpkg-query -W -f='${binary:Package}=${Version}\n' > /src/post_installed.txt
     && bash /app/api/download_sources.sh
 USER 1000
 
-
-
 # Default stage
 FROM api

dev/docker-compose.yml

@@ -26,6 +26,27 @@ services:
       REDIS_PASSWORD: reportcreator
     expose:
       - 6379
+  integration-tests:
+    profiles: [test]
+    image: mcr.microsoft.com/playwright:v1.49.0-noble
+    working_dir: /app/packages/frontend/
+    environment:
+      DEBUG: pw:browser
+      # DISPLAY: :0 # for running with --ui in WSL 2 with XServer
+      FRONTEND_ADMIN_USER: "reptor"
+    env_file: ../deploy/app.env
+    volumes:
+      - type: bind
+        source: ../packages/
+        target: /app/packages/
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/.X11-unix:/tmp/.X11-unix
+    command: npx playwright test # --ui
+    # depends_on:
+    #   frontend:
+    #     condition: service_healthy
+    #   api:
+    #     condition: service_healthy
   # rabbitmq:
   #   image: rabbitmq:3
   #   hostname: rabbitmq
@@ -172,6 +193,12 @@ services:
       # CELERY_BROKER_URL: amqp://reportcreator:reportcreator@rabbitmq:5672/
       # CELERY_RESULT_BACKEND: reportcreator_api.tasks.rendering.celery_worker:CustomRPCBackend://reportcreator:reportcreator@rabbitmq:5672/
     env_file: ../deploy/app.env
+    # healthcheck:
+    #   test: ["CMD", "curl", "-f", "-so", "/dev/null", "http://localhost:8000/api/public/utils/healthcheck/"]
+    #   interval: 30s
+    #   timeout: 30s
+    #   retries: 5
+    #   start_period: 10s
     depends_on:
       db:
         condition: service_healthy
@@ -189,6 +216,12 @@ services:
       context: ..
       target: frontend-dev
     command: npm run --workspace=frontend dev
+    # healthcheck: 
+    #   test: ["CMD", "sh", "-c", "curl -s http://localhost:3000/ | grep __nuxt && exit 0 || exit 1"]
+    #   interval: 30s
+    #   timeout: 30s
+    #   retries: 5
+    #   start_period: 10s
     volumes:
       - type: bind
         source: ../packages/

packages/frontend/.gitignore

@@ -36,3 +36,12 @@ coverage
 
 # Unit test output
 test-reports
+test-results
+
+test/e2e/state.json
+test/e2e/demodata/
+
+
+!src/public/static/pdfviewer/dist/
+src/public/static/pdfviewer/dist/**/*
+!src/public/static/pdfviewer/dist/.gitkeep

packages/frontend/package.json

@@ -14,20 +14,17 @@
   "dependencies": {
     "@sysreptor/nuxt-base-layer": "../nuxt-base-layer",
     "@sysreptor/markdown": "../markdown",
-
     "nuxt": "~3.14",
     "vuetify": "~3.7",
     "pinia": "^2.2.4",
     "sass-embedded": "^1.79.5",
-
     "@vueuse/nuxt": "^12.0.0",
     "monaco-editor": "^0.52.0",
     "vuedraggable": "^4.1.0",
     "@iconify/vue": "^4.1.1",
     "@iconify/icons-fluent-emoji": "^1.2.10",
     "@github/webauthn-json": "^2.1.1",
     "@elastic/apm-rum-vue": "^2.1.5",
-
     "date-fns": "^4.1.0",
     "lodash-es": "^4.17.21",
     "dompurify": "^3.1.6"
@@ -44,4 +41,4 @@
   "overrides": {
     "vue": "latest"
   }
-}
+}

packages/frontend/playwright.config.ts

@@ -0,0 +1,33 @@
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  globalSetup: './test/e2e/global-setup',
+  reporter: [
+    ['junit', {  outputFile: 'test-reports/junit.xml' }]
+  ],
+  outputDir: './test-reports',
+  testDir: './test/e2e',
+  // fullyParallel: true,
+
+  // Fail the build on CI if you accidentally left test.only in the source code.
+  forbidOnly: !!process.env.CI,
+
+  // Retry on CI only.
+  retries: process.env.CI ? 2 : 0,
+
+  // Opt out of parallel tests on CI.
+  workers: 1,
+
+  use: {
+    baseURL: 'http://localhost:3000',
+    // Collect trace when retrying the failed test.
+    trace: 'on-first-retry',
+    storageState: './test/e2e/state.json',
+  },
+  webServer: {
+    command: 'echo "starting webserver" && npm run dev',
+    url: 'http://localhost:3000',
+    reuseExistingServer: true,
+    stdout: "pipe",
+  },
+});

packages/frontend/src/components/CreateFindingDialog.vue

@@ -1,10 +1,11 @@
 <template>
-  <s-dialog v-model="dialogVisible" width="60%" max-width="60%">
+  <s-dialog v-model="dialogVisible" width="60%" max-width="60%" data-testid="create-finding-dialog">
     <template #activator>
       <btn-confirm
         :action="() => dialogVisible = true"
         :confirm="false"
         :disabled="project.readonly"
+        data-testid="create-finding-button"
         button-text="Add"
         button-icon="mdi-plus"
         tooltip-text="Add Finding (Ctrl+J)"
@@ -49,10 +50,10 @@
               </template>
 
               <template #no-data v-if="templates.data.value.length === 0 && templates.hasNextPage.value">
-                <page-loader :items="templates" />
+                <page-loader :items="templates" data-testid="page-loader" />
               </template>
               <template #append-item v-if="templates.data.value.length > 0">
-                <page-loader :items="templates" />
+                <page-loader :items="templates" data-testid="page-loader" />
               </template>
             </s-combobox>
           </v-col>

packages/frontend/src/components/Design/CreateDesignDialog.vue

@@ -14,6 +14,7 @@
         <s-project-type-selection
           v-model="currentDesign"
           label="Copy Existing Design (optional)"
+          data-testid="copy-existing-design"
           return-object
           :required="false"
           autofocus
@@ -36,6 +37,7 @@
           v-else
           @click="createEmptyDesign"
           :loading="actionInProgress"
+          data-testid="submit-design"
           text="Create Empty Design"
         />
       </v-card-actions>

packages/frontend/src/components/Design/InputFieldDefinition.vue

@@ -248,6 +248,7 @@
         v-model:markdown-editor-mode="localSettings.designMarkdownEditorMode"
         :readonly="props.readonly"
         :disable-validation="true"
+        data-testid='default-value'
       />
 
       <!-- List Item -->

packages/frontend/src/components/History/Timeline.vue

@@ -40,12 +40,13 @@
           v-if="props.currentUrl"
           :value="{history_type: '~', history_change_reason: 'Current Version'} as unknown as HistoryTimelineRecord"
           :to="currentUrl"
+
         >
           <template #info><span /></template>
         </history-timeline-item>
         <template v-for="item, idx in historyRecords.data.value" :key="idx">
           <slot v-if="!(idx === 0 && item.history_type === '~' && !item.history_change_reason)" name="item" :item="item">
-            <history-timeline-item :value="item" />
+    <history-timeline-item :value="item" />
           </slot>
         </template>
       </v-timeline>

packages/frontend/src/components/History/TimelineItem.vue

@@ -6,6 +6,7 @@
     :class="{'timeline-item-link': !!to, 'nuxt-link-active': isExactActive}"
     @click="to ? navigate() : undefined"
     @auxclick="to ? openInNewTab($event) : undefined"
+    :data-testid="`timeline-item-${props.value.history_title}`"
   >
     <slot name="info">
       <chip-date :value="props.value.history_date" />
@@ -14,10 +15,10 @@
       <br>
     </slot>
 
-    <slot name="title">
-      <span v-if="props.value.history_change_reason">{{ props.value.history_change_reason }}</span>
-      <span v-else-if="props.value.history_type === '+'">Created</span>
-      <span v-else-if="props.value.history_type === '-'">Deleted</span>
+    <slot name="title" >
+      <span v-if="props.value.history_change_reason" :data-testid="`timeline-item-${props.value.history_title}`">{{ props.value.history_change_reason }}</span>
+      <span v-else-if="props.value.history_type === '+'" :data-testid="`timeline-item-${props.value.history_title}`">Created</span>
+      <span v-else-if="props.value.history_type === '-'" :data-testid="`timeline-item-${props.value.history_title}`">Deleted</span>
     </slot>
   </v-timeline-item>
 </template>

packages/frontend/src/components/LoginForm.vue

@@ -45,6 +45,7 @@
           <slot name="actions"></slot>
           <s-btn-primary
             type="submit"
+            data-testid="login-submit"
             text="Login"
           />
         </v-card-actions>
@@ -157,9 +158,9 @@
 </template>
 
 <script setup lang="ts">
+import { LoginResponseStatus, mfaMethodChoices, MfaMethodType } from '#imports';
 import { get as navigatorCredentialsGet, parseRequestOptionsFromJSON } from "@github/webauthn-json/browser-ponyfill";
 import type { VOtpInput } from "vuetify/components";
-import { LoginResponseStatus, mfaMethodChoices, MfaMethodType } from '#imports';
 
 const props = defineProps({
   username: {

packages/frontend/src/components/Markdown/Toolbar.vue

@@ -72,7 +72,7 @@
       size="small"
       density="comfortable"
     >
-      <v-icon icon="mdi-dots-vertical" />
+      <v-icon icon="mdi-dots-vertical"/>
       <v-tooltip activator="parent" location="top" text="More actions" />
 
       <v-menu activator="parent">
@@ -95,30 +95,30 @@
 </template>
 
 <script setup lang="ts">
+import { MarkdownEditorMode, type ReferenceItem } from '#imports';
+import { levelNumberFromLevelName } from '@base/utils/cvss';
 import {
-  type EditorView,
   type EditorState,
+  type EditorView,
   insertCodeBlock,
   insertTable,
   insertText,
+  isTaskListInSelection,
+  isTypeInSelection,
   redo,
   redoDepth,
   toggleEmphasis,
+  toggleFootnote,
   toggleLink,
   toggleListOrdered,
   toggleListUnordered,
-  toggleTaskList,
   toggleStrikethrough,
   toggleStrong,
-  toggleFootnote,
+  toggleTaskList,
   undo,
   undoDepth,
-  isTypeInSelection,
-  isTaskListInSelection,
 } from '@sysreptor/markdown/editor';
 import type { VToolbar } from 'vuetify/lib/components/index.mjs';
-import { levelNumberFromLevelName } from '@base/utils/cvss';
-import { MarkdownEditorMode, type ReferenceItem } from '#imports';
 
 const props = defineProps<{
   editorView?: EditorView|null;

packages/frontend/src/components/Notes/Menu.vue

@@ -90,6 +90,7 @@
             :action="props.createNote!"
             :disabled="!canCreate"
             :confirm="false"
+            data-testid="create-note"
             button-text="Add"
             button-icon="mdi-plus"
             tooltip-text="Add Note (Ctrl+J)"