From 78b0695a6c8dd6687c610ba21d093fd4ca216a43 Mon Sep 17 00:00:00 2001 From: Tiffany Forkner Date: Tue, 17 Sep 2024 17:31:13 -0400 Subject: [PATCH] UPLOAD-1695 Show upload status of file in info (#475) * updated the /info/{uploadId} endpoint to add upload_status to the response * upload_status contains * status [Initiated, In Progress, Complete] * chunk_received_at * updated the upload page to display the file detail data from the /info/{uploadId} when the upload status is In Progress or Complete * added checks to determine the host (client uploading the file) and the peer (client viewing the file status) * added ability for host to resume an upload * added ability for host to pause and resume an upload * added logic for handling different upload statuses for hosts and peers * if the upload status is Initiated, the upload form is displayed * if the host begins uploading a file, the upload form is hidden, the progress bar is shown, and the file details are shown * if the host refreshes while uploading, a resume upload form is shown and the file details are shown * if the host resumes an upload, the resume upload form is hidden, the progress bar is shown, and the file details are shown * if the peer views the page while the upload status is In Progress, only the file details are shown * if the upload status is complete, only the file details are shown * added styling to the file details * various clean ups for the html, css, and javascript * playwright tests and accessibility changes --------- Co-authored-by: Nicole Zonnenberg Co-authored-by: Alex de los Reyes --- docs/openapi.yml | 411 ++++++++-------- .../test/upload-test-accessibility.spec.ts | 22 +- .../playwright/test/upload-test-e2e.spec.ts | 8 +- .../playwright/test/upload-test-pages.spec.ts | 122 ++++- .../smoke/playwright/test/upload-test.spec.ts | 2 +- upload-server/cmd/cli/datastore.go | 5 +- upload-server/cmd/cli/info.go | 33 +- upload-server/cmd/cli/status.go | 2 + upload-server/go.mod | 1 + upload-server/go.sum | 2 + upload-server/internal/delivery/deliver.go | 7 +- upload-server/internal/delivery/s3.go | 7 +- upload-server/internal/metadata/metadata.go | 17 +- upload-server/internal/stores3/client.go | 1 + upload-server/internal/ui/assets/demo.css | 11 - upload-server/internal/ui/assets/form.css | 74 --- upload-server/internal/ui/assets/index.css | 234 +++++++-- upload-server/internal/ui/assets/progress.css | 12 +- upload-server/internal/ui/assets/tusclient.js | 443 ++++++++++++++++++ upload-server/internal/ui/assets/upload.css | 155 ++++++ .../internal/ui/components/navbar.go | 12 +- .../internal/ui/components/navbar.html | 21 +- .../internal/ui/components/newuploadbtn.go | 3 + .../internal/ui/components/newuploadbtn.html | 5 + upload-server/internal/ui/index.html | 10 +- upload-server/internal/ui/manifest.tmpl | 78 ++- upload-server/internal/ui/ui.go | 95 +++- upload-server/internal/ui/upload.tmpl | 267 +++++++---- upload-server/pkg/fileinspector/status.go | 82 +++- upload-server/pkg/info/info.go | 36 +- upload-server/pkg/s3inspector/s3.go | 13 +- 31 files changed, 1645 insertions(+), 546 deletions(-) delete mode 100644 upload-server/internal/ui/assets/demo.css delete mode 100644 upload-server/internal/ui/assets/form.css create mode 100644 upload-server/internal/ui/assets/tusclient.js create mode 100644 upload-server/internal/ui/assets/upload.css create mode 100644 upload-server/internal/ui/components/newuploadbtn.go create mode 100644 upload-server/internal/ui/components/newuploadbtn.html diff --git a/docs/openapi.yml b/docs/openapi.yml index c61868688..67c747e66 100644 --- a/docs/openapi.yml +++ b/docs/openapi.yml @@ -6,13 +6,13 @@ info: contact: email: knu1@cdc.gov servers: - - url: 'https://apidev.cdc.gov' + - url: "https://apidev.cdc.gov" description: Development server - - url: 'https://apitst.cdc.gov' + - url: "https://apitst.cdc.gov" description: Test server - - url: 'https://apistg.cdc.gov' + - url: "https://apistg.cdc.gov" description: Staging server - - url: 'https://api.cdc.gov' + - url: "https://api.cdc.gov" description: Production server paths: @@ -22,34 +22,34 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: JSON object describing the provided auth token. content: application/json: schema: - $ref: '#/components/schemas/Oauth-Token-Validation' - + $ref: "#/components/schemas/Oauth-Token-Validation" + post: summary: This POST request goes to SAMS to obtain a session token for authentication. requestBody: content: application/x-www-form-urlencoded: schema: - $ref: '#/components/schemas/Oauth-Form' + $ref: "#/components/schemas/Oauth-Form" responses: - '200': + "200": description: JSON object containing the new auth token. content: application/json: schema: - $ref: '#/components/schemas/Oauth-Token-Grant' - '400': + $ref: "#/components/schemas/Oauth-Token-Grant" + "400": description: Bad Request content: application/json: schema: - $ref: '#/components/schemas/SAMS-Error' - + $ref: "#/components/schemas/SAMS-Error" + /oauth/refresh: post: summary: This POST request goes to SAMS to generate a valid refresh token. @@ -72,19 +72,19 @@ paths: type: string description: Refresh token given by SAMS in the oauth grant response. responses: - '200': - description: JSON object containing the new auth token. - content: - application/json: - schema: - $ref: '#/components/schemas/Oauth-Token-Grant' - '400': + "200": + description: JSON object containing the new auth token. + content: + application/json: + schema: + $ref: "#/components/schemas/Oauth-Token-Grant" + "400": description: Bad Request content: application/json: schema: - $ref: '#/components/schemas/SAMS-Error' - + $ref: "#/components/schemas/SAMS-Error" + /upload: get: summary: tusd Heartbeat @@ -92,76 +92,82 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Success - + post: - summary: An empty POST request is used to create a new upload resource. The + summary: + An empty POST request is used to create a new upload resource. The Upload-Length header indicates the size of the entire upload in bytes. - If the Creation With Upload extension is available, the Client MAY include + If the Creation With Upload extension is available, the Client MAY include parts of the upload in the initial Creation request - description: Endpoint for the Creation extension. Modified by the Creation With + description: + Endpoint for the Creation extension. Modified by the Creation With Upload extension. security: - bearerAuth: [] parameters: - - name: Content-Length - in: header - description: Must be 0 for creation extension. May be a positive number for - Creation With Upload extension. - schema: - type: integer - - name: Upload-Length - in: header - schema: - $ref: "#/components/schemas/Upload-Length" - - name: Tus-Resumable - in: header - schema: - $ref: "#/components/schemas/Tus-Resumable" - - name: Upload-Metadata - in: header - description: Added by the Creation extension. The Upload-Metadata request - and response header MUST consist of one or more comma-separated key-value - pairs. The key and value MUST be separated by a space. The key MUST NOT - contain spaces and commas and MUST NOT be empty. The key SHOULD be ASCII - encoded and the value MUST be Base64 encoded. All keys MUST be unique. The - value MAY be empty. In these cases, the space, which would normally separate - the key and the value, MAY be left out. Since metadata can contain arbitrary - binary values, Servers SHOULD carefully validate metadata values or sanitize - them before using them as header values to avoid header smuggling. - schema: - type: string - - name: Upload-Concat - in: header - description: Added by the Concatenation extension. The Upload-Concat request - and response header MUST be set in both partial and final upload creation - requests. It indicates whether the upload is either a partial or final upload. - If the upload is a partial one, the header value MUST be partial. In the - case of a final upload, its value MUST be final followed by a semicolon - and a space-separated list of partial upload URLs that will be concatenated. - The partial uploads URLs MAY be absolute or relative and MUST NOT contain - spaces as defined in RFC 3986. - schema: - type: string - - name: Upload-Defer-Length - in: header - description: Added by the Creation Defer Length extension. The Upload-Defer-Length - request and response header indicates that the size of the upload is not - known currently and will be transferred later. Its value MUST be 1. If the - length of an upload is not deferred, this header MUST be omitted. - schema: - type: integer - enum: - - 1 - - name: Upload-Offset - in: header - schema: - $ref: "#/components/schemas/Upload-Offset" - - name: Upload-Checksum - in: header - schema: - $ref: "#/components/schemas/Upload-Checksum" + - name: Content-Length + in: header + description: + Must be 0 for creation extension. May be a positive number for + Creation With Upload extension. + schema: + type: integer + - name: Upload-Length + in: header + schema: + $ref: "#/components/schemas/Upload-Length" + - name: Tus-Resumable + in: header + schema: + $ref: "#/components/schemas/Tus-Resumable" + - name: Upload-Metadata + in: header + description: + Added by the Creation extension. The Upload-Metadata request + and response header MUST consist of one or more comma-separated key-value + pairs. The key and value MUST be separated by a space. The key MUST NOT + contain spaces and commas and MUST NOT be empty. The key SHOULD be ASCII + encoded and the value MUST be Base64 encoded. All keys MUST be unique. The + value MAY be empty. In these cases, the space, which would normally separate + the key and the value, MAY be left out. Since metadata can contain arbitrary + binary values, Servers SHOULD carefully validate metadata values or sanitize + them before using them as header values to avoid header smuggling. + schema: + type: string + - name: Upload-Concat + in: header + description: + Added by the Concatenation extension. The Upload-Concat request + and response header MUST be set in both partial and final upload creation + requests. It indicates whether the upload is either a partial or final upload. + If the upload is a partial one, the header value MUST be partial. In the + case of a final upload, its value MUST be final followed by a semicolon + and a space-separated list of partial upload URLs that will be concatenated. + The partial uploads URLs MAY be absolute or relative and MUST NOT contain + spaces as defined in RFC 3986. + schema: + type: string + - name: Upload-Defer-Length + in: header + description: + Added by the Creation Defer Length extension. The Upload-Defer-Length + request and response header indicates that the size of the upload is not + known currently and will be transferred later. Its value MUST be 1. If the + length of an upload is not deferred, this header MUST be omitted. + schema: + type: integer + enum: + - 1 + - name: Upload-Offset + in: header + schema: + $ref: "#/components/schemas/Upload-Offset" + - name: Upload-Checksum + in: header + schema: + $ref: "#/components/schemas/Upload-Checksum" requestBody: description: (Possibly partial) content of the file. Required if Content-Length > 0. required: false @@ -171,7 +177,7 @@ paths: type: string format: binary responses: - '201': + "201": description: Created headers: Tus-Resumable: @@ -181,7 +187,8 @@ paths: schema: $ref: "#/components/schemas/Upload-Offset" Upload-Expires: - description: Added by the Creation With Upload Extension in combination + description: + Added by the Creation With Upload Extension in combination with the expiration extension. The Upload-Expires response header indicates the time after which the unfinished upload expires. A Server MAY wish to remove incomplete uploads after a given period of time @@ -203,15 +210,16 @@ paths: description: Url of the created resource. schema: type: string - '400': - description: Added by the Creation With Upload Extension in combination + "400": + description: + Added by the Creation With Upload Extension in combination with the checksum extension. The checksum algorithm is not supported by the server headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '412': + "412": description: Precondition Failed headers: Tus-Resumable: @@ -220,23 +228,26 @@ paths: Tus-Version: schema: $ref: "#/components/schemas/Tus-Version" - '413': - description: If the length of the upload exceeds the maximum, which MAY + "413": + description: + If the length of the upload exceeds the maximum, which MAY be specified using the Tus-Max-Size header, the Server MUST respond with the 413 Request Entity Too Large status. headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '415': - description: Added by the Creation With Upload Extension. Content-Type was + "415": + description: + Added by the Creation With Upload Extension. Content-Type was not application/offset+octet-stream headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '460': - description: Added by the Creation With Upload Extension in combination + "460": + description: + Added by the Creation With Upload Extension in combination with the checksum extension. Checksums mismatch headers: Tus-Resumable: @@ -244,12 +255,13 @@ paths: $ref: "#/components/schemas/Tus-Resumable" options: summary: Request to gather information about the Server's current configuration - description: An OPTIONS request MAY be used to gather information about the + description: + An OPTIONS request MAY be used to gather information about the Server's current configuration. A successful response indicated by the 204 No Content or 200 OK status MUST contain the Tus-Version header. It MAY include the Tus-Extension and Tus-Max-Size headers. responses: - '200': + "200": description: Success headers: Tus-Resumable: @@ -267,7 +279,7 @@ paths: Tus-Extension: schema: $ref: "#/components/schemas/Tus-Extension" - '204': + "204": description: Success headers: Tus-Resumable: @@ -288,31 +300,32 @@ paths: /upload/{tguid}: delete: summary: Added by the Termination extension. - description: When receiving a DELETE request for an existing upload the Server + description: + When receiving a DELETE request for an existing upload the Server SHOULD free associated resources and MUST respond with the 204 No Content status confirming that the upload was terminated. For all future requests to this URL, the Server SHOULD respond with the 404 Not Found or 410 Gone status. operationId: FilesDelete parameters: - - name: tguid - in: path - required: true - schema: - type: string - - name: Tus-Resumable - in: header - required: true - schema: - $ref: "#/components/schemas/Tus-Resumable" + - name: tguid + in: path + required: true + schema: + type: string + - name: Tus-Resumable + in: header + required: true + schema: + $ref: "#/components/schemas/Tus-Resumable" responses: - '204': + "204": description: Upload was terminated headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '412': + "412": description: Precondition Failed headers: Tus-Resumable: @@ -328,18 +341,18 @@ paths: security: - bearerAuth: [] parameters: - - name: tguid - in: path - required: true - schema: - type: string - - name: Tus-Resumable - in: header - required: true - schema: - $ref: "#/components/schemas/Tus-Resumable" + - name: tguid + in: path + required: true + schema: + type: string + - name: Tus-Resumable + in: header + required: true + schema: + $ref: "#/components/schemas/Tus-Resumable" responses: - '200': + "200": description: Returns offset headers: Tus-Resumable: @@ -349,38 +362,41 @@ paths: schema: type: string enum: - - no-store + - no-store Upload-Offset: schema: $ref: "#/components/schemas/Upload-Offset" Upload-Length: schema: $ref: "#/components/schemas/Upload-Length" - '403': - description: If the resource is not found, the Server SHOULD return either + "403": + description: + If the resource is not found, the Server SHOULD return either the 404 Not Found, 410 Gone or 403 Forbidden status without the Upload-Offset header. headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '404': - description: If the resource is not found, the Server SHOULD return either + "404": + description: + If the resource is not found, the Server SHOULD return either the 404 Not Found, 410 Gone or 403 Forbidden status without the Upload-Offset header. headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '410': - description: If the resource is not found, the Server SHOULD return either + "410": + description: + If the resource is not found, the Server SHOULD return either the 404 Not Found, 410 Gone or 403 Forbidden status without the Upload-Offset header. headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '412': + "412": description: Precondition Failed headers: Tus-Resumable: @@ -391,39 +407,40 @@ paths: $ref: "#/components/schemas/Tus-Version" patch: summary: Used to resume the upload - description: 'The Server SHOULD accept PATCH requests against any upload URL + description: + "The Server SHOULD accept PATCH requests against any upload URL and apply the bytes contained in the message at the given offset specified by the Upload-Offset header. All PATCH requests MUST use Content-Type: application/offset+octet-stream, - otherwise the server SHOULD return a 415 Unsupported Media Type status.' + otherwise the server SHOULD return a 415 Unsupported Media Type status." operationId: FilePatch security: - bearerAuth: [] parameters: - - name: tguid - in: path - required: true - schema: - type: string - - name: Tus-Resumable - in: header - required: true - schema: - $ref: "#/components/schemas/Tus-Resumable" - - name: Content-Length - in: header - description: Length of the body of this request - required: true - schema: - type: integer - - name: Upload-Offset - in: header - required: true - schema: - $ref: "#/components/schemas/Upload-Offset" - - name: Upload-Checksum - in: header - schema: - $ref: "#/components/schemas/Upload-Checksum" + - name: tguid + in: path + required: true + schema: + type: string + - name: Tus-Resumable + in: header + required: true + schema: + $ref: "#/components/schemas/Tus-Resumable" + - name: Content-Length + in: header + description: Length of the body of this request + required: true + schema: + type: integer + - name: Upload-Offset + in: header + required: true + schema: + $ref: "#/components/schemas/Upload-Offset" + - name: Upload-Checksum + in: header + schema: + $ref: "#/components/schemas/Upload-Checksum" requestBody: description: Remaining (possibly partial) content of the file. Required if Content-Length > 0. required: false @@ -433,7 +450,7 @@ paths: type: string format: binary responses: - '204': + "204": description: Upload offset was updated headers: Tus-Resumable: @@ -443,7 +460,8 @@ paths: schema: $ref: "#/components/schemas/Tus-Resumable" Upload-Expires: - description: Added by the expiration extension. The Upload-Expires response + description: + Added by the expiration extension. The Upload-Expires response header indicates the time after which the unfinished upload expires. A Server MAY wish to remove incomplete uploads after a given period of time to prevent abandoned uploads from taking up extra storage. @@ -460,28 +478,30 @@ paths: Upload-Expires header MUST be in RFC 7231 datetime format. schema: type: string - '400': - description: Added by the checksum extension. The checksum algorithm is + "400": + description: + Added by the checksum extension. The checksum algorithm is not supported by the server headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '403': - description: In the concatenation extension, the Server MUST respond with + "403": + description: + In the concatenation extension, the Server MUST respond with the 403 Forbidden status to PATCH requests against a final upload URL and MUST NOT modify the final or its partial uploads. headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '404': + "404": description: PATCH request against a non-existent resource headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '409': + "409": description: PATCH request with Upload-Offset unequal to the offset of the resource on the server. The Upload-Offset header's value MUST be equal to the current offset of the resource. headers: Tus-Resumable: @@ -502,13 +522,13 @@ paths: Tus-Version: schema: $ref: "#/components/schemas/Tus-Version" - '415': + "415": description: Content-Type was not application/offset+octet-stream headers: Tus-Resumable: schema: $ref: "#/components/schemas/Tus-Resumable" - '460': + "460": description: Added by the checksum extension. Checksums mismatch headers: Tus-Resumable: @@ -531,7 +551,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Info-Response' + $ref: "#/components/schemas/Info-Response" 404: description: Info file or upload file not found. @@ -544,14 +564,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Health-Response' - + $ref: "#/components/schemas/Health-Response" components: securitySchemes: - bearerAuth: - type: http - scheme: bearer + bearerAuth: + type: http + scheme: bearer schemas: Oauth-Token-Validation: type: object @@ -571,16 +590,16 @@ components: token_type: type: string description: The Oauth token type. - expires_in: + expires_in: type: integer description: Duration in seconds when the token will expire. refresh_token: type: string description: Token used to get a new auth token when it expires. - scope: + scope: type: string description: Authorization scopes associated with this auth token. - resource: + resource: type: array description: List of resources this token has access to. Oauth-Form: @@ -611,7 +630,7 @@ components: Tus-Resumable: type: string enum: - - 1.0.0 + - 1.0.0 description: Protocol version Tus-Version: description: The Tus-Version response header MUST be a comma-separated @@ -631,7 +650,8 @@ components: limit. type: integer Upload-Length: - description: The Upload-Length request and response header indicates the size + description: + The Upload-Length request and response header indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. In the concatenation extension, the Client MUST NOT include the Upload-Length header in the final upload creation @@ -649,9 +669,9 @@ components: type: string Upload-Checksum: description: Added by the checksum extension. The Upload-Checksum request - header contains information about the checksum of the current body payload. - The header MUST consist of the name of the used checksum algorithm and the - Base64 encoded checksum separated by a space. + header contains information about the checksum of the current body payload. + The header MUST consist of the name of the used checksum algorithm and the + Base64 encoded checksum separated by a space. type: string Info-Response: type: object @@ -665,7 +685,7 @@ components: description: Unique identifier of the uploaded file assigned by the API. dex_ingest_datetime: type: string - description: Timestamp of the API's first interaction with the uploaded file. + description: Timestamp of the API's first interaction with the uploaded file. version: type: string description: The version of the sender manifest utilized for the uploaded file. @@ -689,21 +709,35 @@ components: description: The name of the original uploaded file. custom_metadata: type: string - description: Data stream specific metadata fields; varies per data stream. + description: Data stream specific metadata fields; varies per data stream. file_info: type: object description: Information about the file that was uploaded. properties: size_bytes: type: integer - description: Current size of the file in bytes. + description: Current size of the file in bytes. updated_at: type: string description: Timestamp of the most recent time the file was changed. + upload_status: + type: object + description: Information about the status of the file upload. + properties: + status: + type: string + description: Current status of the file upload. + enum: + - Initiated + - In Progress + - Complete + chunk_received_at: + type: string + description: Timestamp that the most recent upload chunk was received. deliveries: type: array description: Information about the file destination delivery to target locations. - items: + items: type: object properties: status: @@ -714,7 +748,7 @@ components: description: Name of the file destination delivery target. location: type: string - description: Location path of the file destination delivery target. + description: Location path of the file destination delivery target. delivered_at: type: string description: Timestamp of the file delivery to the target destination location. @@ -734,7 +768,7 @@ components: type: object properties: schema: - $ref: '#/components/schemas/Health-Response-Item' + $ref: "#/components/schemas/Health-Response-Item" Health-Response-Item: type: object properties: @@ -748,7 +782,6 @@ components: type: string description: Message containing more details as to the issue with the service. Provided when the status is not UP. - security: - {} - - bearerAuth: [] \ No newline at end of file + - bearerAuth: [] diff --git a/tests/smoke/playwright/test/upload-test-accessibility.spec.ts b/tests/smoke/playwright/test/upload-test-accessibility.spec.ts index a7c8fbb97..0e83e3005 100644 --- a/tests/smoke/playwright/test/upload-test-accessibility.spec.ts +++ b/tests/smoke/playwright/test/upload-test-accessibility.spec.ts @@ -1,13 +1,11 @@ import { test, expect } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; - test.describe.configure({ mode: 'parallel' }); -test.describe('Upload User Interface', () => { - - const axeRuleTags = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]; +const axeRuleTags = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]; - test('Checks accessiblity for the upload landing page', async ({ page }, testInfo) => { +test.describe('Upload Landing Page', () => { + test('has accessible features when loaded', async ({ page }, testInfo) => { await page.goto(`/`) const results = await new AxeBuilder({ page }) .withTags(axeRuleTags) @@ -15,19 +13,18 @@ test.describe('Upload User Interface', () => { expect(results.violations).toEqual([]); }); +}); +test.describe('Upload Manifest Page', () => { [ - { dataStream: "celr", route: "csv" }, - { dataStream: "celr", route: "hl7v2" }, { dataStream: "covid", route: "all-monthly-vaccination-csv" }, { dataStream: "covid", route: "bridge-vaccination-csv" }, - { dataStream: "daart", route: "hl7" }, { dataStream: "dex", route: "hl7-hl7ingress" }, { dataStream: "dextesting", route: "testevent1" }, { dataStream: "ehdi", route: "csv" }, { dataStream: "eicr", route: "fhir" }, + { dataStream: "h5", route: "influenza-vaccination-csv" }, { dataStream: "influenza", route: "vaccination-csv" }, - { dataStream: "ndlp", route: "aplhistoricaldata" }, { dataStream: "ndlp", route: "covidallmonthlyvaccination" }, { dataStream: "ndlp", route: "covidbridgevaccination" }, { dataStream: "ndlp", route: "influenzavaccination" }, @@ -45,14 +42,15 @@ test.describe('Upload User Interface', () => { expect(results.violations).toEqual([]); }) }); +}); - test(`Checks accessibliity for the upload page for the daart/hl7 manifest`, async ({ page }) => { - await page.goto(`/manifest?data_stream=daart&data_stream_route=hl7`); +test.describe('File Upload Page', () => { + test(`Checks accessibliity for the upload page for the dextesting/testevent1 manifest`, async ({ page }) => { + await page.goto(`/manifest?data_stream=dextesting&data_stream_route=testevent1`); await page.getByLabel('Sender Id').fill('Sender123'); await page.getByLabel('Data Producer Id').fill('Producer123'); await page.getByLabel('Jurisdiction').fill('Jurisdiction123'); await page.getByLabel('Received Filename').fill('small-test-file'); - await page.getByLabel('Original File Timestamp').fill('Timestamp123'); await page.getByRole('button', { name: /next/i }).click(); await expect(page).toHaveURL(/status/) const results = await new AxeBuilder({ page }) diff --git a/tests/smoke/playwright/test/upload-test-e2e.spec.ts b/tests/smoke/playwright/test/upload-test-e2e.spec.ts index 3f73945bc..defc9453f 100644 --- a/tests/smoke/playwright/test/upload-test-e2e.spec.ts +++ b/tests/smoke/playwright/test/upload-test-e2e.spec.ts @@ -8,7 +8,7 @@ test.describe("Upload End to End Tests", () => { await page.goto(`/`); await page.getByLabel('Data Stream', {exact: true}).fill(dataStream); await page.getByLabel('Data Stream Route').fill(route); - await page.getByRole('button', {name: /submit/i }).click(); + await page.getByRole('button', {name: /next/i }).click(); await page.getByLabel('Sender Id').fill('Sender123') await page.getByLabel('Data Producer Id').fill('Producer123') @@ -18,11 +18,11 @@ test.describe("Upload End to End Tests", () => { const fileChooserPromise = page.waitForEvent('filechooser'); - await page.locator('input[type="file"]').click(); + await page.getByRole('button', {name: 'Browse Files'}).click(); const fileChooser = await fileChooserPromise; - await fileChooser.setFiles('../upload-files/10KB-test-file'); + await fileChooser.setFiles('../upload-files/10KB-test-file'); - await page.getByText('Download 10KB-test-file') + await expect(page.getByText('Upload Status: Complete')).toBeVisible(); }) }) diff --git a/tests/smoke/playwright/test/upload-test-pages.spec.ts b/tests/smoke/playwright/test/upload-test-pages.spec.ts index 2cd92cdfa..7173b4393 100644 --- a/tests/smoke/playwright/test/upload-test-pages.spec.ts +++ b/tests/smoke/playwright/test/upload-test-pages.spec.ts @@ -2,18 +2,29 @@ import { expect, test } from '@playwright/test'; test.describe.configure({ mode: 'parallel' }); +test.describe("Upload Landing Page", () => { + test("has the expected elements to start a file upload process", async ({page}) => { + await page.goto(`/`); + const nav = page.getByRole('navigation') + await expect(nav.getByRole("link").and(nav.getByText('Skip to main content Upload'))).toBeHidden() + await expect(nav.getByRole("link").and(nav.getByText('Upload'))).toBeVisible() + await expect(page.getByRole('heading', { level: 1 })).toHaveText('Welcome to DEX Upload') + await expect(page.getByRole('heading', { level: 2 })).toHaveText('Start the upload process by entering a data stream and route.') + await expect(page.getByRole("textbox", { name: "Data Stream", exact: true })).toBeVisible() + await expect(page.getByRole("textbox", { name: "Data Stream Route", exact: true })).toBeVisible() + await expect(page.getByRole("button", {name: "Next"})).toBeVisible() + }) +}); + [ - { dataStream: "celr", route: "csv" }, - { dataStream: "celr", route: "hl7v2" }, { dataStream: "covid", route: "all-monthly-vaccination-csv" }, { dataStream: "covid", route: "bridge-vaccination-csv" }, - { dataStream: "daart", route: "hl7" }, { dataStream: "dex", route: "hl7-hl7ingress" }, { dataStream: "dextesting", route: "testevent1" }, { dataStream: "ehdi", route: "csv" }, { dataStream: "eicr", route: "fhir" }, + { dataStream: "h5", route: "influenza-vaccination-csv" }, { dataStream: "influenza", route: "vaccination-csv" }, - { dataStream: "ndlp", route: "aplhistoricaldata" }, { dataStream: "ndlp", route: "covidallmonthlyvaccination" }, { dataStream: "ndlp", route: "covidbridgevaccination" }, { dataStream: "ndlp", route: "influenzavaccination" }, @@ -23,9 +34,12 @@ test.describe.configure({ mode: 'parallel' }); { dataStream: "routine", route: "immunization-other" }, { dataStream: "rsv", route: "prevention-csv" }, ].forEach(({ dataStream, route }) => { - test.describe("Manifest UI endpoint", () => { - test(`Data stream: ${dataStream} / Route: ${route} displays the appropriate UI fields`, async ({ page }) => { + test.describe("Upload Manifest Page", () => { + test(`has the expected metadata elements for Data stream: ${dataStream} / Route: ${route}`, async ({ page }) => { await page.goto(`/manifest?data_stream=${dataStream}&data_stream_route=${route}`); + const nav = page.locator('nav') + await expect(nav.getByRole("link").and(nav.getByText('Skip to main content'))).toBeHidden() + await expect(nav.getByRole("link").and(nav.getByText('Upload'))).toBeVisible() const title = page.locator('h1'); await expect(title).toHaveText('Please fill in the sender manifest form for your file'); // TODO: Add more assertions on individual manifest page elements @@ -34,3 +48,99 @@ test.describe.configure({ mode: 'parallel' }); }) }) }); + +test.describe("File Uploader Page", () => { + test("has the expected elements to prepare to upload a file", async ({ page, baseURL }) => { + const apiURL = baseURL.replace('8081', '8080') + const dataStream = 'dextesting'; + const route = 'testevent1'; + + await page.goto(`/manifest?data_stream=${dataStream}&data_stream_route=${route}`); + + await page.getByLabel('Sender Id').fill('Sender123') + await page.getByLabel('Data Producer Id').fill('Producer123') + await page.getByLabel('Jurisdiction').fill('Jurisdiction123') + await page.getByLabel('Received Filename').fill('small-test-file') + await page.getByRole('button', { name: /next/i }).click(); + + await expect(page.getByRole('heading', { level: 1, includeHidden: false }).nth(0)).toHaveText('File Uploader') + const uploadEndpoint = page.getByRole('textbox', { name: 'Upload Endpoint' }); + // not the greatest way to interpret the endpoint value here, but this will have to work for now... + await expect(uploadEndpoint).toHaveValue(`${apiURL}/files/`) + + const chunkSize = page.getByLabel('Chunk size (bytes)'); + const chunkSizeLabel = page.locator('label', { hasText: 'Chunk size (bytes)' }) + await expect(chunkSizeLabel).toContainText('Note: Chunksize should be set on the client for uploading files of large size (1GB or over).') + await expect(chunkSize).toHaveValue('40000000') + + const parallelUploadRequests = page.getByRole('spinbutton', { name: 'Parallel upload requests' }) + await expect(parallelUploadRequests).toHaveValue('1') + + const browseFilesButton = page.getByRole('button', { name: 'Browse Files' }) + await expect(browseFilesButton).toHaveAttribute("onclick", "files.click()") + }) +}) + +test.describe("Upload Status Page", () => { + + test("has the expected elements to display upload status", async ({ page, baseURL }) => { + const apiURL = baseURL.replace('8081', '8080') + const dataStream = 'dextesting'; + const route = 'testevent1'; + const expectedFileName = 'small-test-file' + const expectedSender = 'Sender123' + const expectedDataProducer = 'Producer123' + const expectedJurisdiction = 'Jurisdiction123' + + await page.goto(`/manifest?data_stream=${dataStream}&data_stream_route=${route}`); + + await page.getByLabel('Sender Id').fill(expectedSender) + await page.getByLabel('Data Producer Id').fill(expectedDataProducer) + await page.getByLabel('Jurisdiction').fill(expectedJurisdiction) + await page.getByLabel('Received Filename').fill(expectedFileName) + await page.getByRole('button', {name: /next/i }).click(); + + const fileChooserPromise = page.waitForEvent('filechooser'); + const uploadId = page.url().split('/').slice(-1)[0] + + const uploadHeadResponsePromise = page.waitForResponse(response => + response.url() === `${apiURL}/files/${uploadId}` && response.status() === 200 + && response.request().method() === 'HEAD' + ); + + const uploadPatchResponsePromise = page.waitForResponse(response => + response.url() === `${apiURL}/files/${uploadId}` && response.status() === 204 + && response.request().method() === 'PATCH' + ); + + await page.getByRole('button', {name: 'Browse Files'}).click(); + + const fileChooser = await fileChooserPromise; + await fileChooser.setFiles('../upload-files/10KB-test-file'); + + await expect((await uploadPatchResponsePromise).ok()).toBeTruthy() + await expect((await uploadHeadResponsePromise).ok()).toBeTruthy() + + await page.reload(); + + const fileHeaderContainer= page.locator('.file-header-container') + await expect(fileHeaderContainer.getByRole('heading', { level: 1 }).nth(0)).toHaveText(expectedFileName) + await expect(fileHeaderContainer.getByRole('heading', { level: 1 }).nth(1)).toHaveText("Upload Status: Complete") + await expect(fileHeaderContainer).toContainText(`ID: ${uploadId}`) + + const fileDeliveriesContainer = page.locator('.file-deliveries-container'); + await expect(fileDeliveriesContainer.getByRole('heading', { level: 2 }).nth(0)).toHaveText('Delivery Status') + await expect(fileDeliveriesContainer.getByRole('heading', { level: 2 }).nth(1)).toHaveText('EDAV') + await expect(fileDeliveriesContainer.getByRole('heading', {level: 3})).toHaveText('Delivery Status: SUCCESS') + await expect(fileDeliveriesContainer).toContainText(`Location: uploads/edav/${uploadId}`) + + const uploadDetailsContainer = page.locator('.file-details-container') + await expect(uploadDetailsContainer.getByRole('heading', { level: 2 })).toHaveText('Upload Details') + await expect(uploadDetailsContainer).toContainText(`File Size: 10 KB`) + await expect(uploadDetailsContainer).toContainText(`Sender ID: ${expectedSender}`) + await expect(uploadDetailsContainer).toContainText(`Producer ID: ${expectedDataProducer}`) + await expect(uploadDetailsContainer).toContainText(`Stream ID: ${dataStream}`) + await expect(uploadDetailsContainer).toContainText(`Stream Route: ${route}`) + await expect(uploadDetailsContainer).toContainText(`Jurisdiction: ${expectedJurisdiction}`) + }) +}) diff --git a/tests/smoke/playwright/test/upload-test.spec.ts b/tests/smoke/playwright/test/upload-test.spec.ts index 27aeaa4d7..a4fe34b7f 100644 --- a/tests/smoke/playwright/test/upload-test.spec.ts +++ b/tests/smoke/playwright/test/upload-test.spec.ts @@ -7,7 +7,7 @@ dotenv.config({ path: '../../.env' }); // Use test.describe to group your tests and use hooks like beforeAll -test.describe('File Upload and Trace Response Flow', () => { +test.describe.skip('File Upload and Trace Response Flow', () => { let uploadId; let accessToken: string; let psApiUrl: string; diff --git a/upload-server/cmd/cli/datastore.go b/upload-server/cmd/cli/datastore.go index bcc1e261d..354b1c081 100644 --- a/upload-server/cmd/cli/datastore.go +++ b/upload-server/cmd/cli/datastore.go @@ -3,11 +3,12 @@ package cli import ( "context" "fmt" - "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" - "github.com/tus/tusd/v2/pkg/s3store" "os" "path/filepath" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" + "github.com/tus/tusd/v2/pkg/s3store" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" "github.com/cdcgov/data-exchange-upload/upload-server/internal/handlertusd" "github.com/cdcgov/data-exchange-upload/upload-server/internal/health" diff --git a/upload-server/cmd/cli/info.go b/upload-server/cmd/cli/info.go index 30b03c3a1..d437bc6fd 100644 --- a/upload-server/cmd/cli/info.go +++ b/upload-server/cmd/cli/info.go @@ -5,11 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/azureinspector" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/fileinspector" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/s3inspector" - "net/http" "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" "github.com/cdcgov/data-exchange-upload/upload-server/internal/storeaz" @@ -35,11 +36,6 @@ func (ih *InfoHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } - response := &info.InfoResponse{ - Manifest: fileInfo, - Deliveries: []info.FileDeliveryStatus{}, - } - uploadedFileInfo, err := ih.inspector.InspectUploadedFile(r.Context(), id) if err != nil { // skip not found errors to handle deferred uploads. @@ -49,17 +45,30 @@ func (ih *InfoHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } } + uploadStatus, err := ih.statusInspector.InspectFileUploadStatus(r.Context(), id) + if err != nil { + // skip not found errors to handle deferred uploads. + if !errors.Is(err, info.ErrNotFound) { + http.Error(rw, fmt.Sprintf("error getting file upload status. Manifest: %#v", fileInfo), getStatusFromError(err)) + return + } + } + deliveries, err := ih.statusInspector.InspectFileDeliveryStatus(r.Context(), id) if err != nil { // skip not found errors to handle deferred uploads. if !errors.Is(err, info.ErrNotFound) { - http.Error(rw, fmt.Sprintf("error getting file status. Manifest: %#v", fileInfo), getStatusFromError(err)) + http.Error(rw, fmt.Sprintf("error getting file delivery status. Manifest: %#v", fileInfo), getStatusFromError(err)) return } } - response.FileInfo = uploadedFileInfo - response.Deliveries = deliveries + response := &info.InfoResponse{ + Manifest: fileInfo, + FileInfo: uploadedFileInfo, + UploadStatus: uploadStatus, + Deliveries: deliveries, + } rw.Header().Set("Content-Type", "application/json") enc := json.NewEncoder(rw) @@ -90,11 +99,7 @@ func createInspector(ctx context.Context, appConfig *appconfig.AppConfig) (Uploa return nil, err } - return &s3inspector.S3UploadInspector{ - Client: s3Client, - BucketName: appConfig.S3Connection.BucketName, - TusPrefix: appConfig.TusUploadPrefix, - }, nil + return s3inspector.NewS3UploadInspector(s3Client, appConfig.S3Connection.BucketName, appConfig.TusUploadPrefix), nil } if appConfig.LocalFolderUploadsTus != "" { return fileinspector.NewFileSystemUploadInspector(appConfig.LocalFolderUploadsTus, appConfig.TusUploadPrefix), nil diff --git a/upload-server/cmd/cli/status.go b/upload-server/cmd/cli/status.go index 8f3ec3c0f..014b2b3f1 100644 --- a/upload-server/cmd/cli/status.go +++ b/upload-server/cmd/cli/status.go @@ -2,9 +2,11 @@ package cli import ( "context" + "github.com/cdcgov/data-exchange-upload/upload-server/pkg/info" ) type UploadStatusInspector interface { InspectFileDeliveryStatus(ctx context.Context, id string) ([]info.FileDeliveryStatus, error) + InspectFileUploadStatus(ctx context.Context, id string) (info.FileUploadStatus, error) } diff --git a/upload-server/go.mod b/upload-server/go.mod index 804dde066..6a3b15297 100644 --- a/upload-server/go.mod +++ b/upload-server/go.mod @@ -32,6 +32,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect diff --git a/upload-server/go.sum b/upload-server/go.sum index 7508af35a..d06085d7b 100644 --- a/upload-server/go.sum +++ b/upload-server/go.sum @@ -134,6 +134,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/eventials/go-tus v0.0.0-20220610120217-05d0564bb571 h1:0i+Y7klNNqXwzAQ2qlIWeZyiMtDB/rf5fSaFzIW7lsk= diff --git a/upload-server/internal/delivery/deliver.go b/upload-server/internal/delivery/deliver.go index dea10097f..2ecad6315 100644 --- a/upload-server/internal/delivery/deliver.go +++ b/upload-server/internal/delivery/deliver.go @@ -4,9 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata" - "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" - metadataPkg "github.com/cdcgov/data-exchange-upload/upload-server/pkg/metadata" "io" "log/slog" "os" @@ -16,6 +13,10 @@ import ( "text/template" "time" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" + metadataPkg "github.com/cdcgov/data-exchange-upload/upload-server/pkg/metadata" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" "github.com/cdcgov/data-exchange-upload/upload-server/internal/health" "github.com/cdcgov/data-exchange-upload/upload-server/internal/storeaz" diff --git a/upload-server/internal/delivery/s3.go b/upload-server/internal/delivery/s3.go index 74b9e1d6b..7107f8976 100644 --- a/upload-server/internal/delivery/s3.go +++ b/upload-server/internal/delivery/s3.go @@ -3,15 +3,16 @@ package delivery import ( "context" "fmt" + "io" + "log/slog" + "strings" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" "github.com/cdcgov/data-exchange-upload/upload-server/internal/models" "github.com/cdcgov/data-exchange-upload/upload-server/internal/stores3" - "io" - "log/slog" - "strings" ) func NewS3Destination(ctx context.Context, target string, conn *appconfig.S3StorageConfig) (*S3Destination, error) { diff --git a/upload-server/internal/metadata/metadata.go b/upload-server/internal/metadata/metadata.go index 0190546fd..ff9c6fbea 100644 --- a/upload-server/internal/metadata/metadata.go +++ b/upload-server/internal/metadata/metadata.go @@ -5,6 +5,15 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" "github.com/cdcgov/data-exchange-upload/upload-server/internal/loaders" @@ -15,14 +24,6 @@ import ( "github.com/cdcgov/data-exchange-upload/upload-server/pkg/metadata" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/reports" "github.com/google/uuid" - "log/slog" - "net/http" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "time" v1 "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata/v1" v2 "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata/v2" diff --git a/upload-server/internal/stores3/client.go b/upload-server/internal/stores3/client.go index a9f68ffe9..483c47ab2 100644 --- a/upload-server/internal/stores3/client.go +++ b/upload-server/internal/stores3/client.go @@ -2,6 +2,7 @@ package stores3 import ( "context" + "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig" diff --git a/upload-server/internal/ui/assets/demo.css b/upload-server/internal/ui/assets/demo.css deleted file mode 100644 index 3c02cecae..000000000 --- a/upload-server/internal/ui/assets/demo.css +++ /dev/null @@ -1,11 +0,0 @@ -body { - padding-top: 40px; -} - -.progress { - height: 32px; -} - -a.btn { - margin-bottom: 2px; -} diff --git a/upload-server/internal/ui/assets/form.css b/upload-server/internal/ui/assets/form.css deleted file mode 100644 index fdbeda728..000000000 --- a/upload-server/internal/ui/assets/form.css +++ /dev/null @@ -1,74 +0,0 @@ -body { - font-family: Arial, Helvetica, sans-serif; - margin: 0 auto; - width: 100vw; -} - -.upload-container { - margin: 0 auto; - padding: 1.2rem; -} - -.form-container { - margin: 0; - max-width: 550px; -} - -.form-container button { - background: #0057b7; - border: none; - border-radius: 5px; - color: white; - cursor: pointer; - display: inline-block; - font-size: 16px; - margin: 20px 0 10px 0; - padding: 10px; - width: 100%; -} - -.form-container input { - background: #f1f1f1; - border: 1px solid #aaa; - border-radius: 5px; - display: inline-block; - margin: 5px 0; - padding: 10px; - width: 100%; -} - -.form-container input[type=file] { - display: block; - font-style: oblique; - margin-left: 15px; -} - -.form-container input[type=file]::file-selector-button { - display: none; -} - -.form-container label { - display: block; - font-weight: 600; - margin-top: 5px; -} - -.form-container select { - background: #f1f1f1; - border: 1px solid #aaa; - border-radius: 5px; - display: inline-block; - margin: 5px 0; - padding: 10px; - width: 100%; -} - -.form-container .upload-container { - display: flex; - flex-direction: row; -} - -.form-container .upload-container input { - background: none; - border: none; -} \ No newline at end of file diff --git a/upload-server/internal/ui/assets/index.css b/upload-server/internal/ui/assets/index.css index 0fef11e65..09b1f78e0 100644 --- a/upload-server/internal/ui/assets/index.css +++ b/upload-server/internal/ui/assets/index.css @@ -1,66 +1,222 @@ * { - box-sizing: border-box; + box-sizing: border-box; } html { - /* 1 rem = 10px */ - font-size: 62.5% + /* 1 rem = 10px */ + font-size: 62.5%; } body { font-family: Arial, Helvetica, sans-serif; font-size: 1.4rem; margin: 0 auto; + width: 100vw; + + --upload-light-text: #fff; + --upload-dark-text: #000; + --upload-secondary-bg: #f4f4f4; + --upload-border-color: #aaa; + + --upload-accent: rgb(0, 94, 201); + --upload-accent-alpha: rgba(0, 119, 255, 0.1); + --upload-active: rgb(243, 186, 17); + + --upload-success: rgb(2, 119, 2); + --upload-success-alpha: rgba(6, 162, 6, 0.05); + --upload-failure: rgb(155, 0, 0); + --upload-failure-alpha: rgba(155, 0, 0, 0.05); + --upload-warning: rgb(187, 77, 0); +} + +main, +nav { + margin: 0 auto; + max-width: 750px; +} + +nav { + display: flex; + justify-content: space-between; } a.skip-main { - left:-999px; - position:absolute; - top:auto; - width:1px; - height:1px; - overflow:hidden; - z-index:-999; -} -a.skip-main:focus, a.skip-main:active { - color: #fff; - background-color:#000; - left: auto; - top: auto; - width: 30%; - height: auto; - overflow:auto; - margin: 10px 35%; - padding:5px; - border-radius: 15px; - border:4px solid yellow; - text-align:center; - font-size:1.2em; - z-index:999; + left: -999px; + position: absolute; + top: auto; + width: 1px; + height: 1px; + overflow: hidden; + z-index: -999; +} + +a.skip-main:focus, +a.skip-main:active { + color: var(--upload-light-text); + background-color: var(--upload-dark-text); + left: auto; + top: auto; + width: 30%; + height: auto; + overflow: auto; + margin: 10px 35%; + padding: 5px; + border-radius: 15px; + border: 4px solid var(--upload-active); + text-align: center; + font-size: 1.2em; + z-index: 999; +} + +button { + background: var(--upload-accent); + border: none; + border-radius: 5px; + color: var(--upload-light-text); + cursor: pointer; + display: inline-block; + margin: 5px 0; + font-size: 16px; + padding: 10px; + width: 100%; } .hidden { - display: none; + display: none; } +.small-italics { + font-size: small; + font-style: italic; +} + +/* -------------- */ +/* Navbar Styling */ +/* -------------- */ + .nav-container { - background-color: #f4f4f4; - border-bottom: 1px solid #c5c5c5; - font-size: 1.6rem; - font-weight: 700; - padding: 1rem; + background-color: var(--upload-secondary-bg); + border-bottom: 1px solid var(--upload-border-color); + font-size: 1.6rem; + font-weight: 700; + padding: 1rem; } -.nav-container a, a:visited { - color: #000; - text-decoration: none; +.nav-container a, +a:visited { + color: var(--upload-dark-text); + text-decoration: none; + border-bottom: 1px solid var(--upload-secondary-bg); } .nav-container a:hover { - color: #000; - border-bottom: 1px solid #000; + color: var(--upload-dark-text); + border-bottom: 1px solid var(--upload-border-color); +} + +.nav-container .link-button:hover { + border-bottom: 0; } .nav-container img { - width: 7.5rem !important; -} \ No newline at end of file + width: 7.5rem !important; +} + +/* ------------------- */ +/* Link Button Styling */ +/* ------------------- */ + +.link-button { + color: var(--upload-accent) !important; + font-weight: 700; + text-decoration: none; +} + +.link-button-container { + background-color: var(--upload-accent-alpha); + border: 2px solid var(--upload-accent); + border-radius: 5px; + margin: auto 0; + padding: 0.5rem 1rem; + text-align: center; +} + +.link-button-container:hover { + background-color: var(--upload-accent); + color: var(--upload-light-text); + cursor: pointer; +} + +/* ------------ */ +/* Form Styling */ +/* ------------ */ + +.upload-container { + margin: 0 auto; + padding: 1.2rem; +} + +.form-container { + margin: 0; + max-width: 550px; +} + +.form-container .submit-button { + margin: 20px 0 10px 0; +} + +.form-container input { + background: var(--upload-secondary-bg); + border: 1px solid var(--upload-border-color); + border-radius: 5px; + display: inline-block; + margin: 5px 0; + padding: 10px; + width: 100%; +} + +.form-container input[type="file"] { + display: block; + font-style: oblique; + margin-left: 15px; +} + +.form-container input[type="file"]::file-selector-button { + display: none; +} + +.form-container label, +.form-container .label { + display: block; + font-weight: 600; + margin-top: 1rem; +} + +.form-container label .input-tip, +.form-container .label .input-tip { + font-weight: normal; + font-size: small; + font-style: italic; +} + +.form-container select { + background: var(--upload-secondary-bg); + border: 1px solid var(--upload-border-color); + border-radius: 5px; + display: inline-block; + margin: 5px 0; + padding: 10px; + width: 100%; +} + +.form-container .file-container { + display: flex; + flex-direction: row; + align-items: center; + margin-top: 0.5rem; +} + +.form-container .file-container input { + background: none; + border: none; +} diff --git a/upload-server/internal/ui/assets/progress.css b/upload-server/internal/ui/assets/progress.css index 2f9c8defb..c9af8feff 100644 --- a/upload-server/internal/ui/assets/progress.css +++ b/upload-server/internal/ui/assets/progress.css @@ -1,20 +1,20 @@ .progress-container { - height: 32px; + height: 32px; width: 50%; max-width: 400px; } .progress { - border: 1px solid #aaa; - height: 100%; + border: 1px solid var(--upload-border-color); + height: 100%; width: 100%; } .bar { - background-color: rgb(6, 162, 6); - color: white; + background-color: var(--upload-success); + color: var(--upload-light-text); line-height: 30px; height: 100%; margin: auto 0; padding-left: 10px; -} \ No newline at end of file +} diff --git a/upload-server/internal/ui/assets/tusclient.js b/upload-server/internal/ui/assets/tusclient.js new file mode 100644 index 000000000..cbef7861b --- /dev/null +++ b/upload-server/internal/ui/assets/tusclient.js @@ -0,0 +1,443 @@ +// ------------------------------------------ +// Variables +// ------------------------------------------ + +let upload = null; +let previousUpload = null; +let uploadIsRunning = false; +let file = null; + +const fileInput = document.querySelector("input[type=file]"); +const pauseButton = document.querySelector("#pause-upload-button"); +const resumeButton = document.querySelector("#resume-upload-button"); + +const progressContainer = document.querySelector(".progress"); +const progressBar = progressContainer.querySelector(".bar"); + +// Values also set in upload-server/pkg/info/info.go +// these values must match +const UPLOAD_INITIATED = "Initiated"; +const UPLOAD_IN_PROGRESS = "In Progress"; +const UPLOAD_COMPLETE = "Complete"; + +const UPLOAD_STATUS_LABEL_INITIALIZED = " Upload Initialized At: "; +const UPLOAD_STATUS_LABEL_IN_PROGRESS = " Last Chunk Received At: "; +const UPLOAD_STATUS_LABEL_COMPLETE = " Upload Completed At: "; +const UPLOAD_STATUS_LABEL_DEFAULT = " Uploaded At: "; + +// ------------------------------------------ +// Functions +// ------------------------------------------ + +// Hides or shows an element +function _toggleVisibility(element, show) { + if (show) { + element.classList.remove("hidden"); + } else { + element.classList.add("hidden"); + } +} + +// Hides or shows the progress bar +function _toggleProgressBar(show) { + const uploaderContainer = document.querySelector(".uploader-container"); + _toggleVisibility(uploaderContainer, show); + _toggleVisibility(progressContainer, show); + _toggleVisibility(progressBar, show); + + const pauseResumeContainer = document.querySelector( + ".pause-resume-upload-container" + ); + _toggleVisibility(pauseResumeContainer, show); + if (show) { + _togglePauseButton(show); + } else { + _toggleVisibility(pauseButton, false); + _toggleVisibility(resumeButton, false); + } +} + +function _togglePauseButton(pause) { + if (pause) { + _toggleVisibility(pauseButton, true); + _toggleVisibility(resumeButton, false); + } else { + _toggleVisibility(pauseButton, false); + _toggleVisibility(resumeButton, true); + } +} + +// Hides or shows the upload forms and the progress bar +function _toggleUploadContainer(show) { + const uploadContainer = document.querySelector(".upload-container"); + _toggleVisibility(uploadContainer, show); +} + +// Hides or shows the full upload form +function _toggleNewUploadForm(show) { + const matches = document.querySelectorAll(".new-upload"); + for (const element of matches) { + _toggleVisibility(element, show); + } +} + +// Hides or shows the resume upload form +function _toggleResumeUploadForm(show) { + const matches = document.querySelectorAll(".resume-upload"); + for (const element of matches) { + _toggleVisibility(element, show); + } +} + +function _toggleFormContainer(show) { + const formContainer = document.querySelector(".form-container"); + _toggleVisibility(formContainer, show); +} + +// Hides or shows the file info +function _toggleInfoContainer(show) { + const infoContainer = document.querySelector(".file-detail-container"); + _toggleVisibility(infoContainer, show); +} + +function _toggleNewUploadButtonContainer(show) { + const buttonContainer = document.querySelector( + ".new-upload-button-container" + ); + _toggleVisibility(buttonContainer, show); +} + +// Sets the view up as the initial upload form +function _showInitiatedUploadForm() { + _toggleUploadContainer(true); + _toggleNewUploadForm(true); + _toggleResumeUploadForm(false); + _toggleProgressBar(false); + _toggleInfoContainer(false); + _toggleNewUploadButtonContainer(false); +} + +// Set the view up as the resume upload form +function _showResumableUploadForm() { + _toggleUploadContainer(true); + _toggleNewUploadForm(false); + _toggleResumeUploadForm(true); + _toggleProgressBar(false); + _toggleInfoContainer(true); + _toggleNewUploadButtonContainer(false); +} + +// Sets the view up to only show the file info +function _showReadOnlyFileInfo(statusLabel) { + _toggleUploadContainer(false); + _toggleInfoContainer(true); + _toggleNewUploadButtonContainer(true); + _setUploadLastChunkReceivedLabel(statusLabel); +} + +function _updateUploadStatusInProgress() { + const statusValue = document.querySelector("#upload-status-value"); + statusValue.innerHTML = "In Progress"; + statusValue.classList.remove("upload-initiated"); + statusValue.classList.remove("upload-complete"); + statusValue.classList.add("upload-in-progress"); + _setUploadLastChunkReceivedLabel(UPLOAD_STATUS_LABEL_IN_PROGRESS); +} + +function _setUploadLastChunkReceivedLabel(text) { + document.querySelector("#upload-datetime-label").innerHTML = text; +} + +function _updateLastChunkReceived() { + const currTime = new Date(); + document.querySelector("#upload-datetime-value").innerHTML = + currTime.toUTCString(); +} + +function _refreshPage() { + // Adding a 1 second wait to make the refresh look a little smoother + // when the file uploads really quickly + setTimeout(() => { + location.reload(); + }, 1000); +} + +// Triggered by a file being selected. Gets the file from +// the file input. Gets the other values from the form, if +// this is a new upload, or from the previous upload metadata, +// if this is a resumed upload. Then sends those values to the +// uploadFile function to start the upload. +async function submitUploadForm() { + const fileList = Array.from(fileInput.files); //[0] + // Only continue if a file has actually been selected. + // IE will trigger a change event even if we reset the input element + // using reset() and we do not want to blow up later. + + if (!fileList || fileList.length < 1) { + return; + } + + // Retrieving the first file because this is only handling uploading + // one file at a time + file = fileList[0]; + + let endpoint; + let chunkSize; + let parallelUploads; + if (previousUpload) { + // If there is a previous upload in local storage + // retrieve the previously entered values for that upload + const { metadata } = previousUpload; + if (!metadata) { + return; + } + + // check the file to make sure it matches the previous file + const { filename, fileType, fileSize, fileLastModified } = metadata; + if ( + file.name != filename || + file.type != fileType || + fileSize != file.size || + fileLastModified != file.lastModified + ) { + fileInput.value = ""; + // if it doesn't match the expected file, alert the user that they should try again + window.alert( + `This file does not match the previously partially uploaded file.\nPlease try with another file.` + ); + return; + } + ({ endpoint, chunkSize, parallelUploads } = metadata); + } else { + // retrieve the values entered in the form + const endpointInput = document.querySelector("#endpoint"); + endpoint = endpointInput.value; + + const chunkInput = document.querySelector("#chunksize"); + chunkSize = parseInt(chunkInput.value, 10); + if (Number.isNaN(chunkSize)) { + chunkSize = Infinity; + } + + // currently this is disabled as only parallelUploads = 1 works + // multiple uploads can be started concurrent ( new tus.Upload ), however each one sends chunks serially to the server + // tusd azure does not support chunks concatenation, ref: https://github.com/tus/tusd/issues/843 + parallelUploads = 1; + } + + // hide the forms + _toggleFormContainer(false); + + // Upload the file + await uploadFile(file, { endpoint, chunkSize, parallelUploads }); +} + +// Creates the tus client and uploads the file. +// Handles onProgress, onSuccess, and onError. +// Will resume an upload if one has already been started. +async function uploadFile(file, { endpoint, chunkSize, parallelUploads }) { + console.log(`start uploading file: ${file.name}`); + + // used to determine the duration + const startTimeUpload = new Date().getTime(); + // because uploadLengthDeferred is true, tus will not know how large + // the file is until the upload is complete, if we want to show the + // upload progress we need to set the value outside of tus + const fileSize = file.size; + + console.log("starting upload to endpoint: ", endpoint); + const options = { + headers: { + "Tus-Resumable": "1.0.0", + }, + metadata: { + filename: file.name, + fileType: file.type, + fileSize: file.size, + fileLastModified: file.lastModified, + endpoint, + chunkSize, + parallelUploads, + }, + protocol: "ietf-draft-03", + uploadLengthDeferred: true, + retryDelays: [0, 1000, 3000, 5000], + removeFingerprintOnSuccess: true, + endpoint, + uploadUrl, + chunkSize, + parallelUploads, + onError(error) { + if (error.originalRequest) { + // if the upload failed but is recoverable, ask if the user wants to retry + if (window.confirm(`Failed because: ${error}\nDo you want to retry?`)) { + upload.start(); + uploadIsRunning = true; + return; + } + } else { + // if upload is not recoverable, just show the error + window.alert(`Failed because: ${error}`); + } + + console.log(`file: ${file.name} upload failed: ${error}`); + + // Set running flag to false and reset the file input + uploadIsRunning = false; + fileInput.value = ""; + + // display the file info + _toggleInfoContainer(true); + }, + onProgress(bytesUploaded, bytesTotal) { + // because uploadLengthDeferred is true, the bytesTotal should always + // be 0, if it is 0, use the fileSize set above + const total = bytesTotal || fileSize; + + // the percent of the file that has been uploaded + const percentageTotal = ((bytesUploaded / total) * 100).toFixed(2); + const percentageTotalRound = Math.round(percentageTotal); + + // update the tool bar to show these new values + // only show the progress bar while there is progress + if (percentageTotal >= 0 && percentageTotal <= 100) { + _toggleProgressBar(true); + progressBar.style.width = `${percentageTotal}%`; + progressBar.textContent = `${percentageTotalRound}%`; + } else { + _toggleProgressBar(false); + progressBar.style.width = 0; + } + + console.log( + `uploadedBytes: ${bytesUploaded}, totalBytes: ${total}, percentComplete: ${percentageTotal}` + ); + + _updateLastChunkReceived(); + }, + onSuccess() { + console.log(`file: ${file.name} uploaded successfully`); + + // get the duration of the upload + const durationUpload = new Date().getTime() - startTimeUpload; + + console.log( + `total upload duration [ms]: ${durationUpload}, [s]: ${ + durationUpload / 1000 + }` + ); + + // reset all of the values no longer needed + upload = null; + previousUpload = null; + uploadIsRunning = false; + file = null; + + fileInput.value = ""; + + // refresh the page for the complete upload info + _refreshPage(); + }, + }; + + // create the tus client + upload = new tus.Upload(file, options); + + if (previousUpload) { + console.log(`Resuming the upload of ${file.name}`); + // if there is a previous upload for this file + // set it here so that it will resume + upload.resumeFromPreviousUpload(previousUpload); + } + + // Start the upload + uploadIsRunning = true; + upload.start(); + + _updateUploadStatusInProgress(); + _toggleInfoContainer(true); +} + +function resumeUpload() { + if (!upload) { + console.log(`No upload client, please try uploading again`); + return; + } + + // Check if there are any previous uploads to continue. + upload.findPreviousUploads().then(function (previousUploads) { + // Found previous uploads so we select the first one. + if (previousUploads.length) { + previousUpload = previousUploads[0]; + upload.resumeFromPreviousUpload(previousUploads[0]); + } + + // Start the upload + upload.start(); + _togglePauseButton(true); + }); +} + +function pauseUpload() { + if (!upload) { + console.log(`No upload client, please try uploading again`); + return; + } + + upload.abort(); + _togglePauseButton(false); +} + +// checks the local storage to see if there is a previous upload +// that was not completed and has the same uploadUrl +async function findResumableUpload() { + let uploads = await tus.defaultOptions.urlStorage.findAllUploads(); + + for (const upload of uploads) { + if (upload.uploadUrl == uploadUrl) { + return upload; + } + } + + return null; +} + +(async () => { + // initializes what is hidden/shown on the page base on + // the uploadStatus and upload data in the local storage + let isHost = false; + + switch (uploadStatus) { + case UPLOAD_INITIATED: + isHost = true; + _showInitiatedUploadForm(); + break; + case UPLOAD_IN_PROGRESS: + previousUpload = await findResumableUpload(); + if (previousUpload) { + isHost = true; + _showResumableUploadForm(); + } else { + _showReadOnlyFileInfo(UPLOAD_STATUS_LABEL_IN_PROGRESS); + } + break; + case UPLOAD_COMPLETE: + _showReadOnlyFileInfo(UPLOAD_STATUS_LABEL_COMPLETE); + break; + default: + console.error(`${uploadStatus} is an invalid status`); + _showReadOnlyFileInfo(UPLOAD_STATUS_LABEL_DEFAULT); + } + + if (isHost) { + // we only need to use tus and add the listener if we are the host + // otherwise the forms will not be displayed + if (!tus.isSupported) { + document.querySelector("#support-alert").classList.remove("hidden"); + } // .if + + fileInput.addEventListener("change", () => submitUploadForm()); + pauseButton.addEventListener("click", () => pauseUpload()); + resumeButton.addEventListener("click", () => resumeUpload()); + } +})(); diff --git a/upload-server/internal/ui/assets/upload.css b/upload-server/internal/ui/assets/upload.css new file mode 100644 index 000000000..b3b1c4483 --- /dev/null +++ b/upload-server/internal/ui/assets/upload.css @@ -0,0 +1,155 @@ +.file-detail-container { + margin: 0 auto; + padding: 1.2rem; +} + +main > div:first-child:not(.hidden) + .file-detail-container { + border-top: 1px solid var(--upload-border-color); +} + +.uploader-container { + display: flex; + flex-direction: row; + justify-content: start; + align-items: center; + gap: 2rem; +} + +.uploader-container .progress-container { + flex-grow: 1; +} +.uploader-container .pause-resume-upload-container { + flex-grow: 1; +} + +/* -------------------------- */ +/* File ID and Status Styling */ +/* -------------------------- */ + +.file-header-container, +.file-deliveries-container, +.file-details-container { + margin-bottom: 4rem; +} + +.file-header-container, +.delivery-header-container { + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: start; +} + +.file-header-container .file-id-container, +.file-header-container .upload-status-container, +.delivery-header-container.delivery-id-container, +.delivery-header-container .delivery-status-container { + display: flex; + flex-direction: column; + justify-content: start; + align-items: start; +} + +.file-header-container .upload-status-container, +.delivery-header-container .delivery-status-container { + align-items: end; +} + +.file-header-container .file-id-container h1, +.file-header-container .upload-status-container h1 { + margin-bottom: 0.5rem; +} + +.upload-complete, +.delivery-success { + color: var(--upload-success); +} + +.upload-in-progress { + color: var(--upload-active); +} + +.upload-failure, +.delivery-failure { + color: var(--upload-failure); +} + +/* ----------------------- */ +/* File Deliveries Styling */ +/* ----------------------- */ + +.file-delivery-container { + border: 3px solid var(--upload-border-color); + border-radius: 5px; + margin-bottom: 2rem; + padding: 1rem 2rem; + align-items: start; +} + +.file-delivery-success { + border-color: var(--upload-success); + background-color: var(--upload-success-alpha); +} + +.file-delivery-failure { + border-color: var(--upload-failure); + background-color: var(--upload-failure-alpha); +} + +.delivery-status-container h3 { + margin-bottom: 0.25rem; +} + +.delivery-location-container { + padding: 3rem 0; + font-size: larger; + font-weight: 500; + width: 100%; + text-align: center; +} + +/* ----------------------- */ +/* Issues and Info Styling */ +/* ----------------------- */ + +.issue-container, +.detail-container { + line-height: 2rem; + display: flex; + flex-direction: row; + margin: 0.5rem 0; + padding: 0.5rem; + align-items: start; +} + +.issue-container:not(:last-child), +.detail-container:not(:last-child) { + border-bottom: 1px solid var(--upload-border-color); +} + +.issue-container .issue-level { + font-weight: 600; + width: 10rem; +} + +.issue-container .issue-level-error { + color: var(--upload-failure); +} + +.issue-container .issue-level-warning { + color: var(--upload-warning); +} + +.detail-container .detail-label { + font-weight: bold; + min-width: 18rem; +} + +.detail-container .detail-value { + word-break: break-all; +} + +.file-size-initiated, +.file-size-in-progress { + display: none; +} diff --git a/upload-server/internal/ui/components/navbar.go b/upload-server/internal/ui/components/navbar.go index 79b3c2a66..227a0e039 100644 --- a/upload-server/internal/ui/components/navbar.go +++ b/upload-server/internal/ui/components/navbar.go @@ -1,3 +1,13 @@ package components -type Navbar struct{} +type Navbar struct { + ShouldShowActions bool + NewUploadBtn NewUploadBtn +} + +func NewNavbar(ShouldShowActions bool) Navbar { + return Navbar{ + ShouldShowActions: ShouldShowActions, + NewUploadBtn: NewUploadBtn{}, + } +} diff --git a/upload-server/internal/ui/components/navbar.html b/upload-server/internal/ui/components/navbar.html index 7257d0c7c..fc1dc326a 100644 --- a/upload-server/internal/ui/components/navbar.html +++ b/upload-server/internal/ui/components/navbar.html @@ -1,10 +1,13 @@ {{define "navbar"}} - -{{end}} \ No newline at end of file +
+ Skip to main content +
+ +
+
+{{end}} diff --git a/upload-server/internal/ui/components/newuploadbtn.go b/upload-server/internal/ui/components/newuploadbtn.go new file mode 100644 index 000000000..064256510 --- /dev/null +++ b/upload-server/internal/ui/components/newuploadbtn.go @@ -0,0 +1,3 @@ +package components + +type NewUploadBtn struct{} diff --git a/upload-server/internal/ui/components/newuploadbtn.html b/upload-server/internal/ui/components/newuploadbtn.html new file mode 100644 index 000000000..b41e0b6e4 --- /dev/null +++ b/upload-server/internal/ui/components/newuploadbtn.html @@ -0,0 +1,5 @@ +{{define "newuploadbtn"}} + +
Upload New File
+ +{{end}} diff --git a/upload-server/internal/ui/index.html b/upload-server/internal/ui/index.html index fc5308211..88b16f6e3 100644 --- a/upload-server/internal/ui/index.html +++ b/upload-server/internal/ui/index.html @@ -1,10 +1,10 @@ - - - + + + DEX Upload @@ -28,8 +28,8 @@

Start the upload process by entering a data stream and route.

required /> -
- +
+
diff --git a/upload-server/internal/ui/manifest.tmpl b/upload-server/internal/ui/manifest.tmpl index afaca3208..8abd60eca 100644 --- a/upload-server/internal/ui/manifest.tmpl +++ b/upload-server/internal/ui/manifest.tmpl @@ -1,55 +1,53 @@ - - - - - - - Manifest - - - {{template "navbar" .Navbar}} -
-
-

Please fill in the sender manifest form for your file

-

- * indicates a required field -

-
-
- {{range .MetadataFields}} -
+ + + + + + Manifest + + + {{template "navbar" .Navbar}} +
+

Please fill in the sender manifest form for your file

+

* indicates a required field

+
+ + {{range .MetadataFields}} +
{{if not .AllowedValues}} {{ if .Required }} - + {{ else }} - + {{ end }} {{ else }} - + {{ end }}
- {{end}} - - + {{end}} + + +
- -
-
- +
+ +
+
+ diff --git a/upload-server/internal/ui/ui.go b/upload-server/internal/ui/ui.go index 26438c263..41e43b088 100644 --- a/upload-server/internal/ui/ui.go +++ b/upload-server/internal/ui/ui.go @@ -3,19 +3,26 @@ package ui import ( "context" "embed" + "encoding/json" "fmt" + "io" "net/http" "net/url" "path/filepath" "strings" + "time" "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata/validation" "github.com/cdcgov/data-exchange-upload/upload-server/internal/ui/components" + "github.com/cdcgov/data-exchange-upload/upload-server/pkg/info" "html/template" "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata" + "github.com/dustin/go-humanize" "github.com/eventials/go-tus" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) // content holds our static web server content. @@ -25,17 +32,56 @@ var content embed.FS func FixNames(name string) string { removeChars := strings.ReplaceAll(name, "_", " ") - newName := strings.Title(strings.ToLower(removeChars)) + + caser := cases.Title(language.English) + newName := caser.String(removeChars) + return newName } +func KebabCase(value string) string { + kebabValue := strings.ReplaceAll(value, " ", "-") + return strings.ToLower(kebabValue) +} + +func FormatDateTime(dateTimeString string) string { + date, err := time.Parse(time.RFC3339, dateTimeString) + + if err != nil { + fmt.Println(err) + return "" + } + + return date.UTC().Format(time.RFC850) +} + +func FormatBytes(bytes float64) string { + intBytes := uint64(bytes) + strBytes := humanize.Bytes(intBytes) + + return strings.ToUpper(strBytes) +} + var usefulFuncs = template.FuncMap{ - "FixNames": FixNames, + "FixNames": FixNames, + "AllLowerCase": strings.ToLower, + "AllUpperCase": strings.ToUpper, + "FormatDateTime": FormatDateTime, + "KebabCase": KebabCase, + "FormatBytes": FormatBytes, +} + +func generateTemplate(templatePath string, useFuncs bool) *template.Template { + var templatePaths = []string{templatePath, "components/navbar.html", "components/newuploadbtn.html"} + if useFuncs { + return template.Must(template.New(templatePath).Funcs(usefulFuncs).ParseFS(content, templatePaths...)) + } + return template.Must(template.ParseFS(content, templatePaths...)) } -var indexTemplate = template.Must(template.ParseFS(content, "index.html", "components/navbar.html")) -var manifestTemplate = template.Must(template.New("manifest.tmpl").Funcs(usefulFuncs).ParseFS(content, "manifest.tmpl", "components/navbar.html")) -var uploadTemplate = template.Must(template.ParseFS(content, "upload.tmpl", "components/navbar.html")) +var indexTemplate = generateTemplate("index.html", false) +var manifestTemplate = generateTemplate("manifest.tmpl", true) +var uploadTemplate = generateTemplate("upload.tmpl", true) type ManifestTemplateData struct { DataStream string @@ -45,8 +91,11 @@ type ManifestTemplateData struct { } type UploadTemplateData struct { - UploadURL string - Navbar components.Navbar + UploadUrl string + UploadStatus string + Info info.InfoResponse + Navbar components.Navbar + NewUploadBtn components.NewUploadBtn } var StaticHandler = http.FileServer(http.FS(content)) @@ -75,7 +124,7 @@ func GetRouter(uploadUrl string, infoUrl string) *http.ServeMux { DataStream: dataStream, DataStreamRoute: dataStreamRoute, MetadataFields: filterMetadataFields(config), - Navbar: components.Navbar{}, + Navbar: components.NewNavbar(false), }) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) @@ -155,14 +204,23 @@ func GetRouter(uploadUrl string, infoUrl string) *http.ServeMux { http.Redirect(rw, r, "/", http.StatusFound) return } + + // Get the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + if resp.StatusCode != http.StatusOK { - var respMsg []byte - _, err := resp.Body.Read(respMsg) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - http.Error(rw, string(respMsg), resp.StatusCode) + http.Error(rw, string(body), resp.StatusCode) + return + } + + var fileInfo info.InfoResponse + err = json.Unmarshal(body, &fileInfo) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) return } @@ -173,8 +231,11 @@ func GetRouter(uploadUrl string, infoUrl string) *http.ServeMux { } err = uploadTemplate.Execute(rw, &UploadTemplateData{ - UploadURL: uploadUrl, - Navbar: components.Navbar{}, + UploadUrl: uploadUrl, + Info: fileInfo, + UploadStatus: fileInfo.UploadStatus.Status, + Navbar: components.NewNavbar(true), + NewUploadBtn: components.NewUploadBtn{}, }) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) diff --git a/upload-server/internal/ui/upload.tmpl b/upload-server/internal/ui/upload.tmpl index bba019b04..4d0a33b70 100644 --- a/upload-server/internal/ui/upload.tmpl +++ b/upload-server/internal/ui/upload.tmpl @@ -1,95 +1,196 @@ - - - - - - - - DEX Upload - - - {{template "navbar" .Navbar}} -
-
-

Demo File Uploader

-

- This demo purpose is to upload files to the custom tusd go server. -

-

- You can select a file using the controls below. -

-

Note: Chunksize should be set on the client for uploading files of large size (1GB or over).

- + + + + + + + + DEX Upload + + + {{template "navbar" .Navbar}} +
+ -
-
- - - - - -
-
- - -
-
- - -
-
- -
-
+
- + {{end}} +
+ {{end}} + + {{end}} + +
+

Upload Details

+
+
File Size:
+
{{FormatBytes .Info.FileInfo.size_bytes}}
+
+
+
Sender ID:
+
{{.Info.Manifest.sender_id}}
+
+
+
Producer ID:
+
{{.Info.Manifest.data_producer_id}}
+
+
+
Stream ID:
+
{{.Info.Manifest.data_stream_id}}
+
+
+
Stream Route:
+
{{.Info.Manifest.data_stream_route}}
+
+
+
Jurisdiction:
+
{{.Info.Manifest.jurisdiction}}
+
+
+ +
+ {{template "newuploadbtn" .NewUploadBtn}} +
+
+ - - - + + + diff --git a/upload-server/pkg/fileinspector/status.go b/upload-server/pkg/fileinspector/status.go index 4ca429dba..ce183e971 100644 --- a/upload-server/pkg/fileinspector/status.go +++ b/upload-server/pkg/fileinspector/status.go @@ -5,11 +5,13 @@ import ( "context" "encoding/json" "errors" + "os" + "path/filepath" + "time" + "github.com/cdcgov/data-exchange-upload/upload-server/internal/event" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/info" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/reports" - "os" - "path/filepath" ) type FileSystemUploadStatusInspector struct { @@ -21,6 +23,7 @@ func (fsusi *FileSystemUploadStatusInspector) InspectFileDeliveryStatus(_ contex deliveries := []info.FileDeliveryStatus{} deliveryReportFilename := filepath.Join(fsusi.ReportsDir, id+event.TypeSeparator+reports.StageFileCopy) f, err := os.Open(deliveryReportFilename) + defer f.Close() if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, errors.Join(err, info.ErrNotFound) @@ -48,19 +51,82 @@ func (fsusi *FileSystemUploadStatusInspector) InspectFileDeliveryStatus(_ contex return deliveries, err } - issues := []string{} - for _, issue := range report.StageInfo.Issues { - issues = append(issues, issue.String()) - } - deliveries = append(deliveries, info.FileDeliveryStatus{ Status: report.StageInfo.Status, Name: content.DestinationName, Location: content.FileDestinationBlobUrl, DeliveredAt: report.StageInfo.EndProcessTime, - Issues: issues, + Issues: report.StageInfo.Issues, }) } return deliveries, nil } + +func (fsusi *FileSystemUploadStatusInspector) InspectFileUploadStatus(ctx context.Context, id string) (info.FileUploadStatus, error) { + // check if the upload-completed file exists + uploadCompletedReportFilename := filepath.Join(fsusi.ReportsDir, id+event.TypeSeparator+reports.StageUploadCompleted) + uploadCompletedFileInfo, err := os.Stat(uploadCompletedReportFilename) + if err == nil { + // if there are no errors, then the upload-complete file has been created + // we can stop here and set the status to Complete and set the last chunk received + // to the time the file was created, no other calculations are needed + lastChunkReceived := uploadCompletedFileInfo.ModTime().UTC().Format(time.RFC3339) + return info.FileUploadStatus{ + Status: info.UploadComplete, + LastChunkReceived: lastChunkReceived, + }, nil + } + + // if the error from checking the upload-completed file is something other than + // the file not existing, return the error + if !errors.Is(err, os.ErrNotExist) { + return info.FileUploadStatus{}, err + } + + // get the file info for the upload-started file + uploadStartedReportFilename := filepath.Join(fsusi.ReportsDir, id+event.TypeSeparator+reports.StageUploadStarted) + uploadStartedFileInfo, err := os.Stat(uploadStartedReportFilename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return info.FileUploadStatus{}, errors.Join(err, info.ErrNotFound) + } + return info.FileUploadStatus{}, err + } + + // get the file info for the upload-status file + uploadStatusReportFilename := filepath.Join(fsusi.ReportsDir, id+event.TypeSeparator+reports.StageUploadStatus) + uploadStatusReportFileInfo, err := os.Stat(uploadStatusReportFilename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return info.FileUploadStatus{}, errors.Join(err, info.ErrNotFound) + } + return info.FileUploadStatus{}, err + } + + uploadStartedModTime := uploadStartedFileInfo.ModTime() + uploadStatusModTime := uploadStatusReportFileInfo.ModTime() + + lastChunkReceived := uploadStatusModTime.UTC().Format(time.RFC3339) + + if uploadStartedModTime.Unix() == uploadStatusModTime.Unix() { + // when the file upload is initiated the upload-started and upload-status reports + // are created at the same time, so if the file modified times are still equal + // it means the file hasn't started uploading yet + return info.FileUploadStatus{ + Status: info.UploadInitiated, + LastChunkReceived: lastChunkReceived, + }, nil + } + + if uploadStartedModTime.Unix() < uploadStatusModTime.Unix() { + // if the file modified time of the upload-status report is later than the + // upload-started report, then the file is being uploaded + return info.FileUploadStatus{ + Status: info.UploadInProgress, + LastChunkReceived: lastChunkReceived, + }, nil + } + + return info.FileUploadStatus{}, errors.New("Unable to determine the status of the upload") +} diff --git a/upload-server/pkg/info/info.go b/upload-server/pkg/info/info.go index 1f9ea0bd3..a9469ece5 100644 --- a/upload-server/pkg/info/info.go +++ b/upload-server/pkg/info/info.go @@ -1,25 +1,43 @@ package info -import "errors" +import ( + "errors" + + "github.com/cdcgov/data-exchange-upload/upload-server/pkg/reports" +) var ( ErrNotFound = errors.New("expected file not found") ) +// Values also set in the tusclient.js +// these values must match +const ( + UploadInitiated string = "Initiated" + UploadInProgress string = "In Progress" + UploadComplete string = "Complete" +) + type InfoResponse struct { - Manifest map[string]any `json:"manifest"` - FileInfo map[string]any `json:"file_info"` - Deliveries []FileDeliveryStatus `json:"deliveries"` + Manifest map[string]any `json:"manifest"` + FileInfo map[string]any `json:"file_info"` + UploadStatus FileUploadStatus `json:"upload_status"` + Deliveries []FileDeliveryStatus `json:"deliveries"` } type InfoFileData struct { MetaData map[string]any `json:"MetaData"` } +type FileUploadStatus struct { + Status string `json:"status"` + LastChunkReceived string `json:"chunk_received_at"` +} + type FileDeliveryStatus struct { - Status string `json:"status"` - Name string `json:"name"` - Location string `json:"location"` - DeliveredAt string `json:"delivered_at"` - Issues []string `json:"issues"` + Status string `json:"status"` + Name string `json:"name"` + Location string `json:"location"` + DeliveredAt string `json:"delivered_at"` + Issues []reports.ReportIssue `json:"issues"` } diff --git a/upload-server/pkg/s3inspector/s3.go b/upload-server/pkg/s3inspector/s3.go index f9e3f8c2c..3f18b0554 100644 --- a/upload-server/pkg/s3inspector/s3.go +++ b/upload-server/pkg/s3inspector/s3.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "errors" + "io" + "strings" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/cdcgov/data-exchange-upload/upload-server/pkg/info" - "io" - "strings" ) type S3UploadInspector struct { @@ -17,6 +18,14 @@ type S3UploadInspector struct { TusPrefix string } +func NewS3UploadInspector(containerClient *s3.Client, bucketName string, tusPrefix string) *S3UploadInspector { + return &S3UploadInspector{ + Client: containerClient, + BucketName: bucketName, + TusPrefix: tusPrefix, + } +} + func (sui *S3UploadInspector) InspectInfoFile(c context.Context, id string) (map[string]any, error) { // temp solution for handling hash that tus s3 store puts on upload IDs. See https://github.com/tus/tusd/pull/1167 id = strings.Split(id, "+")[0]