From 05ed8d36185ef9dd323a9b5224f57a36e84a6cc6 Mon Sep 17 00:00:00 2001 From: Simen Fivelstad Smaaberg <66635118+simensma-fresh@users.noreply.github.com> Date: Fri, 12 Jan 2024 01:54:52 +0000 Subject: [PATCH 1/5] Fixed errors when uploading files --- .../src/components/forms/RenderFileUpload.tsx | 150 ++++++++++++++---- services/common/src/redux/customAxios.js | 37 +++-- .../common/src/utils/fileUploadHelper.spec.ts | 63 +++++++- services/common/src/utils/fileUploadHelper.ts | 24 ++- .../document-manager/backend/app/__init__.py | 6 +- .../ProjectSummaryFileUpload.tsx | 84 +++++----- 6 files changed, 277 insertions(+), 87 deletions(-) diff --git a/services/common/src/components/forms/RenderFileUpload.tsx b/services/common/src/components/forms/RenderFileUpload.tsx index 0b239fc9c7..a383f55a25 100644 --- a/services/common/src/components/forms/RenderFileUpload.tsx +++ b/services/common/src/components/forms/RenderFileUpload.tsx @@ -18,7 +18,10 @@ import withFeatureFlag from "@mds/common/providers/featureFlags/withFeatureFlag" import { createRequestHeader } from "@mds/common/redux/utils/RequestHeaders"; import { FLUSH_SOUND, WATER_SOUND } from "@mds/common/constants/assets"; import { getSystemFlag } from "@mds/common/redux/selectors/authenticationSelectors"; -import { MultipartDocumentUpload } from "@mds/common/utils/fileUploadHelper.interface"; +import { + MultipartDocumentUpload, + UploadResult, +} from "@mds/common/utils/fileUploadHelper.interface"; import { HttpRequest, HttpResponse } from "tus-js-client"; registerPlugin(FilePondPluginFileValidateSize, FilePondPluginFileValidateType); @@ -93,17 +96,30 @@ export const FileUpload = (props: FileUploadProps) => { const system = useSelector(getSystemFlag); const [showWhirlpool, setShowWhirlpool] = useState(false); - const [uploadResults, setUploadResults] = useState([]); - const [uploadData, setUploadData] = useState(null); - const [uploadProcess, setUploadProcess] = useState({ - fieldName: null, - file: props.file || null, - metadata: null, - load: null, - error: null, - progress: null, - abort: null, - }); + + // Used to store intermittent results of upload parts to enable + // retries of parts that fail. + const [uploadResults, setUploadResults] = useState<{ [fileId: string]: UploadResult[] }>({}); + + // Used to store upload information about each upload and part + // including pre-signed urls to enable retries of parts that fail, + // and replace file functionality + const [uploadData, setUploadData] = useState<{ [fileId: string]: MultipartDocumentUpload }>({}); + + // Stores metadata and process function for each file, so we can manually + // trigger it. Currently, this is being used for the replace file functionality + // which dynamically changes the URL of the upload if you confirm the replacement + const [uploadProcess, setUploadProcess] = useState<{ + [fileId: string]: { + fieldName: string; + file: File; + metadata: any; + load: (documentGuid: string) => void; + error: (file: File, err: any) => void; + progress: (file: File, progress: number) => void; + abort: () => void; + }; + }>({}); let waterSound; let flushSound; @@ -111,7 +127,9 @@ export const FileUpload = (props: FileUploadProps) => { const handleError = (file, err) => { try { - const response = JSON.parse(err.originalRequest.getUnderlyingObject().response); + const response = err.originalRequest + ? JSON.parse(err.originalRequest.getUnderlyingObject().response) + : err || {}; if ( !( @@ -124,6 +142,7 @@ export const FileUpload = (props: FileUploadProps) => { duration: 10, }); } + if (props.onError) { props.onError(file && file.name ? file.name : "", err); } @@ -190,24 +209,69 @@ export const FileUpload = (props: FileUploadProps) => { intervalId = setInterval(pollUploadStatus, 1000); }; - function _s3MultipartUpload(uploadUrl, file, metadata, load, error, progress, abort) { + const setUploadResultsFor = (fileId, results) => { + setUploadResults({ + ...uploadResults, + [fileId]: results, + }); + }; + + const setUploadProcessFor = (fileId, process) => { + setUploadProcess({ + ...uploadProcess, + [fileId]: process, + }); + }; + const setUploadDataFor = (fileId, data) => { + setUploadData({ + ...uploadData, + [fileId]: data, + }); + }; + + function _s3MultipartUpload(fileId, uploadUrl, file, metadata, load, error, progress, abort) { + const timestamp = new Date().getTime(); + const start = timestamp; + function formatBytes(kiloBytes, decimals) { + if (kiloBytes == 0) return ["0", "KB"]; + + const k = 1024, + dm = decimals + 1 || 1, + sizes = ["KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"], + i = Math.floor(Math.log(kiloBytes) / Math.log(k)); + + return [parseFloat((kiloBytes / Math.pow(k, i)).toFixed(dm)).toLocaleString(), sizes[i]]; // parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + + } + return new FileUploadHelper(file, { uploadUrl: ENVIRONMENT.apiUrl + uploadUrl, - uploadResults: uploadResults, - uploadData: uploadData, + // Pass along results and upload configuration if exists from + // previous upload attempts for this file. Occurs if retrying a failed upload. + uploadResults: uploadResults[fileId], + uploadData: uploadData[fileId], metadata: { filename: file.name, filetype: file.type || APPLICATION_OCTET_STREAM, }, onError: (err, uploadResults) => { - setUploadResults(uploadResults); + setUploadResultsFor(fileId, uploadResults); handleError(file, err); error(err); }, onInit: (uploadData) => { - setUploadData(uploadData); + setUploadDataFor(fileId, uploadData); }, onProgress: (bytesUploaded, bytesTotal) => { + const ms = new Date().getTime(); + const elapsed = (ms - timestamp) / 1000.0; + const bps = bytesUploaded / elapsed; + + console.log( + `Uploading ${fileId}:`, + (bytesUploaded * 100) / bytesTotal + "%", + formatBytes(bps / 1024, 2) + "/s" + ); + progress(true, bytesUploaded, bytesTotal); }, onSuccess: (documentGuid) => { @@ -217,7 +281,7 @@ export const FileUpload = (props: FileUploadProps) => { }); } - function _tusdUpload(uploadUrl, file, metadata, load, error, progress, abort) { + function _tusdUpload(fileId, uploadUrl, file, metadata, load, error, progress, abort) { const upload = new tus.Upload(file, { endpoint: ENVIRONMENT.apiUrl + uploadUrl, retryDelays: [100, 1000, 3000], @@ -268,10 +332,20 @@ export const FileUpload = (props: FileUploadProps) => { } const server = { - process: (fieldName, file, metadata, load, error, progress, abort) => { + process: ( + fieldName, + file, + metadata, + load, + error, + progress, + abort, + transfer = null, + options = null + ) => { let upload; - setUploadProcess({ + setUploadProcessFor(metadata.filepondid, { fieldName, file, metadata, @@ -280,15 +354,34 @@ export const FileUpload = (props: FileUploadProps) => { progress, abort, }); - setUploadData(null); - setUploadResults([]); + + setUploadDataFor(metadata.filepondid, null); + setUploadResultsFor(metadata.filepondid, []); const uploadUrl = props.shouldReplaceFile ? props.replaceFileUploadUrl : props.uploadUrl; if (props.isFeatureEnabled("s3_multipart_upload")) { - upload = _s3MultipartUpload(uploadUrl, file, metadata, load, error, progress, abort); + upload = _s3MultipartUpload( + metadata.filepondid, + uploadUrl, + file, + metadata, + load, + error, + progress, + abort + ); } else { - upload = _tusdUpload(uploadUrl, file, metadata, load, error, progress, abort); + upload = _tusdUpload( + metadata.filepondid, + uploadUrl, + file, + metadata, + load, + error, + progress, + abort + ); } upload.start(); @@ -333,7 +426,9 @@ export const FileUpload = (props: FileUploadProps) => { } }; }, []); - + const onProcessFiles = (err, file) => { + file.setMetadata("filepondid", file.id); + }; const fileValidateTypeLabelExpectedTypesMap = invert(props.acceptedFileTypesMap); const acceptedFileTypes = uniq(Object.values(props.acceptedFileTypesMap)); @@ -361,6 +456,7 @@ export const FileUpload = (props: FileUploadProps) => { }} /> )} + (filepond = ref)} server={server} @@ -376,7 +472,7 @@ export const FileUpload = (props: FileUploadProps) => { // maxFiles={props.maxFiles || undefined} allowFileTypeValidation={acceptedFileTypes.length > 0} acceptedFileTypes={acceptedFileTypes} - onprocessfiles={props.onProcessFiles} + onaddfile={onProcessFiles} onprocessfileabort={props.onAbort} // oninit={props.onInit} labelIdle={props?.labelIdle} diff --git a/services/common/src/redux/customAxios.js b/services/common/src/redux/customAxios.js index 9ff5a1185e..f7075f575b 100644 --- a/services/common/src/redux/customAxios.js +++ b/services/common/src/redux/customAxios.js @@ -1,7 +1,7 @@ import axios from "axios"; import { notification, Button } from "antd"; import * as String from "@mds/common/constants/strings"; -import React from 'react'; +import React from "react"; import * as API from "@mds/common/constants/API"; import { ENVIRONMENT } from "@mds/common"; import { createRequestHeader } from "./utils/RequestHeaders"; @@ -20,18 +20,19 @@ const formatErrorMessage = (errorMessage) => { return errorMessage.replace("(psycopg2.", "(DatabaseError."); }; -const notifymAdmin = (error) => { +let CustomAxios; +const notifymAdmin = (error) => { const business_message = error?.response?.data?.message; const detailed_error = error?.response?.data?.detailed_error; const payload = { - "business_error": business_message, - "detailed_error": detailed_error + business_error: business_message, + detailed_error: detailed_error, }; - // @ts-ignore - CustomAxios().post(ENVIRONMENT.apiUrl + API.REPORT_ERROR, payload, createRequestHeader()) + CustomAxios() + .post(ENVIRONMENT.apiUrl + API.REPORT_ERROR, payload, createRequestHeader()) .then((response) => { notification.success({ message: "Error details sent to Admin. Thank you.", @@ -41,11 +42,10 @@ const notifymAdmin = (error) => { }) .catch((err) => { throw new Error(err); - }) + }); }; -// @ts-ignore -const CustomAxios = ({ errorToastMessage, suppressErrorNotification = false } = {}) => { +CustomAxios = ({ errorToastMessage = null, suppressErrorNotification = false } = {}) => { const instance = axios.create(); instance.interceptors.response.use( @@ -63,21 +63,29 @@ const CustomAxios = ({ errorToastMessage, suppressErrorNotification = false } = (errorToastMessage === "default" || errorToastMessage === undefined) && !suppressErrorNotification ) { - console.error('Detailed Error: ', error?.response?.data?.detailed_error) - const notificationKey = 'errorNotification'; + console.error("Detailed Error: ", error?.response?.data?.detailed_error); + const notificationKey = "errorNotification"; if (isFeatureEnabled(Feature.REPORT_ERROR)) { notification.error({ key: notificationKey, message: formatErrorMessage(error?.response?.data?.message ?? String.ERROR), - description:

If you think this is a system error please help us to improve by informing the system Admin

, + description: ( +

+ If you think this is a system error please help us to improve by informing the + system Admin +

+ ), duration: 10, btn: ( - ), @@ -89,7 +97,6 @@ const CustomAxios = ({ errorToastMessage, suppressErrorNotification = false } = duration: 10, }); } - } else if (errorToastMessage && !suppressErrorNotification) { notification.error({ message: errorToastMessage, diff --git a/services/common/src/utils/fileUploadHelper.spec.ts b/services/common/src/utils/fileUploadHelper.spec.ts index 826966ed0a..4188fbf96d 100644 --- a/services/common/src/utils/fileUploadHelper.spec.ts +++ b/services/common/src/utils/fileUploadHelper.spec.ts @@ -196,8 +196,8 @@ describe("FileUploadHelper", () => { { upload_id: "uploadId", parts: [ - { etag: "etagpart2", part: 2 }, { etag: "etagpart1", part: 1 }, + { etag: "etagpart2", part: 2 }, ], version_guid: "versionguid", }, @@ -208,5 +208,66 @@ describe("FileUploadHelper", () => { expect(mockedAxios.post).not.toHaveBeenCalled(); expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); }); + it("should complete upload with parts in ascending order", async () => { + const uploadData: MultipartDocumentUpload = { + document_manager_guid: "guid", + document_manager_version_guid: "versionguid", + upload: { + uploadId: "uploadId", + parts: [ + { part: 2, size: 200, url: "part2-url" }, + { part: 3, size: 300, url: "part3-url" }, + { part: 1, size: 100, url: "part1-url" }, + ], + }, + }; + const uploadResults = [ + { part: { part: 2, size: 100, url: "part2-url" }, etag: "etagpart2", status: "failed" }, + { part: { part: 3, size: 100, url: "part3-url" }, etag: "etagpart3", status: "failed" }, + { part: { part: 1, size: 100, url: "part1-url" }, etag: "etagpart1", status: "failed" }, + ]; + config.uploadData = uploadData; + config.uploadResults = uploadResults; + + const fileUploadHelper = new FileUploadHelper(file, config); + mockedAxios.put.mockResolvedValueOnce({ headers: { etag: "etagpart2" } }); + mockedAxios.put.mockResolvedValueOnce({ headers: { etag: "etagpart3" } }); + mockedAxios.put.mockResolvedValueOnce({ headers: { etag: "etagpart1" } }); + mockedAxios.patch.mockResolvedValueOnce({}); + + await fileUploadHelper.start(); + expect(mockedAxios.put).toHaveBeenCalledWith( + "part2-url", + expect.any(Blob), + expect.any(Object) + ); + expect(mockedAxios.put).toHaveBeenCalledWith( + "part3-url", + expect.any(Blob), + expect.any(Object) + ); + expect(mockedAxios.put).toHaveBeenCalledWith( + "part1-url", + expect.any(Blob), + expect.any(Object) + ); + expect(mockedAxios.patch).toHaveBeenCalledWith( + "/documents/guid/complete-upload", + { + upload_id: "uploadId", + parts: [ + { etag: "etagpart1", part: 1 }, + { etag: "etagpart2", part: 2 }, + { etag: "etagpart3", part: 3 }, + ], + version_guid: "versionguid", + }, + expect.any(Object) + ); + expect(mockedAxios.put).toHaveBeenCalledTimes(3); + expect(mockedAxios.patch).toHaveBeenCalledTimes(1); + expect(mockedAxios.post).not.toHaveBeenCalled(); + expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + }); }); }); diff --git a/services/common/src/utils/fileUploadHelper.ts b/services/common/src/utils/fileUploadHelper.ts index c537eabd1f..ce741251a8 100644 --- a/services/common/src/utils/fileUploadHelper.ts +++ b/services/common/src/utils/fileUploadHelper.ts @@ -56,7 +56,17 @@ export class FileUploadHelper { Object.values(bytesUploaded).reduce((total: number, c: number) => total + c, 0); if (!this.config.uploadData) { - uploadData = await this._createMultipartUpload(); + try { + uploadData = await this._createMultipartUpload(); + } catch (e) { + if (this.config.onError) { + this.config.onError(e, uploadResults); + + return; + } else { + throw e; + } + } } if (this.config.onUploadResponse) { @@ -113,9 +123,13 @@ export class FileUploadHelper { const payload = { upload_id: uploadData.upload.uploadId, version_guid: uploadData.document_manager_version_guid, - parts: uploadResults.map((result) => ({ part: result.part.part, etag: result.etag })), + parts: uploadResults + .map((result) => ({ part: result.part.part, etag: result.etag })) + // S3 API requires parts to be passed along in ascending order + .sort((r1, r2) => r1.part - r2.part), }; + console.log(payload); await CustomAxios().patch( ENVIRONMENT.docManUrl + COMPLETE_MULTIPART_UPLOAD(uploadData.document_manager_guid), payload, @@ -131,7 +145,11 @@ export class FileUploadHelper { "Upload-Protocol": "s3-multipart", }); - const fileUploadResponse = await CustomAxios().post(this.config.uploadUrl, null, headers); + const fileUploadResponse = await CustomAxios({ suppressErrorNotification: true }).post( + this.config.uploadUrl, + null, + headers + ); this.config.onInit(fileUploadResponse.data); return fileUploadResponse.data; diff --git a/services/document-manager/backend/app/__init__.py b/services/document-manager/backend/app/__init__.py index af499f4a3e..e4f5e589d3 100644 --- a/services/document-manager/backend/app/__init__.py +++ b/services/document-manager/backend/app/__init__.py @@ -87,7 +87,11 @@ def register_extensions(app, test_config=None): jwt.init_app(app) if os.environ.get('ALLOW_CYPRESS_AUTH') == 'true': - jwt_cypress.init_app(app) + try: + jwt_cypress.init_app(app) + except Exception as e: + with app.app_context(): + current_app.logger.error('Failed to initialize cypress auth. Make sure keycloak is running', e) migrate.init_app(app, db) CORS(app) diff --git a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx index c1984123b6..a5174b1cb6 100644 --- a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx +++ b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx @@ -35,7 +35,7 @@ export const ProjectSummaryFileUpload: FC(null); const [replaceFileUploadUrl, setReplaceFileUploadUrl] = useState(); const [mineDocumentGuid, setMineDocumentGuid] = useState(null); const [mineGuid, setMineGuid] = useState(null); @@ -77,7 +77,8 @@ export const ProjectSummaryFileUpload: FC { - setShouldReplaceFile(false); + console.log("beeefore upload"); + setShouldReplaceFile(null); setShouldAbortUpload(false); setFileName(""); setMineDocumentGuid(null); @@ -138,7 +139,7 @@ export const ProjectSummaryFileUpload: FC { + setFileName(fileName); + if ( + e.response.status_code && + notificationDisabledStatusCodes.includes(e.response.status_code) + ) { + if (e.response.status === "ARCHIVED_FILE_EXIST") { + setShouldAbortUpload(false); + const message = `An archived file named ${filename} already exists. If you would like to restore it, download the archived file and upload it again with a different file name.`; + setArchivedFileModalMessage(message); + setIsArchivedFileModalVisible(true); + } + if (e.response.status === "REPLACEABLE_FILE_EXIST") { + setShouldAbortUpload(false); + const message = `A file with the same name already exists in this project. Replacing it will create a new version of the original file and replace it as part of this submission.`; + setReplaceableFileModalMessage(message); + setReplaceableFileModalVisible(true); + setMineGuid(e.response.mine_guid); + setMineDocumentGuid(e.response.mine_document_guid); + + const date = new Date(e.response.update_timestamp); + const options: Intl.DateTimeFormatOptions = { + year: "numeric", + month: "short", + day: "2-digit", + }; + const formattedDate = date.toLocaleDateString("en-US", options); + + setFileDetails({ + file_name: filename, + file_type: e.response.file_type, + update_timestamp: `${formattedDate}`, + update_user: e.response.update_user, + }); + } + } + }; + return ( <> { - setFileName(fileName); - if ( - e.response.status_code && - notificationDisabledStatusCodes.includes(e.response.status_code) - ) { - if (e.response.status === "ARCHIVED_FILE_EXIST") { - setShouldAbortUpload(false); - const message = `An archived file named ${filename} already exists. If you would like to restore it, download the archived file and upload it again with a different file name.`; - setArchivedFileModalMessage(message); - setIsArchivedFileModalVisible(true); - } - if (e.response.status === "REPLACEABLE_FILE_EXIST") { - setShouldAbortUpload(false); - const message = `A file with the same name already exists in this project. Replacing it will create a new version of the original file and replace it as part of this submission.`; - setReplaceableFileModalMessage(message); - setReplaceableFileModalVisible(true); - setMineGuid(e.response.mine_guid); - setMineDocumentGuid(e.response.mine_document_guid); - - const date = new Date(e.response.update_timestamp); - const options: Intl.DateTimeFormatOptions = { - year: "numeric", - month: "short", - day: "2-digit", - }; - const formattedDate = date.toLocaleDateString("en-US", options); - - setFileDetails({ - file_name: filename, - file_type: e.response.file_type, - update_timestamp: `${formattedDate}`, - update_user: e.response.update_user, - }); - } - } - }, + onError, onUploadResponse, }} /> From e7aee0c510a78c42748e78ae2ca7acb92af29df9 Mon Sep 17 00:00:00 2001 From: Simen Fivelstad Smaaberg <66635118+simensma-fresh@users.noreply.github.com> Date: Fri, 12 Jan 2024 04:40:05 +0000 Subject: [PATCH 2/5] MDS-5720 Fixed Project summary file replacement functionality --- .../common/src/components/forms/RenderFileUpload.tsx | 10 +++++----- .../common/src/utils/fileUploadHelper.interface.ts | 2 +- services/common/src/utils/fileUploadHelper.ts | 5 ++++- .../projectSummary/ProjectSummaryFileUpload.tsx | 9 ++++++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/services/common/src/components/forms/RenderFileUpload.tsx b/services/common/src/components/forms/RenderFileUpload.tsx index a383f55a25..11dc18a94e 100644 --- a/services/common/src/components/forms/RenderFileUpload.tsx +++ b/services/common/src/components/forms/RenderFileUpload.tsx @@ -35,7 +35,7 @@ type AfterSuccessActionType = [ interface FileUploadProps { uploadUrl: string; acceptedFileTypesMap?: { [key: string]: string }; - onFileLoad?: (fileName?: string, documentGuid?: string) => void; + onFileLoad?: (fileName?: string, documentGuid?: string, versionGuid?: string) => void; chunkSize?: number; onAbort?: () => void; onUploadResponse?: (data: MultipartDocumentUpload) => void; @@ -154,7 +154,7 @@ export const FileUpload = (props: FileUploadProps) => { } }; - const handleSuccess = (documentGuid, file, load, abort) => { + const handleSuccess = (documentGuid, file, load, abort, versionGuid?) => { let intervalId; // eslint-disable-line prefer-const const pollUploadStatus = async () => { @@ -163,7 +163,7 @@ export const FileUpload = (props: FileUploadProps) => { clearInterval(intervalId); if (response.data.status === "Success") { load(documentGuid); - props.onFileLoad(file.name, documentGuid); + props.onFileLoad(file.name, documentGuid, versionGuid); if (props?.afterSuccess?.action) { try { @@ -274,8 +274,8 @@ export const FileUpload = (props: FileUploadProps) => { progress(true, bytesUploaded, bytesTotal); }, - onSuccess: (documentGuid) => { - handleSuccess(documentGuid, file, load, abort); + onSuccess: (documentGuid, versionGuid) => { + handleSuccess(documentGuid, file, load, abort, versionGuid); }, onUploadResponse: props.onUploadResponse, }); diff --git a/services/common/src/utils/fileUploadHelper.interface.ts b/services/common/src/utils/fileUploadHelper.interface.ts index 4dd477e0a6..870e803ead 100644 --- a/services/common/src/utils/fileUploadHelper.interface.ts +++ b/services/common/src/utils/fileUploadHelper.interface.ts @@ -11,7 +11,7 @@ export interface FileUploadHelperProps { uploadData?: MultipartDocumentUpload; onError: (err, uploadResults: UploadResult[]) => void; onProgress: (bytesUploaded: number, bytesTotal: number) => void; - onSuccess: (documentManagerGuid: string) => void; + onSuccess: (documentManagerGuid: string, documentManagerVersionGuid?: string) => void; onInit?: (uploadData: MultipartDocumentUpload) => void; onUploadResponse?: (data: MultipartDocumentUpload) => void; retryDelayMs?: number; diff --git a/services/common/src/utils/fileUploadHelper.ts b/services/common/src/utils/fileUploadHelper.ts index ce741251a8..779a240549 100644 --- a/services/common/src/utils/fileUploadHelper.ts +++ b/services/common/src/utils/fileUploadHelper.ts @@ -108,7 +108,10 @@ export class FileUploadHelper { if (uploadResults?.length && uploadResults.every((result) => result.status === "success")) { await this._completeMultipartUpload(uploadData, uploadResults); - this.config.onSuccess(uploadData.document_manager_guid); + this.config.onSuccess( + uploadData.document_manager_guid, + uploadData.document_manager_version_guid + ); } }; diff --git a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx index a5174b1cb6..ec50fd66af 100644 --- a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx +++ b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx @@ -83,6 +83,7 @@ export const ProjectSummaryFileUpload: FC((resolve, reject) => { const existingDocument = existingFiles.find( (document) => document.document_name === file.filename @@ -156,9 +157,9 @@ export const ProjectSummaryFileUpload: FC { + const handleFileLoad = (fileName: string, document_guid: string, versionGuid: string) => { props.onFileLoad(fileName, document_guid, { - document_manager_version_guid: version, + document_manager_version_guid: version || versionGuid, document_manager_guid: mineDocumentGuid, }); }; @@ -211,7 +212,9 @@ export const ProjectSummaryFileUpload: FC + handleFileLoad(fileName, documentGuid, versionGuid) + } onRemoveFile={props.onRemoveFile} notificationDisabledStatusCodes={notificationDisabledStatusCodes} shouldAbortUpload={shouldAbortUpload} From 701bc1c9222d4304513f9e6460982752a5d0fae0 Mon Sep 17 00:00:00 2001 From: Simen Fivelstad Smaaberg <66635118+simensma-fresh@users.noreply.github.com> Date: Fri, 12 Jan 2024 16:08:18 +0000 Subject: [PATCH 3/5] MDS-5720 Removed comments --- .../src/components/forms/RenderFileUpload.tsx | 31 ++++--------------- services/common/src/utils/fileUploadHelper.ts | 1 - .../ProjectSummaryFileUpload.tsx | 11 ++----- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/services/common/src/components/forms/RenderFileUpload.tsx b/services/common/src/components/forms/RenderFileUpload.tsx index 11dc18a94e..065a21a19b 100644 --- a/services/common/src/components/forms/RenderFileUpload.tsx +++ b/services/common/src/components/forms/RenderFileUpload.tsx @@ -230,19 +230,6 @@ export const FileUpload = (props: FileUploadProps) => { }; function _s3MultipartUpload(fileId, uploadUrl, file, metadata, load, error, progress, abort) { - const timestamp = new Date().getTime(); - const start = timestamp; - function formatBytes(kiloBytes, decimals) { - if (kiloBytes == 0) return ["0", "KB"]; - - const k = 1024, - dm = decimals + 1 || 1, - sizes = ["KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"], - i = Math.floor(Math.log(kiloBytes) / Math.log(k)); - - return [parseFloat((kiloBytes / Math.pow(k, i)).toFixed(dm)).toLocaleString(), sizes[i]]; // parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + - } - return new FileUploadHelper(file, { uploadUrl: ENVIRONMENT.apiUrl + uploadUrl, // Pass along results and upload configuration if exists from @@ -262,16 +249,6 @@ export const FileUpload = (props: FileUploadProps) => { setUploadDataFor(fileId, uploadData); }, onProgress: (bytesUploaded, bytesTotal) => { - const ms = new Date().getTime(); - const elapsed = (ms - timestamp) / 1000.0; - const bps = bytesUploaded / elapsed; - - console.log( - `Uploading ${fileId}:`, - (bytesUploaded * 100) / bytesTotal + "%", - formatBytes(bps / 1024, 2) + "/s" - ); - progress(true, bytesUploaded, bytesTotal); }, onSuccess: (documentGuid, versionGuid) => { @@ -426,9 +403,12 @@ export const FileUpload = (props: FileUploadProps) => { } }; }, []); - const onProcessFiles = (err, file) => { + + const handleFileAdd = (err, file) => { + // Add ID to file metadata so we can reference it later file.setMetadata("filepondid", file.id); }; + const fileValidateTypeLabelExpectedTypesMap = invert(props.acceptedFileTypesMap); const acceptedFileTypes = uniq(Object.values(props.acceptedFileTypesMap)); @@ -472,7 +452,8 @@ export const FileUpload = (props: FileUploadProps) => { // maxFiles={props.maxFiles || undefined} allowFileTypeValidation={acceptedFileTypes.length > 0} acceptedFileTypes={acceptedFileTypes} - onaddfile={onProcessFiles} + onaddfile={handleFileAdd} + onprocessfiles={props.onProcessFiles} onprocessfileabort={props.onAbort} // oninit={props.onInit} labelIdle={props?.labelIdle} diff --git a/services/common/src/utils/fileUploadHelper.ts b/services/common/src/utils/fileUploadHelper.ts index 779a240549..cf91de305c 100644 --- a/services/common/src/utils/fileUploadHelper.ts +++ b/services/common/src/utils/fileUploadHelper.ts @@ -132,7 +132,6 @@ export class FileUploadHelper { .sort((r1, r2) => r1.part - r2.part), }; - console.log(payload); await CustomAxios().patch( ENVIRONMENT.docManUrl + COMPLETE_MULTIPART_UPLOAD(uploadData.document_manager_guid), payload, diff --git a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx index ec50fd66af..dfba15612f 100644 --- a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx +++ b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx @@ -35,7 +35,7 @@ export const ProjectSummaryFileUpload: FC(null); + const [shouldReplaceFile, setShouldReplaceFile] = useState(false); const [replaceFileUploadUrl, setReplaceFileUploadUrl] = useState(); const [mineDocumentGuid, setMineDocumentGuid] = useState(null); const [mineGuid, setMineGuid] = useState(null); @@ -77,13 +77,11 @@ export const ProjectSummaryFileUpload: FC { - console.log("beeefore upload"); setShouldReplaceFile(null); setShouldAbortUpload(false); setFileName(""); setMineDocumentGuid(null); setVersion(null); - console.log("Resetting version to null"); return new Promise((resolve, reject) => { const existingDocument = existingFiles.find( (document) => document.document_name === file.filename @@ -140,7 +138,7 @@ export const ProjectSummaryFileUpload: FC - handleFileLoad(fileName, documentGuid, versionGuid) - } + onFileLoad={handleFileLoad} onRemoveFile={props.onRemoveFile} notificationDisabledStatusCodes={notificationDisabledStatusCodes} shouldAbortUpload={shouldAbortUpload} - onError={onError} allowRevert allowMultiple props={{ From 349a348529755278920f1ec92663ddd2cb6b5926 Mon Sep 17 00:00:00 2001 From: Simen Fivelstad Smaaberg <66635118+simensma-fresh@users.noreply.github.com> Date: Fri, 12 Jan 2024 16:25:41 +0000 Subject: [PATCH 4/5] MDS-5720 Cleanup --- .../Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx index dfba15612f..ec7568640a 100644 --- a/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx +++ b/services/minespace-web/src/components/Forms/projects/projectSummary/ProjectSummaryFileUpload.tsx @@ -77,7 +77,7 @@ export const ProjectSummaryFileUpload: FC { - setShouldReplaceFile(null); + setShouldReplaceFile(false); setShouldAbortUpload(false); setFileName(""); setMineDocumentGuid(null); From 79aaddfce3a45a5927c13e99cce8ee172054baa3 Mon Sep 17 00:00:00 2001 From: Simen Fivelstad Smaaberg <66635118+simensma-fresh@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:38:31 +0000 Subject: [PATCH 5/5] Fixed broken tests --- .../common/src/utils/fileUploadHelper.spec.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/services/common/src/utils/fileUploadHelper.spec.ts b/services/common/src/utils/fileUploadHelper.spec.ts index 4188fbf96d..b14cc85b63 100644 --- a/services/common/src/utils/fileUploadHelper.spec.ts +++ b/services/common/src/utils/fileUploadHelper.spec.ts @@ -68,7 +68,7 @@ describe("FileUploadHelper", () => { expect.any(Blob), expect.any(Object) ); - expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid, ""); }); it("should retry parts when uploadResults is provided", async () => { @@ -97,7 +97,7 @@ describe("FileUploadHelper", () => { expect(mockedAxios.post).not.toHaveBeenCalled(); expect(mockedAxios.patch).toHaveBeenCalledTimes(1); - expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid, ""); }); it("should not upload successful parts when retrying", async () => { @@ -134,7 +134,10 @@ describe("FileUploadHelper", () => { expect(mockedAxios.post).not.toHaveBeenCalled(); expect(mockedAxios.patch).toHaveBeenCalledTimes(1); - expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + expect(config.onSuccess).toHaveBeenCalledWith( + uploadData.document_manager_guid, + "versionguid" + ); }); it("should abort the upload if a part fails to upload", async () => { @@ -206,7 +209,10 @@ describe("FileUploadHelper", () => { expect(mockedAxios.put).toHaveBeenCalledTimes(1); expect(mockedAxios.patch).toHaveBeenCalledTimes(1); expect(mockedAxios.post).not.toHaveBeenCalled(); - expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + expect(config.onSuccess).toHaveBeenCalledWith( + uploadData.document_manager_guid, + "versionguid" + ); }); it("should complete upload with parts in ascending order", async () => { const uploadData: MultipartDocumentUpload = { @@ -267,7 +273,10 @@ describe("FileUploadHelper", () => { expect(mockedAxios.put).toHaveBeenCalledTimes(3); expect(mockedAxios.patch).toHaveBeenCalledTimes(1); expect(mockedAxios.post).not.toHaveBeenCalled(); - expect(config.onSuccess).toHaveBeenCalledWith(uploadData.document_manager_guid); + expect(config.onSuccess).toHaveBeenCalledWith( + uploadData.document_manager_guid, + "versionguid" + ); }); }); });