From 44b0af53b14d3d66dc9ea0fc9d6f360eb40f5070 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Tue, 25 Jun 2024 10:08:21 +0200 Subject: [PATCH 01/80] friendlier error message --- packages/deposit/src/features/submit/submitApi.ts | 7 ++++--- packages/utils/error/index.tsx | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index c0c14be5..540b6bd2 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -174,17 +174,18 @@ export const submitApi = createApi({ if (metadataResult.error) { // enable form again if there's an error, so user can try and resubmit store.dispatch(setFormDisabled(false)); - console.log(metadataResult.error); + const error = metadataResult.error as FetchBaseQueryError; return { error: { - ...(metadataResult.error as FetchBaseQueryError), + ...error, data: i18n.t( actionType === "save" ? "saveErrorNotification" : "submitErrorNotification", { ns: "submit", - error: (metadataResult.error as FetchBaseQueryError).data, + error: (error.data as any).detail || + (typeof error.data === 'object' ? JSON.stringify(error.data) : error.data), }, ), }, diff --git a/packages/utils/error/index.tsx b/packages/utils/error/index.tsx index 77d16d33..07d7e2ba 100644 --- a/packages/utils/error/index.tsx +++ b/packages/utils/error/index.tsx @@ -23,7 +23,6 @@ export const errorLogger: Middleware = () => (next) => async (action) => { // Set error message, keep it simple for the user const error = action.payload.error || action.payload.data || action.error.message; - console.log(error); // Set conditions for when to post a ticket to freshdesk, if freshdesk is enabled let ticket; From 15bdb19e93070686c33d9af67336cfe4bb6d67dc Mon Sep 17 00:00:00 2001 From: Laurens Tobias Date: Thu, 27 Jun 2024 16:50:20 +0200 Subject: [PATCH 02/80] Add option to hide private for file upload --- apps/rda/src/config/form.ts | 5 +++++ .../deposit/src/features/files/FilesTable.tsx | 22 ++++++++++++------- packages/deposit/src/types/Metadata.ts | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/rda/src/config/form.ts b/apps/rda/src/config/form.ts index eb13bf15..5c12a673 100644 --- a/apps/rda/src/config/form.ts +++ b/apps/rda/src/config/form.ts @@ -35,6 +35,11 @@ const form: FormConfig = { ], submitKey: import.meta.env.VITE_PACKAGING_KEY, // still needed?? skipValidation: import.meta.env.DEV, + filesUpload: { + displayProcesses: false, + displayRoles: false, + displayPrivate: false, + } }; export default form; diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 6500226d..4ae778fc 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -56,6 +56,7 @@ const FilesTable = () => { const { displayRoles = true, displayProcesses = true, + displayPrivate = true, embargoDate = false, } = formConfig?.filesUpload || {}; @@ -68,7 +69,9 @@ const FilesTable = () => { {t("fileName")} {t("fileSize")} {t("fileType")} - {t("private")} + {displayPrivate && ( + {t("private")} + )} {displayRoles && ( {t("role")} )} @@ -230,6 +233,7 @@ const FileTableRow = ({ file }: FileItemProps) => { const { displayRoles = true, displayProcesses = true, + displayPrivate = true, convertFiles = true, embargoDate = false, } = formConfig?.filesUpload || {}; @@ -312,7 +316,8 @@ const FileTableRow = ({ file }: FileItemProps) => { - + {displayPrivate && ( + @@ -322,12 +327,13 @@ const FileTableRow = ({ file }: FileItemProps) => { type: "private", value: e.target.checked, }), - ) - } - data-testid={`private-${file.name}`} - disabled={file.valid === false || formDisabled} - /> - + ) + } + data-testid={`private-${file.name}`} + disabled={file.valid === false || formDisabled} + /> + + )} {displayRoles && ( Date: Tue, 9 Jul 2024 08:59:54 +0200 Subject: [PATCH 03/80] removed auth from file upload --- packages/deposit/src/features/files/FilesTable.tsx | 4 +--- packages/deposit/src/features/submit/Submit.tsx | 1 - packages/deposit/src/features/submit/submitApi.ts | 5 +---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 6500226d..31f50385 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -39,7 +39,6 @@ import { useSubmitFilesMutation } from "../submit/submitApi"; import { formatFileData } from "../submit/submitHelpers"; import { motion, AnimatePresence, HTMLMotionProps } from "framer-motion"; import { getFormDisabled, getData } from "../../deposit/depositSlice"; -import { useAuth } from "react-oidc-context"; import FileStatusIndicator from "./FileStatusIndicator"; import { lookupLanguageString } from "@dans-framework/utils"; import { useFetchGroupedListQuery } from "./api/dansFormats"; @@ -373,16 +372,15 @@ const UploadProgress = ({ file }: FileItemProps) => { const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); const [submitFiles] = useSubmitFilesMutation(); - const auth = useAuth(); const formConfig = useAppSelector(getData); const metadataSubmitStatus = useAppSelector(getMetadataSubmitStatus); const handleSingleFileUpload = () => { formatFileData(sessionId, [file]).then((d) => { + console.log('submit') submitFiles({ data: d, headerData: { - submitKey: auth.user?.access_token, target: formConfig.target, }, actionType: metadataSubmitStatus === "saved" ? "save" : "submit", diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 612ae80f..bae4d326 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -147,7 +147,6 @@ const Submit = ({ if (result.data?.data?.status === "OK") { // if metadata has been submitted ok, we start the file submit submitFiles({ - user: user, actionType: actionType, }); } diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index 540b6bd2..628c024e 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -197,7 +197,7 @@ export const submitApi = createApi({ }), submitFiles: build.mutation({ async queryFn( - { user, actionType }, + { actionType }, queryApi, _extraOptions, fetchWithBQ, @@ -227,9 +227,6 @@ export const submitApi = createApi({ method: "POST", data: file, headers: { - Authorization: `Bearer ${ - deposit.config.submitKey || user?.access_token - }`, "auth-env-name": deposit.config.target?.envName, }, }), From 21010cbccc0daa994f7d8dccdc3e7bcd7248bf0d Mon Sep 17 00:00:00 2001 From: D-Unit Date: Tue, 9 Jul 2024 11:27:43 +0200 Subject: [PATCH 04/80] test initial tus setup --- packages/deposit/package.json | 1 + .../deposit/src/features/files/FilesTable.tsx | 26 ++-- .../deposit/src/features/submit/Submit.tsx | 46 ++++--- .../deposit/src/features/submit/submitApi.ts | 111 ++++++++++++++--- .../deposit/src/features/submit/submitFile.ts | 94 +++++++++++++++ .../src/languages/locales/en/submit.json | 4 +- .../src/languages/locales/nl/submit.json | 4 +- pnpm-lock.yaml | 114 ++++++++++++++++++ 8 files changed, 351 insertions(+), 49 deletions(-) create mode 100644 packages/deposit/src/features/submit/submitFile.ts diff --git a/packages/deposit/package.json b/packages/deposit/package.json index 4128382f..3fc44245 100644 --- a/packages/deposit/package.json +++ b/packages/deposit/package.json @@ -31,6 +31,7 @@ "react-redux": "^8.1.2", "react-router-dom": "^6.14.2", "react-transition-group": "^4.4.5", + "tus-js-client": "^4.1.0", "use-debounce": "^9.0.4", "uuid": "^9.0.0" }, diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 8b7a1f1d..5c742b00 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -27,16 +27,16 @@ import type { FileItemProps, FileActions, } from "../../types/Files"; -import { getSessionId } from "../metadata/metadataSlice"; +// import { getSessionId } from "../metadata/metadataSlice"; import LinearProgress from "@mui/material/LinearProgress"; import Box from "@mui/material/Box"; import Stack from "@mui/material/Stack"; import { getSingleFileSubmitStatus, - getMetadataSubmitStatus, + // getMetadataSubmitStatus, } from "../submit/submitSlice"; -import { useSubmitFilesMutation } from "../submit/submitApi"; -import { formatFileData } from "../submit/submitHelpers"; +// import { useSubmitFilesMutation } from "../submit/submitApi"; +// import { formatFileData } from "../submit/submitHelpers"; import { motion, AnimatePresence, HTMLMotionProps } from "framer-motion"; import { getFormDisabled, getData } from "../../deposit/depositSlice"; import FileStatusIndicator from "./FileStatusIndicator"; @@ -374,24 +374,24 @@ const UploadProgress = ({ file }: FileItemProps) => { // We handle progress and retrying/restarting of file uploads here // If metadata submission is successful, and file fails right away, there needs to be an option to manually start file upload. // So we check if the submit button has been touched. - const sessionId = useAppSelector(getSessionId); + // const sessionId = useAppSelector(getSessionId); const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); - const [submitFiles] = useSubmitFilesMutation(); - const formConfig = useAppSelector(getData); - const metadataSubmitStatus = useAppSelector(getMetadataSubmitStatus); + // const [submitFiles] = useSubmitFilesMutation(); + // const formConfig = useAppSelector(getData); + // const metadataSubmitStatus = useAppSelector(getMetadataSubmitStatus); const handleSingleFileUpload = () => { - formatFileData(sessionId, [file]).then((d) => { - console.log('submit') - submitFiles({ + // formatFileData(sessionId, [file]).then((d) => { + // console.log('submit') + /*submitFiles({ data: d, headerData: { target: formConfig.target, }, actionType: metadataSubmitStatus === "saved" ? "save" : "submit", - }); - }); + });*/ + // }); }; return ( diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index bae4d326..2b0e2bcf 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -16,9 +16,11 @@ import { setOpenTab, getMetadata, getTouchedStatus, + getSessionId, } from "../metadata/metadataSlice"; import { getFiles, resetFiles } from "../files/filesSlice"; -import { useSubmitDataMutation, useSubmitFilesMutation } from "./submitApi"; +import { useSubmitDataMutation, /*useSubmitFilesMutation*/ } from "./submitApi"; +import { uploadFile } from "./submitFile"; import { setMetadataSubmitStatus, getMetadataSubmitStatus, @@ -54,6 +56,7 @@ const Submit = ({ const metadata = useAppSelector(getMetadata); const isTouched = useAppSelector(getTouchedStatus); const [fileWarning, setFileWarning] = useState(false); + const sessionId = useAppSelector(getSessionId); // get form config const formConfig = useAppSelector(getData); @@ -82,6 +85,8 @@ const Submit = ({ fileStatus === "error" && dispatch(setOpenTab(1)); }, [fileStatus]); + console.log(fileStatus) + const [ submitData, { @@ -90,13 +95,13 @@ const Submit = ({ reset: resetMeta, }, ] = useSubmitDataMutation(); - const [ - submitFiles, - { - isLoading: isLoadingFiles, - reset: resetSubmittedFiles, - }, - ] = useSubmitFilesMutation(); + // const [ + // submitFiles, + // { + // isLoading: isLoadingFiles, + // reset: resetSubmittedFiles, + // }, + // ] = useSubmitFilesMutation(); // Access token might just be expiring, or user settings just changed // So we do a callback to signinSilent, which refreshes the current user @@ -137,18 +142,23 @@ const Submit = ({ dispatch(setFormDisabled(true)); dispatch(setMetadataSubmitStatus("submitting")); + console.log(formConfig) + // do the actual submit getUser().then((user) => // with fresh headerdata/user info, we can submit the metadata submitData({ user: user, actionType: actionType, + id: sessionId, + metadata: metadata, + config: formConfig, + files: selectedFiles, }).then((result: { data?: any; error?: any }) => { - if (result.data?.data?.status === "OK") { + if (result.data?.status === "OK") { // if metadata has been submitted ok, we start the file submit - submitFiles({ - actionType: actionType, - }); + const fileUpload = Promise.all(selectedFiles.map( file => uploadFile(file, sessionId) ) ).then(res => console.log(res)); + console.log(fileUpload) } }), ); @@ -168,7 +178,7 @@ const Submit = ({ // Reset the entire form to initial state const resetForm = () => { // reset RTK mutations - resetSubmittedFiles(); + // resetSubmittedFiles(); resetMeta(); // reset metadata in metadata slice dispatch(resetMetadata()); @@ -221,7 +231,7 @@ const Submit = ({ : ( metadataSubmitStatus === "submitting" || fileStatus === "submitting" || - isLoadingFiles || + // isLoadingFiles || isLoadingMeta ) ? t("submitting") @@ -275,7 +285,7 @@ const Submit = ({ ( metadataSubmitStatus === "submitting" || fileStatus === "submitting" || - isLoadingFiles || + // isLoadingFiles || isLoadingMeta ) ? 0.5 @@ -296,15 +306,15 @@ const Submit = ({ isErrorMeta) && !( metadataSubmitStatus === "submitting" || - fileStatus === "submitting" || - isLoadingFiles + fileStatus === "submitting" //|| + // isLoadingFiles ) ) ? : } {(fileStatus === "submitting" || - isLoadingFiles || + // isLoadingFiles || isLoadingMeta) && ( ({ submitData: build.mutation({ + query: ({ user, actionType, id, metadata, config, files }) => { + console.log(metadata) + // format data + const data = formatFormData(id, metadata, files, config.formTitle); + console.log("Submit metadata:"); + console.log(data); + + // format headers + const headers = { + Authorization: `Bearer ${ + config.submitKey || user?.access_token + }`, + "user-id": user?.profile.sub, + "auth-env-name": config.target?.envName, + "assistant-config-name": config.target?.configName, + "targets-credentials": JSON.stringify( + config.targetCredentials.map((t: Target) => ({ + "target-repo-name": t.repo, + credentials: { + username: t.auth, + password: Object.assign( + {}, + ...config.targetCredentials.map((t: Target) => ({ + [t.authKey]: user?.profile[t.authKey], + })), + )[t.authKey], + }, + })), + ), + }; + console.log("Submit req headers:"); + console.log(headers); + + const submitUrl = + actionType === "resubmit" ? + `resubmit/${data.id}` + : `dataset/${actionType === "save" ? "DRAFT" : "PUBLISH"}`; + + return ({ + url: submitUrl, + method: "POST", + headers: headers, + body: data, + }) + }, + transformResponse: (response, _meta, arg) => { + // TODO: check for pending files + if (arg.actionType === "save") { + // show notice and enable form again after successful save + enqueueSnackbar( + i18n.t("saveSuccess", { + ns: "submit", + dateTime: moment().format("D-M-YYYY @ HH:mm"), + }), + { + variant: "success", + }, + ); + store.dispatch(setMetadataSubmitStatus( + arg.actionType === "save" ? "saved" : "submitted", + )); + } + store.dispatch(setFormDisabled(false)); + return response; + }, + transformErrorResponse: (response: FetchBaseQueryError) => { + store.dispatch(setFormDisabled(false)); + return { + error: { + ...response, + data: i18n.t("submitMetadataError", { + ns: "submit", + error: (response.data as any).detail || + (typeof response.data === 'object' ? JSON.stringify(response.data) : response.data), + }), + }, + }; + }, + // Custom query for chaining Post functions // submitKey is the current users Keycloak token - async queryFn( + /*async queryFn( { user, actionType }, queryApi, _extraOptions, @@ -193,9 +272,9 @@ export const submitApi = createApi({ } return { data: metadataResult }; - }, + },*/ }), - submitFiles: build.mutation({ + /*submitFiles: build.mutation({ async queryFn( { actionType }, queryApi, @@ -272,8 +351,8 @@ export const submitApi = createApi({ return { data: filesResults }; }, - }), + }),*/ }), }); -export const { useSubmitDataMutation, useSubmitFilesMutation } = submitApi; +export const { useSubmitDataMutation, /*useSubmitFilesMutation*/ } = submitApi; diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts new file mode 100644 index 00000000..08a2b2b7 --- /dev/null +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -0,0 +1,94 @@ +import * as tus from 'tus-js-client'; +import { store } from "../../redux/store"; +import { setFilesSubmitStatus } from "./submitSlice"; +import { setFileMeta } from "../files/filesSlice"; +import { SelectedFile } from "../../types/Files"; +import { enqueueSnackbar } from "notistack"; + +// Create a new tus upload +export const uploadFile = async (file: SelectedFile, sessionId: string) => { + const fetchedFile = await fetch(file.url); + const fileBlob = await fetchedFile.blob(); + + console.log('upload fn') + + const upload = new tus.Upload(fileBlob, { + // endpoint: `${import.meta.env.VITE_PACKAGING_TARGET}/files`, + endpoint: 'https://tusd.tusdemo.net/files/', + retryDelays: [0, 3000, 5000, 10000, 20000], + metadata: { + fileName: file.name, + fileId: file.id, + datasetId: sessionId, + }, + onBeforeRequest: () => { + console.log('starting TUS request to upload') + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + progress: 0, + status: "submitting", + }), + ); + }, + onError: function (error) { + // Display an error message + console.log('Failed because: ' + error) + }, + onShouldRetry: function (err, retryAttempt, _options) { + console.log("Error", err) + console.log("Request", err.originalRequest) + console.log("Response", err.originalResponse) + + var status = err.originalResponse ? err.originalResponse.getStatus() : 0 + // Do not retry if the status is a 403. + if (status === 403) { + return false + } + + enqueueSnackbar(`Error uploading ${file.name}. Retrying... (${retryAttempt + 1})`, { + variant: "warning", + }); + + // For any other status code, tus-js-client should retry. + return true + }, + onProgress: (bytesUploaded, bytesTotal) => { + var percentage = parseFloat(((bytesUploaded / bytesTotal) * 100).toFixed(1)) || 0; + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + progress: percentage, + status: "submitting", + }), + ); + }, + onSuccess: () => { + console.log('Download %s from %s', file.name, upload.url); + store.dispatch( + setFileMeta({ + id: file.id, + type: "submittedFile", + value: true, + }), + ); + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + status: "success", + }), + ); + }, + }); + + // Check if there are any previous uploads to continue. + upload.findPreviousUploads().then(function (previousUploads) { + // Found previous uploads so we select the first one. + if (previousUploads.length) { + upload.resumeFromPreviousUpload(previousUploads[0]) + } + + // Start the upload + upload.start() + }); +} \ No newline at end of file diff --git a/packages/deposit/src/languages/locales/en/submit.json b/packages/deposit/src/languages/locales/en/submit.json index 1859183b..092fcdee 100644 --- a/packages/deposit/src/languages/locales/en/submit.json +++ b/packages/deposit/src/languages/locales/en/submit.json @@ -17,5 +17,7 @@ "saveErrorNotification": "Failed to save metadata - {{error}}", "submitErrorNotification": "Failed to submit metadata - {{error}}", "saveFileErrorNotification": "Failed to save files - {{error}}", - "submitFileErrorNotification": "Failed to submit files - {{error}}" + "submitFileErrorNotification": "Failed to submit files - {{error}}", + "submitMetadataError": "Failed to send metadata to the server - {{error}}", + "submitFileError": "Failed to send files to the server - {{error}}" } \ No newline at end of file diff --git a/packages/deposit/src/languages/locales/nl/submit.json b/packages/deposit/src/languages/locales/nl/submit.json index b194943a..94e648cd 100644 --- a/packages/deposit/src/languages/locales/nl/submit.json +++ b/packages/deposit/src/languages/locales/nl/submit.json @@ -17,5 +17,7 @@ "saveErrorNotification": "Kan niet opslaan - {{error}}", "submitErrorNotification": "Kan niet submitten - {{error}}", "saveFileErrorNotification": "Kan bestanden niet opslaan - {{error}}", - "submitFileErrorNotification": "Kan bestanden niet submitten - {{error}}" + "submitFileErrorNotification": "Kan bestanden niet submitten - {{error}}", + "submitMetadataError": "Fout bij versturen metadata naar de server - {{error}}", + "submitFileError": "Fout bij versturen bestanden naar de server - {{error}}" } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45d435d8..a2afeadf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -507,6 +507,9 @@ importers: react-transition-group: specifier: ^4.4.5 version: 4.4.5(react-dom@18.2.0)(react@18.2.0) + tus-js-client: + specifier: ^4.1.0 + version: 4.1.0 use-debounce: specifier: ^9.0.4 version: 9.0.4(react@18.2.0) @@ -2561,6 +2564,10 @@ packages: node-releases: 2.0.13 update-browserslist-db: 1.0.11(browserslist@4.21.10) + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + /callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -2638,6 +2645,13 @@ packages: /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + /combine-errors@3.0.3: + resolution: {integrity: sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==} + dependencies: + custom-error-instance: 2.1.1 + lodash.uniqby: 4.5.0 + dev: false + /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -2707,6 +2721,10 @@ packages: /csstype@3.1.2: resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==} + /custom-error-instance@2.1.1: + resolution: {integrity: sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==} + dev: false + /debug@4.3.4(supports-color@5.5.0): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -3387,6 +3405,11 @@ packages: resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} engines: {node: '>=8'} + /is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + dev: false + /isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -3442,6 +3465,10 @@ packages: picomatch: 2.3.1 dev: false + /js-base64@3.7.7: + resolution: {integrity: sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==} + dev: false + /js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -3610,6 +3637,37 @@ packages: resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} dev: true + /lodash._baseiteratee@4.7.0: + resolution: {integrity: sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==} + dependencies: + lodash._stringtopath: 4.8.0 + dev: false + + /lodash._basetostring@4.12.0: + resolution: {integrity: sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==} + dev: false + + /lodash._baseuniq@4.6.0: + resolution: {integrity: sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==} + dependencies: + lodash._createset: 4.0.3 + lodash._root: 3.0.1 + dev: false + + /lodash._createset@4.0.3: + resolution: {integrity: sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==} + dev: false + + /lodash._root@3.0.1: + resolution: {integrity: sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==} + dev: false + + /lodash._stringtopath@4.8.0: + resolution: {integrity: sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==} + dependencies: + lodash._basetostring: 4.12.0 + dev: false + /lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} @@ -3620,6 +3678,17 @@ packages: resolution: {integrity: sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==} dev: true + /lodash.throttle@4.1.1: + resolution: {integrity: sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==} + dev: false + + /lodash.uniqby@4.5.0: + resolution: {integrity: sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==} + dependencies: + lodash._baseiteratee: 4.7.0 + lodash._baseuniq: 4.6.0 + dev: false + /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} dev: false @@ -3921,6 +3990,14 @@ packages: react-is: 16.13.1 dev: false + /proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + dev: false + /protocol-buffers-schema@3.6.0: resolution: {integrity: sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==} dev: false @@ -3933,6 +4010,10 @@ packages: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} + dev: false + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -4123,6 +4204,10 @@ packages: resolution: {integrity: sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==} dev: false + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + /reselect@4.1.8: resolution: {integrity: sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ==} dev: false @@ -4146,6 +4231,11 @@ packages: supports-preserve-symlinks-flag: 1.0.0 dev: false + /retry@0.12.0: + resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} + engines: {node: '>= 4'} + dev: false + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -4208,6 +4298,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + /signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + dev: false + /slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -4413,6 +4507,19 @@ packages: turbo-windows-arm64: 1.10.16 dev: true + /tus-js-client@4.1.0: + resolution: {integrity: sha512-e/nC/kJahvNYBcnwcqzuhFIvVELMMpbVXIoOOKdUn74SdQCvJd2JjqV2jZLv2EFOVbV4qLiO0lV7BxBXF21b6Q==} + engines: {node: '>=18'} + dependencies: + buffer-from: 1.1.2 + combine-errors: 3.0.3 + is-stream: 2.0.1 + js-base64: 3.7.7 + lodash.throttle: 4.1.1 + proper-lockfile: 4.1.2 + url-parse: 1.5.10 + dev: false + /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -4463,6 +4570,13 @@ packages: dependencies: punycode: 2.3.0 + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + dev: false + /use-debounce@9.0.4(react@18.2.0): resolution: {integrity: sha512-6X8H/mikbrt0XE8e+JXRtZ8yYVvKkdYRfmIhWZYsP8rcNs9hk3APV8Ua2mFkKRLcJKVdnX2/Vwrmg2GWKUQEaQ==} engines: {node: '>= 10.0.0'} From e61b54d4d4be8596c5215c8c10b648eebdc39eef Mon Sep 17 00:00:00 2001 From: D-Unit Date: Wed, 10 Jul 2024 11:47:44 +0200 Subject: [PATCH 05/80] tus working, need to work on additional api call after file submit --- .../deposit/src/features/files/FilesTable.tsx | 30 +- .../deposit/src/features/submit/Submit.tsx | 30 +- .../deposit/src/features/submit/submitApi.ts | 274 +----------------- .../deposit/src/features/submit/submitFile.ts | 65 +++-- packages/utils/error/index.tsx | 14 +- packages/utils/error/package.json | 4 +- packages/utils/lib/index.ts | 3 +- packages/utils/user/index.ts | 13 + packages/utils/user/package.json | 11 + pnpm-lock.yaml | 12 +- 10 files changed, 110 insertions(+), 346 deletions(-) create mode 100644 packages/utils/user/index.ts create mode 100644 packages/utils/user/package.json diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 5c742b00..2ead30bd 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -27,16 +27,11 @@ import type { FileItemProps, FileActions, } from "../../types/Files"; -// import { getSessionId } from "../metadata/metadataSlice"; +import { getSessionId } from "../metadata/metadataSlice"; import LinearProgress from "@mui/material/LinearProgress"; import Box from "@mui/material/Box"; import Stack from "@mui/material/Stack"; -import { - getSingleFileSubmitStatus, - // getMetadataSubmitStatus, -} from "../submit/submitSlice"; -// import { useSubmitFilesMutation } from "../submit/submitApi"; -// import { formatFileData } from "../submit/submitHelpers"; +import { getSingleFileSubmitStatus } from "../submit/submitSlice"; import { motion, AnimatePresence, HTMLMotionProps } from "framer-motion"; import { getFormDisabled, getData } from "../../deposit/depositSlice"; import FileStatusIndicator from "./FileStatusIndicator"; @@ -46,6 +41,7 @@ import { findFileGroup } from "./filesHelpers"; import { DatePicker } from '@mui/x-date-pickers/DatePicker'; import moment, { Moment } from "moment"; import type { DateValidationError } from "@mui/x-date-pickers/models"; +import { uploadFile } from "../submit/submitFile"; const FilesTable = () => { const { t } = useTranslation("files"); @@ -371,27 +367,13 @@ const FileTableRow = ({ file }: FileItemProps) => { }; const UploadProgress = ({ file }: FileItemProps) => { - // We handle progress and retrying/restarting of file uploads here - // If metadata submission is successful, and file fails right away, there needs to be an option to manually start file upload. - // So we check if the submit button has been touched. - // const sessionId = useAppSelector(getSessionId); + // We handle progress and manually retrying/restarting of file uploads here + const sessionId = useAppSelector(getSessionId); const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); - // const [submitFiles] = useSubmitFilesMutation(); - // const formConfig = useAppSelector(getData); - // const metadataSubmitStatus = useAppSelector(getMetadataSubmitStatus); const handleSingleFileUpload = () => { - // formatFileData(sessionId, [file]).then((d) => { - // console.log('submit') - /*submitFiles({ - data: d, - headerData: { - target: formConfig.target, - }, - actionType: metadataSubmitStatus === "saved" ? "save" : "submit", - });*/ - // }); + uploadFile(file, sessionId) }; return ( diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 2b0e2bcf..27a53e57 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -85,7 +85,7 @@ const Submit = ({ fileStatus === "error" && dispatch(setOpenTab(1)); }, [fileStatus]); - console.log(fileStatus) + console.log('file status: ' + fileStatus) const [ submitData, @@ -95,13 +95,6 @@ const Submit = ({ reset: resetMeta, }, ] = useSubmitDataMutation(); - // const [ - // submitFiles, - // { - // isLoading: isLoadingFiles, - // reset: resetSubmittedFiles, - // }, - // ] = useSubmitFilesMutation(); // Access token might just be expiring, or user settings just changed // So we do a callback to signinSilent, which refreshes the current user @@ -137,13 +130,11 @@ const Submit = ({ // Clear any form action messages on submit if (actionType === "resubmit" || actionType === "submit") { clearFormActions(); + dispatch(setFormDisabled(true)); } - dispatch(setFormDisabled(true)); dispatch(setMetadataSubmitStatus("submitting")); - console.log(formConfig) - // do the actual submit getUser().then((user) => // with fresh headerdata/user info, we can submit the metadata @@ -156,9 +147,14 @@ const Submit = ({ files: selectedFiles, }).then((result: { data?: any; error?: any }) => { if (result.data?.status === "OK") { + console.log(selectedFiles) + console.log(filesSubmitStatus) // if metadata has been submitted ok, we start the file submit - const fileUpload = Promise.all(selectedFiles.map( file => uploadFile(file, sessionId) ) ).then(res => console.log(res)); - console.log(fileUpload) + selectedFiles.map( file => { + // only call the upload function if file is not yet uploaded, or is not currently uploading + const hasStatus = filesSubmitStatus.find( f => f.id === file.id); + return !file.submittedFile && !hasStatus && uploadFile(file, sessionId); + }); } }), ); @@ -219,7 +215,8 @@ const Submit = ({ { ( !metadataSubmitStatus || - (metadataSubmitStatus === "saved" && !formDisabled) + (metadataSubmitStatus === "saved" && !formDisabled) && + fileStatus !== "submitting" ) ? // metadata has not yet been submitted, so let's just indicate metadata completeness metadataStatus === "error" ? @@ -231,7 +228,6 @@ const Submit = ({ : ( metadataSubmitStatus === "submitting" || fileStatus === "submitting" || - // isLoadingFiles || isLoadingMeta ) ? t("submitting") @@ -285,7 +281,6 @@ const Submit = ({ ( metadataSubmitStatus === "submitting" || fileStatus === "submitting" || - // isLoadingFiles || isLoadingMeta ) ? 0.5 @@ -306,8 +301,7 @@ const Submit = ({ isErrorMeta) && !( metadataSubmitStatus === "submitting" || - fileStatus === "submitting" //|| - // isLoadingFiles + fileStatus === "submitting" ) ) ? diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index 1f720a68..e9f52714 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -1,110 +1,13 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; -import type { /*BaseQueryFn,*/ FetchBaseQueryError } from "@reduxjs/toolkit/query"; -// import axios from "axios"; -// import type { AxiosRequestConfig, AxiosError, AxiosProgressEvent } from "axios"; -import { setMetadataSubmitStatus, /*setFilesSubmitStatus*/ } from "./submitSlice"; -// import { setFileMeta } from "../files/filesSlice"; +import type { FetchBaseQueryError } from "@reduxjs/toolkit/query"; +import { setMetadataSubmitStatus } from "./submitSlice"; import { setFormDisabled } from "../../deposit/depositSlice"; import { store } from "../../redux/store"; import type { Target } from "@dans-framework/user-auth"; import moment from "moment"; import { enqueueSnackbar } from "notistack"; import i18n from "../../languages/i18n"; -import { formatFormData, /*formatFileData*/ } from "./submitHelpers"; -// import type { SubmitData } from "../../types/Submit"; - -// We use Axios to enable file upload progress monitoring -/*const axiosBaseQuery = - ( - { baseUrl }: { baseUrl: string } = { baseUrl: "" }, - ): BaseQueryFn< - { - url: string; - method: AxiosRequestConfig["method"]; - data?: AxiosRequestConfig["data"]; - params?: AxiosRequestConfig["params"]; - headers?: AxiosRequestConfig["headers"]; - actionType?: string; - }, - unknown, - unknown - > => - async ({ url, method, data, params, headers, actionType }) => { - // Perform actions based on server response here, so we can truly separate metadata and file handling - // Files are always a FormData object, metadata is JSON - const isFile = data instanceof FormData; - try { - const result = await axios({ - url: baseUrl + url, - method, - data, - params, - headers, - onUploadProgress: (progressEvent: AxiosProgressEvent) => { - if (isFile) { - // Calculate progress percentage and set state in fileSlice - const percentCompleted = - progressEvent.total ? - Math.round((progressEvent.loaded * 100) / progressEvent.total) - : 0; - store.dispatch( - setFilesSubmitStatus({ - id: data.get("fileId") as string, - progress: percentCompleted, - status: "submitting", - }), - ); - } - }, - }); - // set upload successful in file object - if (isFile && result.data) { - console.log(result); - // we need to remove the actual files from the list, as otherwise on anothes save in the same session the files will reupload - store.dispatch( - setFileMeta({ - id: data.get("fileId") as string, - type: "submittedFile", - value: true, - }), - ); - store.dispatch( - setFilesSubmitStatus({ - id: data.get("fileId") as string, - status: "success", - }), - ); - } - // Metadata has been successfully submitted, so let's store that right away - else if (result.data) { - store.dispatch( - setMetadataSubmitStatus( - actionType === "save" ? "saved" : "submitted", - ), - ); - } - return { data: result.data }; - } catch (axiosError) { - const err = axiosError as AxiosError; - if (isFile) { - // set error in the file object, so user can retry uploading - store.dispatch( - setFilesSubmitStatus({ - id: data.get("fileId") as string, - status: "error", - }), - ); - } else { - store.dispatch(setMetadataSubmitStatus("error")); - } - return { - error: { - status: err.response?.status, - data: err.response?.data || err.message, - }, - }; - } - };*/ +import { formatFormData } from "./submitHelpers"; export const submitApi = createApi({ reducerPath: "submitApi", @@ -159,7 +62,9 @@ export const submitApi = createApi({ }) }, transformResponse: (response, _meta, arg) => { - // TODO: check for pending files + store.dispatch(setMetadataSubmitStatus( + arg.actionType === "save" ? "saved" : "submitted", + )); if (arg.actionType === "save") { // show notice and enable form again after successful save enqueueSnackbar( @@ -171,14 +76,11 @@ export const submitApi = createApi({ variant: "success", }, ); - store.dispatch(setMetadataSubmitStatus( - arg.actionType === "save" ? "saved" : "submitted", - )); } - store.dispatch(setFormDisabled(false)); return response; }, transformErrorResponse: (response: FetchBaseQueryError) => { + // enable form again if there's an error, so user can try and resubmit store.dispatch(setFormDisabled(false)); return { error: { @@ -191,168 +93,8 @@ export const submitApi = createApi({ }, }; }, - - // Custom query for chaining Post functions - // submitKey is the current users Keycloak token - /*async queryFn( - { user, actionType }, - queryApi, - _extraOptions, - fetchWithBQ, - ) { - const { metadata, deposit, files } = queryApi.getState() as SubmitData; - const data = formatFormData(metadata.id, metadata.form, files, deposit.config.formTitle); - - console.log("Submit metadata:"); - console.log(data); - - // Format the headers - const headers = { - Authorization: `Bearer ${ - deposit.config.submitKey || user?.access_token - }`, - "user-id": user?.profile.sub, - "auth-env-name": deposit.config.target?.envName, - "assistant-config-name": deposit.config.target?.configName, - "targets-credentials": JSON.stringify( - deposit.config.targetCredentials.map((t: Target) => ({ - "target-repo-name": t.repo, - credentials: { - username: t.auth, - password: Object.assign( - {}, - ...deposit.config.targetCredentials.map((t) => ({ - [t.authKey]: user?.profile[t.authKey], - })), - )[t.authKey], - }, - })), - ), - }; - - console.log("Submit req headers:"); - console.log(headers); - - // First post the metadata - const submitUrl = - actionType === "resubmit" ? - `resubmit/${data.id}` - : `dataset/${actionType === "save" ? "DRAFT" : "PUBLISH"}`; - - const metadataResult = await fetchWithBQ({ - url: submitUrl, - method: "POST", - data: data, - headers: headers, - actionType: actionType, - }); - - console.log("Metadata server response:"); - console.log(metadataResult); - - if (metadataResult.error) { - // enable form again if there's an error, so user can try and resubmit - store.dispatch(setFormDisabled(false)); - const error = metadataResult.error as FetchBaseQueryError; - return { - error: { - ...error, - data: i18n.t( - actionType === "save" ? - "saveErrorNotification" - : "submitErrorNotification", - { - ns: "submit", - error: (error.data as any).detail || - (typeof error.data === 'object' ? JSON.stringify(error.data) : error.data), - }, - ), - }, - }; - } - - return { data: metadataResult }; - },*/ }), - /*submitFiles: build.mutation({ - async queryFn( - { actionType }, - queryApi, - _extraOptions, - fetchWithBQ, - ) { - const { metadata, deposit, files } = queryApi.getState() as SubmitData; - const filesToUpload = files.filter((f) => !f.submittedFile); - // formatting the file data can take a while, so in the meantime, we activate a spinner - filesToUpload.forEach((f) => - store.dispatch( - setFilesSubmitStatus({ - id: f.id as string, - progress: undefined, - status: "submitting", - }), - ), - ); - // then format and start submitting - const data = await formatFileData(metadata.id, filesToUpload); - console.log("Submitting files"); - - const filesResults = - Array.isArray(data) && - (await Promise.all( - data.map((file: any) => - fetchWithBQ({ - url: "file", - method: "POST", - data: file, - headers: { - "auth-env-name": deposit.config.target?.envName, - }, - }), - ), - )); - - console.log("Files server response:"); - console.log(filesResults); - - const filesErrors = - filesResults && - filesResults.filter((res: any) => res.error as FetchBaseQueryError); - if (Array.isArray(filesErrors) && filesErrors.length > 0) - return { - error: { - // lets just take the first error message - ...(filesErrors[0].error as FetchBaseQueryError), - data: i18n.t( - actionType === "save" ? - "saveFileErrorNotification" - : "submitFileErrorNotification", - { - ns: "submit", - error: (filesErrors[0].error as FetchBaseQueryError).data, - }, - ), - }, - }; - - if (actionType === "save") { - // show notice and enable form again after successful save - enqueueSnackbar( - i18n.t("saveSuccess", { - ns: "submit", - dateTime: moment().format("D-M-YYYY @ HH:mm"), - }), - { - variant: "success", - }, - ); - store.dispatch(setFormDisabled(false)); - } - - return { data: filesResults }; - }, - }),*/ }), }); -export const { useSubmitDataMutation, /*useSubmitFilesMutation*/ } = submitApi; +export const { useSubmitDataMutation } = submitApi; diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index 08a2b2b7..f9b8816b 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -4,36 +4,46 @@ import { setFilesSubmitStatus } from "./submitSlice"; import { setFileMeta } from "../files/filesSlice"; import { SelectedFile } from "../../types/Files"; import { enqueueSnackbar } from "notistack"; +import { getUser } from "@dans-framework/utils/user"; // Create a new tus upload -export const uploadFile = async (file: SelectedFile, sessionId: string) => { +export const uploadFile = async ( + file: SelectedFile, + sessionId: string, +) => { + // set a loader right away, while we wait for blob to be created + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + progress: 0, + status: "submitting", + }), + ); + const fetchedFile = await fetch(file.url); const fileBlob = await fetchedFile.blob(); console.log('upload fn') + console.log(file) + const upload = new tus.Upload(fileBlob, { - // endpoint: `${import.meta.env.VITE_PACKAGING_TARGET}/files`, - endpoint: 'https://tusd.tusdemo.net/files/', + endpoint: `${import.meta.env.VITE_PACKAGING_TARGET}/files`, + // endpoint: 'https://tusd.tusdemo.net/files/', retryDelays: [0, 3000, 5000, 10000, 20000], metadata: { fileName: file.name, fileId: file.id, datasetId: sessionId, }, - onBeforeRequest: () => { - console.log('starting TUS request to upload') - store.dispatch( - setFilesSubmitStatus({ - id: file.id, - progress: 0, - status: "submitting", - }), - ); - }, + storeFingerprintForResuming: false, onError: function (error) { // Display an error message - console.log('Failed because: ' + error) + console.log('Failed because: ' + error); + + enqueueSnackbar(`Error uploading ${file.name}`, { + variant: "error", + }); }, onShouldRetry: function (err, retryAttempt, _options) { console.log("Error", err) @@ -54,7 +64,7 @@ export const uploadFile = async (file: SelectedFile, sessionId: string) => { return true }, onProgress: (bytesUploaded, bytesTotal) => { - var percentage = parseFloat(((bytesUploaded / bytesTotal) * 100).toFixed(1)) || 0; + var percentage = parseFloat(((bytesUploaded / bytesTotal) * 100).toFixed(0)) || 0; store.dispatch( setFilesSubmitStatus({ id: file.id, @@ -63,8 +73,25 @@ export const uploadFile = async (file: SelectedFile, sessionId: string) => { }), ); }, - onSuccess: () => { - console.log('Download %s from %s', file.name, upload.url); + onSuccess: async () => { + const tusId = upload.url?.split('/').pop(); + const user = getUser(); + // Due to incomplete Python TUS implementation, + // we do an extra api PATCH call to the server to signal succesful upload + try { + const response = await fetch(`${import.meta.env.VITE_PACKAGING_TARGET}/inbox/files/${sessionId}/${tusId}`, { + method: 'PATCH', + headers: { + Authorization: `Bearer ${user?.access_token}`, + }, + }); + const json = await response.json(); + console.log(json); + } catch(error){ + console.error(error); + }; + + // set file status to success store.dispatch( setFileMeta({ id: file.id, @@ -85,10 +112,10 @@ export const uploadFile = async (file: SelectedFile, sessionId: string) => { upload.findPreviousUploads().then(function (previousUploads) { // Found previous uploads so we select the first one. if (previousUploads.length) { - upload.resumeFromPreviousUpload(previousUploads[0]) + upload.resumeFromPreviousUpload(previousUploads[0]); } // Start the upload - upload.start() + upload.start(); }); } \ No newline at end of file diff --git a/packages/utils/error/index.tsx b/packages/utils/error/index.tsx index 07d7e2ba..2955211f 100644 --- a/packages/utils/error/index.tsx +++ b/packages/utils/error/index.tsx @@ -10,7 +10,7 @@ import { import Alert from "@mui/material/Alert"; import Typography from "@mui/material/Typography"; import AlertTitle from "@mui/material/AlertTitle"; -import { User } from "oidc-client-ts"; +import { getUser } from "@dans-framework/utils/user"; /** * Log a warning and show a toast! @@ -92,18 +92,6 @@ export const CustomError = forwardRef( ); // Freshdesk ticketing system -const getUser = () => { - const oidcStorage = sessionStorage.getItem( - `oidc.user:${import.meta.env.VITE_OIDC_AUTHORITY}:${ - import.meta.env.VITE_OIDC_CLIENT_ID - }`, - ); - if (!oidcStorage) { - return null; - } - return User.fromStorageString(oidcStorage); -}; - const sendTicket = async (data: any) => { const encodedCredentials = btoa( `${import.meta.env.VITE_FRESHDESK_API_KEY}:X`, diff --git a/packages/utils/error/package.json b/packages/utils/error/package.json index 8e4e926d..d8dc60bd 100644 --- a/packages/utils/error/package.json +++ b/packages/utils/error/package.json @@ -9,8 +9,8 @@ "@reduxjs/toolkit": "^1.9.5", "@mui/material": "^5.14.3", "notistack": "^3.0.1", - "oidc-client-ts": "^2.2.4", - "react": "^18.2.0" + "react": "^18.2.0", + "@dans-framework/utils": "workspace:*" }, "devDependencies": { "@types/react": "^18.2.15" diff --git a/packages/utils/lib/index.ts b/packages/utils/lib/index.ts index 77183405..36232af6 100644 --- a/packages/utils/lib/index.ts +++ b/packages/utils/lib/index.ts @@ -2,4 +2,5 @@ export * from "../language"; export * from "../sitetitle"; export * from "../error"; export * from "../typescript"; -export * from "../preloader"; \ No newline at end of file +export * from "../preloader"; +export * from "../user"; \ No newline at end of file diff --git a/packages/utils/user/index.ts b/packages/utils/user/index.ts new file mode 100644 index 00000000..a7dc7b74 --- /dev/null +++ b/packages/utils/user/index.ts @@ -0,0 +1,13 @@ +import { User } from "oidc-client-ts"; + +export const getUser = () => { + const oidcStorage = sessionStorage.getItem( + `oidc.user:${import.meta.env.VITE_OIDC_AUTHORITY}:${ + import.meta.env.VITE_OIDC_CLIENT_ID + }`, + ); + if (!oidcStorage) { + return null; + } + return User.fromStorageString(oidcStorage); +}; \ No newline at end of file diff --git a/packages/utils/user/package.json b/packages/utils/user/package.json new file mode 100644 index 00000000..ae272707 --- /dev/null +++ b/packages/utils/user/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dans-framework/utils/user", + "main": "index.ts", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "dependencies": { + "oidc-client-ts": "^2.2.4" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2afeadf..6cce9e7b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -797,6 +797,9 @@ importers: packages/utils/error: dependencies: + '@dans-framework/utils': + specifier: workspace:* + version: link:.. '@mui/material': specifier: ^5.14.3 version: 5.14.3(@emotion/react@11.11.1)(@emotion/styled@11.11.0)(@types/react@18.2.15)(react-dom@18.2.0)(react@18.2.0) @@ -806,9 +809,6 @@ importers: notistack: specifier: ^3.0.1 version: 3.0.1(csstype@3.1.2)(react-dom@18.2.0)(react@18.2.0) - oidc-client-ts: - specifier: ^2.2.4 - version: 2.2.4 react: specifier: ^18.2.0 version: 18.2.0 @@ -833,6 +833,12 @@ importers: specifier: ^18.2.15 version: 18.2.15 + packages/utils/user: + dependencies: + oidc-client-ts: + specifier: ^2.2.4 + version: 2.2.4 + packages: /@aashutoshrathi/word-wrap@1.2.6: From e172f92de0c1ba10d0b487806822e7c3d09174af Mon Sep 17 00:00:00 2001 From: D-Unit Date: Wed, 10 Jul 2024 15:32:44 +0200 Subject: [PATCH 06/80] added more header data --- packages/deposit/src/features/files/FilesTable.tsx | 3 ++- packages/deposit/src/features/submit/Submit.tsx | 2 +- packages/deposit/src/features/submit/submitFile.ts | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 2ead30bd..4808e824 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -371,9 +371,10 @@ const UploadProgress = ({ file }: FileItemProps) => { const sessionId = useAppSelector(getSessionId); const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); + const formConfig = useAppSelector(getData); const handleSingleFileUpload = () => { - uploadFile(file, sessionId) + uploadFile(file, sessionId, formConfig.target?.envName) }; return ( diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 27a53e57..5122290d 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -153,7 +153,7 @@ const Submit = ({ selectedFiles.map( file => { // only call the upload function if file is not yet uploaded, or is not currently uploading const hasStatus = filesSubmitStatus.find( f => f.id === file.id); - return !file.submittedFile && !hasStatus && uploadFile(file, sessionId); + return !file.submittedFile && !hasStatus && uploadFile(file, sessionId, formConfig.target?.envName); }); } }), diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index f9b8816b..748e354a 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -10,6 +10,7 @@ import { getUser } from "@dans-framework/utils/user"; export const uploadFile = async ( file: SelectedFile, sessionId: string, + target: string = '' ) => { // set a loader right away, while we wait for blob to be created store.dispatch( @@ -83,6 +84,7 @@ export const uploadFile = async ( method: 'PATCH', headers: { Authorization: `Bearer ${user?.access_token}`, + "auth-env-name": target, }, }); const json = await response.json(); From 2324ff2de7a9a0be0d783a0f5c5b452f751c49cf Mon Sep 17 00:00:00 2001 From: D-Unit Date: Wed, 10 Jul 2024 15:37:41 +0200 Subject: [PATCH 07/80] temp reverted back to using auth for files --- packages/deposit/src/features/files/FilesTable.tsx | 3 +++ packages/deposit/src/features/submit/Submit.tsx | 1 + packages/deposit/src/features/submit/submitApi.ts | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 8b7a1f1d..d4ddac72 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -46,6 +46,7 @@ import { findFileGroup } from "./filesHelpers"; import { DatePicker } from '@mui/x-date-pickers/DatePicker'; import moment, { Moment } from "moment"; import type { DateValidationError } from "@mui/x-date-pickers/models"; +import { useAuth } from "react-oidc-context"; const FilesTable = () => { const { t } = useTranslation("files"); @@ -380,6 +381,7 @@ const UploadProgress = ({ file }: FileItemProps) => { const [submitFiles] = useSubmitFilesMutation(); const formConfig = useAppSelector(getData); const metadataSubmitStatus = useAppSelector(getMetadataSubmitStatus); + const auth = useAuth(); const handleSingleFileUpload = () => { formatFileData(sessionId, [file]).then((d) => { @@ -388,6 +390,7 @@ const UploadProgress = ({ file }: FileItemProps) => { data: d, headerData: { target: formConfig.target, + submitKey: auth.user?.access_token, }, actionType: metadataSubmitStatus === "saved" ? "save" : "submit", }); diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index bae4d326..4dac0c0d 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -148,6 +148,7 @@ const Submit = ({ // if metadata has been submitted ok, we start the file submit submitFiles({ actionType: actionType, + user: user, }); } }), diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index 628c024e..540b6bd2 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -197,7 +197,7 @@ export const submitApi = createApi({ }), submitFiles: build.mutation({ async queryFn( - { actionType }, + { user, actionType }, queryApi, _extraOptions, fetchWithBQ, @@ -227,6 +227,9 @@ export const submitApi = createApi({ method: "POST", data: file, headers: { + Authorization: `Bearer ${ + deposit.config.submitKey || user?.access_token + }`, "auth-env-name": deposit.config.target?.envName, }, }), From fb048b948c11c23fd36277823b34782a0b2812fa Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 10:26:41 +0200 Subject: [PATCH 08/80] TUS ready for demo testing --- .../src/features/files/FilesUpload.tsx | 21 +--- .../deposit/src/features/submit/Submit.tsx | 4 - .../deposit/src/features/submit/submitApi.ts | 2 +- .../deposit/src/features/submit/submitFile.ts | 106 ++++++++++++------ .../src/languages/locales/en/files.json | 5 +- .../src/languages/locales/en/submit.json | 5 +- .../src/languages/locales/nl/files.json | 5 +- .../src/languages/locales/nl/submit.json | 5 +- packages/utils/error/index.tsx | 2 +- 9 files changed, 88 insertions(+), 67 deletions(-) diff --git a/packages/deposit/src/features/files/FilesUpload.tsx b/packages/deposit/src/features/files/FilesUpload.tsx index 7c5d4b8e..8ea56b6a 100644 --- a/packages/deposit/src/features/files/FilesUpload.tsx +++ b/packages/deposit/src/features/files/FilesUpload.tsx @@ -32,9 +32,9 @@ import { getFormDisabled } from "../../deposit/depositSlice"; import { getSessionId } from "../metadata/metadataSlice"; // Temporary soft file limits in bytes +// Probably move this to form config, as it's target dependant const bytes = 1048576; -const lowerLimit = 1000 * bytes; -const upperLimit = 2000 * bytes; +const limit = 10000 * bytes; const FilesUpload = () => { const dispatch = useAppDispatch(); @@ -130,10 +130,7 @@ const FilesUpload = () => { const formDisabled = useAppSelector(getFormDisabled); - const filesTooBig = currentFiles.filter( - (f) => lowerLimit < f.size && f.size < upperLimit, - ); - const filesWayTooBig = currentFiles.filter((f) => upperLimit < f.size); + const filesTooBig = currentFiles.filter((f) => limit < f.size); return ( @@ -194,16 +191,8 @@ const FilesUpload = () => { - )} - {filesWayTooBig.length > 0 && ( - )} diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 5122290d..4d81557e 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -85,8 +85,6 @@ const Submit = ({ fileStatus === "error" && dispatch(setOpenTab(1)); }, [fileStatus]); - console.log('file status: ' + fileStatus) - const [ submitData, { @@ -147,8 +145,6 @@ const Submit = ({ files: selectedFiles, }).then((result: { data?: any; error?: any }) => { if (result.data?.status === "OK") { - console.log(selectedFiles) - console.log(filesSubmitStatus) // if metadata has been submitted ok, we start the file submit selectedFiles.map( file => { // only call the upload function if file is not yet uploaded, or is not currently uploading diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index e9f52714..9b2bcd23 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -68,7 +68,7 @@ export const submitApi = createApi({ if (arg.actionType === "save") { // show notice and enable form again after successful save enqueueSnackbar( - i18n.t("saveSuccess", { + i18n.t(arg.files.length === 0 ? "saveSuccess" : "saveFileSuccess", { ns: "submit", dateTime: moment().format("D-M-YYYY @ HH:mm"), }), diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index 748e354a..28d2d90c 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -5,6 +5,26 @@ import { setFileMeta } from "../files/filesSlice"; import { SelectedFile } from "../../types/Files"; import { enqueueSnackbar } from "notistack"; import { getUser } from "@dans-framework/utils/user"; +import i18n from "../../languages/i18n"; +import { sendTicket } from "@dans-framework/utils/error"; + +const manualError = async (fileName: string, fileId: string, error: any, type: string) => { + console.error("Error", error); + // Since this process is not connected to Redux, we manually + // display a message, send a Freshdesk ticket or error, and set file status to errored + let ticket; + if (import.meta.env.VITE_FRESHDESK_API_KEY) { + ticket = await sendTicket({error: error, function: type}); + } + enqueueSnackbar(`Uploading ${fileName} - ${error}`, { variant: "customError", ticket: ticket }); + store.dispatch( + setFilesSubmitStatus({ + id: fileId, + status: "error", + }), + ); +} + // Create a new tus upload export const uploadFile = async ( @@ -21,47 +41,49 @@ export const uploadFile = async ( }), ); + // convert file url to blob const fetchedFile = await fetch(file.url); const fileBlob = await fetchedFile.blob(); - console.log('upload fn') - console.log(file) - - + // TUS upload logic const upload = new tus.Upload(fileBlob, { endpoint: `${import.meta.env.VITE_PACKAGING_TARGET}/files`, - // endpoint: 'https://tusd.tusdemo.net/files/', - retryDelays: [0, 3000, 5000, 10000, 20000], + // retry 5 times on error + retryDelays: [1000, 5000, 10000, 20000, 30000], + // optional metadata for the file metadata: { fileName: file.name, fileId: file.id, datasetId: sessionId, }, + // no resume after leaving session, since we'd need to load the form first then storeFingerprintForResuming: false, onError: function (error) { - // Display an error message - console.log('Failed because: ' + error); - - enqueueSnackbar(`Error uploading ${file.name}`, { - variant: "error", - }); + manualError(file.name, file.id, error, 'onError function in TUS upload'); }, - onShouldRetry: function (err, retryAttempt, _options) { - console.log("Error", err) - console.log("Request", err.originalRequest) - console.log("Response", err.originalResponse) + onShouldRetry: function (error, retryAttempt, _options) { + console.error("Error", error) + console.log("Request", error.originalRequest) + console.log("Response", error.originalResponse) - var status = err.originalResponse ? err.originalResponse.getStatus() : 0 + var status = error.originalResponse ? error.originalResponse.getStatus() : 0 // Do not retry if the status is a 403. if (status === 403) { return false } - enqueueSnackbar(`Error uploading ${file.name}. Retrying... (${retryAttempt + 1})`, { - variant: "warning", - }); + enqueueSnackbar( + i18n.t("uploadRetry", { + ns: "submit", + file: file.name, + attempt: retryAttempt + 2, + }), + { + variant: "warning", + }, + ); - // For any other status code, tus-js-client should retry. + // For any other status code, we should retry. return true }, onProgress: (bytesUploaded, bytesTotal) => { @@ -89,24 +111,34 @@ export const uploadFile = async ( }); const json = await response.json(); console.log(json); + + // set file status to success + store.dispatch( + setFileMeta({ + id: file.id, + type: "submittedFile", + value: true, + }), + ); + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + status: "success", + }), + ); + enqueueSnackbar( + i18n.t("uploadSuccess", { + ns: "submit", + file: file.name, + }), + { + variant: "success", + }, + ); } catch(error){ - console.error(error); + // on error, file must be set to failed, as server can't processed it properly + manualError(file.name, file.id, error, 'error dispatching PATCH call to inbox/files/{sessionID}/{tusID}'); }; - - // set file status to success - store.dispatch( - setFileMeta({ - id: file.id, - type: "submittedFile", - value: true, - }), - ); - store.dispatch( - setFilesSubmitStatus({ - id: file.id, - status: "success", - }), - ); }, }); diff --git a/packages/deposit/src/languages/locales/en/files.json b/packages/deposit/src/languages/locales/en/files.json index 287c5d1c..72842700 100644 --- a/packages/deposit/src/languages/locales/en/files.json +++ b/packages/deposit/src/languages/locales/en/files.json @@ -38,9 +38,8 @@ "fileTypeCheckError": "Something went wrong checking the file type. Please try again.", "submittedFile": "File has been uploaded previously", "confirmDelete": "Confirm deletion", - "limitHeader": "Warning! Files larger than {{amount}} MB", - "lowerLimitDescription": "If you have a relatively high upload speed, as it should be with fibre and/or in an office setting, you should encounter no problem with this. It may still take up to 5 minutes to upload your files, though. Depending on circumstances (including your upload speed) the files may take too long to upload and you might encounter a server timeout error. If this happens, please <0>contact DANS directly.", - "upperLimitDescription": "It is likely that these files take too long to upload and you might encounter a server timeout error. Please <0>contact DANS directly about depositing these data files.", + "fileLimitHeader": "Warning! Files larger than {{amount}} MB", + "fileLimitDescription": "Unfortunately, we cannot handle files of this size automatically. Please <0>contact DANS directly.", "embargoDate": "Embargo date", "embargoDateDescription": "Fill this in if this file should only be available after a specific date", "minDate": "Must be a future date", diff --git a/packages/deposit/src/languages/locales/en/submit.json b/packages/deposit/src/languages/locales/en/submit.json index 092fcdee..f65902a4 100644 --- a/packages/deposit/src/languages/locales/en/submit.json +++ b/packages/deposit/src/languages/locales/en/submit.json @@ -4,7 +4,10 @@ "metadataSuccess": "All set, ready to submit!", "submitting": "Sending your data to the server...", "submitSuccess": "Your data has been submitted for processing", - "saveSuccess": "Succesfully saved on our server on {{dateTime}}", + "saveSuccess": "Metadata has succesfully been saved on our server on {{dateTime}}", + "uploadSuccess": "File {{file}} has been uploaded succesfully", + "uploadRetry": "Error uploading {{file}}. Retrying... (attempt {{attempt}})", + "saveFileSuccess": "Metadata has been saved on our server. Hold on while your files are being uploaded...", "submitErrorMetadata": "Oh no! Error sending data to the server", "submitErrorFiles": "Oh no! Error uploading your files. Please retry the failed files.", "reset": "Reset form", diff --git a/packages/deposit/src/languages/locales/nl/files.json b/packages/deposit/src/languages/locales/nl/files.json index a58ae120..8bf3594b 100644 --- a/packages/deposit/src/languages/locales/nl/files.json +++ b/packages/deposit/src/languages/locales/nl/files.json @@ -38,9 +38,8 @@ "fileTypeCheckError": "Er ging iets mis met het controleren van het bestandstype. Probeer het nog eens.", "submittedFile": "Bestand is al eerder geupload", "confirmDelete": "Bevestig verwijderen", - "limitHeader": "Waarschuwing! Bestanden groter danm {{amount}} MB", - "lowerLimitDescription": "If you have a relatively high upload speed, as it should be with fibre and/or in an office setting, you should encounter no problem with this. It may still take up to 5 minutes to upload your files, though. Depending on circumstances (including your upload speed) the files may take too long to upload and you might encounter a server timeout error. If this happens, please <0>contact DANS directly.", - "upperLimitDescription": "It is likely that these files take too long to upload and you might encounter a server timeout error. Please <0>contact DANS directly about depositing these data files.", + "fileLimitHeader": "Waarschuwing! Bestanden groter dan {{amount}} MB", + "fileLimitDescription": "Bestanden van deze grootte kunnen wij helaas niet goed verwerken. Neem s.v.p. direct <0>contact met DANS op.", "embargoDate": "Embargodatum", "embargoDateDescription": "Vul dit in als dit bestand pas beschikbaar mag zijn na een bepaalde datum", "minDate": "Moet een toekomstige datum zijn", diff --git a/packages/deposit/src/languages/locales/nl/submit.json b/packages/deposit/src/languages/locales/nl/submit.json index 94e648cd..19ce0a63 100644 --- a/packages/deposit/src/languages/locales/nl/submit.json +++ b/packages/deposit/src/languages/locales/nl/submit.json @@ -4,7 +4,10 @@ "metadataSuccess": "Helemaal klaar om te versturen!", "submitting": "Je data wordt upgeload, een moment geduld...", "submitSuccess": "Je data is ontvangen en wordt verwerkt", - "saveSuccess": "Succesvol op onze server opgeslagen op {{dateTime}}", + "saveSuccess": "Metadata is succesvol op onze server opgeslagen op {{dateTime}}", + "uploadSuccess": "Bestand {{file}} is succesvol geupload", + "uploadRetry": "Fout bij uploaden {{file}}. We proberen het nog een keer... (poging {{attempt}})", + "saveFileSuccess": "Metadata is succesvol op onze server opgeslagen op. Blijf even wachten totdat je bestanden geupload zijn...", "submitErrorMetadata": "Oh nee! Er is iets misgegaan bij het versturen van je metadata", "submitErrorFiles": "Oh nee! Er is iets misgegaan bij het uploaden van je bestanden. Probeer ze nog eens te uploaden.", "reset": "Reset formulier", diff --git a/packages/utils/error/index.tsx b/packages/utils/error/index.tsx index 2955211f..ed97a185 100644 --- a/packages/utils/error/index.tsx +++ b/packages/utils/error/index.tsx @@ -92,7 +92,7 @@ export const CustomError = forwardRef( ); // Freshdesk ticketing system -const sendTicket = async (data: any) => { +export const sendTicket = async (data: any) => { const encodedCredentials = btoa( `${import.meta.env.VITE_FRESHDESK_API_KEY}:X`, ); From f8824522204e60027f64ecf095a441f6fbdba665 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 10:33:04 +0200 Subject: [PATCH 09/80] ready to deploy ohsmart demo --- apps/ohsmart/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/ohsmart/index.html b/apps/ohsmart/index.html index 0887a759..29928a39 100644 --- a/apps/ohsmart/index.html +++ b/apps/ohsmart/index.html @@ -41,4 +41,5 @@ To create a production bundle, use `npm run build` or `yarn build`. --> - \ No newline at end of file + + From 7fcf17b99dd4de2e9092ee1ff646035b6452eca7 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 12:26:11 +0200 Subject: [PATCH 10/80] fixed autosave feature --- packages/deposit/src/features/submit/Submit.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 4d81557e..fd256a14 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -159,7 +159,15 @@ const Submit = ({ // Autosave functionality, debounced on metadata change const autoSave = useDebouncedCallback(() => { if (!formDisabled && isTouched) { - submitData({ user: auth.user, actionType: "save" }); + submitData({ + user: auth.user, + actionType: "save", + id: sessionId, + metadata: metadata, + config: formConfig, + // don't send along files on autosave + files: [], + }); } }, 2000); From ac3b21315dd20c3358c613ed1e19146f18cf618d Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 12:26:44 +0200 Subject: [PATCH 11/80] redeploy ohsmart --- apps/ohsmart/index.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/ohsmart/index.html b/apps/ohsmart/index.html index 29928a39..0887a759 100644 --- a/apps/ohsmart/index.html +++ b/apps/ohsmart/index.html @@ -41,5 +41,4 @@ To create a production bundle, use `npm run build` or `yarn build`. --> - - + \ No newline at end of file From d43a93fa8698107ca50f4713f72be87298b4b85d Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 12:49:09 +0200 Subject: [PATCH 12/80] removed autosave success snackbar msg --- packages/deposit/src/features/submit/Submit.tsx | 3 ++- packages/deposit/src/features/submit/submitApi.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index fd256a14..06fc85d6 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -165,8 +165,9 @@ const Submit = ({ id: sessionId, metadata: metadata, config: formConfig, - // don't send along files on autosave + // don't send along files on autosave and set flag autoSave, so we don't show a snackbar each time files: [], + autoSave: true, }); } }, 2000); diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index 9b2bcd23..3166b6be 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -65,7 +65,7 @@ export const submitApi = createApi({ store.dispatch(setMetadataSubmitStatus( arg.actionType === "save" ? "saved" : "submitted", )); - if (arg.actionType === "save") { + if (arg.actionType === "save" && !arg.autoSave) { // show notice and enable form again after successful save enqueueSnackbar( i18n.t(arg.files.length === 0 ? "saveSuccess" : "saveFileSuccess", { From 4f807fac981be13e10c13899788b3c402f490909 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 11 Jul 2024 12:49:20 +0200 Subject: [PATCH 13/80] redeploy ohsmart --- apps/ohsmart/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ohsmart/index.html b/apps/ohsmart/index.html index 0887a759..17279fb8 100644 --- a/apps/ohsmart/index.html +++ b/apps/ohsmart/index.html @@ -41,4 +41,4 @@ To create a production bundle, use `npm run build` or `yarn build`. --> - \ No newline at end of file + From c05c4e7066502d03215209de24b815f06e33a7e3 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Fri, 12 Jul 2024 11:24:49 +0200 Subject: [PATCH 14/80] added max upload file size to config --- apps/ohsmart/src/config/form.ts | 1 + .../src/features/files/FilesUpload.tsx | 29 +++++++++---------- .../src/languages/locales/en/files.json | 3 +- .../src/languages/locales/nl/files.json | 3 +- packages/deposit/src/types/Metadata.ts | 1 + 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/apps/ohsmart/src/config/form.ts b/apps/ohsmart/src/config/form.ts index 27ff248b..10862e2f 100644 --- a/apps/ohsmart/src/config/form.ts +++ b/apps/ohsmart/src/config/form.ts @@ -49,6 +49,7 @@ const form: FormConfig = { embargoDateMin: 2, // days in the future embargoDateMax: 18250, displayProcesses: false, + maxSize: 9985798963, // max file size that can be uploaded in bytes }, }; diff --git a/packages/deposit/src/features/files/FilesUpload.tsx b/packages/deposit/src/features/files/FilesUpload.tsx index 8ea56b6a..d255bb76 100644 --- a/packages/deposit/src/features/files/FilesUpload.tsx +++ b/packages/deposit/src/features/files/FilesUpload.tsx @@ -28,23 +28,28 @@ import type { import { v4 as uuidv4 } from "uuid"; import { useFetchSimpleListQuery } from "./api/dansFormats"; import { enqueueSnackbar } from "notistack"; -import { getFormDisabled } from "../../deposit/depositSlice"; +import { getFormDisabled, getData } from "../../deposit/depositSlice"; import { getSessionId } from "../metadata/metadataSlice"; -// Temporary soft file limits in bytes -// Probably move this to form config, as it's target dependant -const bytes = 1048576; -const limit = 10000 * bytes; - const FilesUpload = () => { const dispatch = useAppDispatch(); const currentFiles = useAppSelector(getFiles); const { t } = useTranslation("files"); const { data } = useFetchSimpleListQuery(null); + const formConfig = useAppSelector(getData); // Validate added files, needs to be synchronous, so no API calls possible here const fileValidator = (file: File) => { if (!file.name) return null; + + // No files over the file size limit set in formConfig + if (formConfig?.filesUpload?.maxSize && file.size > formConfig?.filesUpload?.maxSize) { + return { + code: "file-too-large", + message: t("fileTooLarge", { size: (formConfig?.filesUpload?.maxSize / 1073741824).toFixed(2) }), + }; + } + // No duplicate files const extensionIndex = file.name.lastIndexOf("."); const baseName = file.name.slice(0, extensionIndex); @@ -66,6 +71,7 @@ const FilesUpload = () => { message: t("fileAlreadyAdded", { file: file.name }), }; } + // No files with these file names if (file.name.indexOf("__generated__") !== -1) { return { @@ -73,6 +79,7 @@ const FilesUpload = () => { message: t("fileNotAllowed"), }; } + return null; }; @@ -130,8 +137,6 @@ const FilesUpload = () => { const formDisabled = useAppSelector(getFormDisabled); - const filesTooBig = currentFiles.filter((f) => limit < f.size); - return ( @@ -187,14 +192,6 @@ const FilesUpload = () => { title={t("fileTypeError")} /> )} - {filesTooBig.length > 0 && ( - - )} ); diff --git a/packages/deposit/src/languages/locales/en/files.json b/packages/deposit/src/languages/locales/en/files.json index 72842700..28275ebd 100644 --- a/packages/deposit/src/languages/locales/en/files.json +++ b/packages/deposit/src/languages/locales/en/files.json @@ -43,5 +43,6 @@ "embargoDate": "Embargo date", "embargoDateDescription": "Fill this in if this file should only be available after a specific date", "minDate": "Must be a future date", - "dateInvalid": "Invalid date" + "dateInvalid": "Invalid date", + "fileTooLarge": "Files over {{ size }} GB cannot be processed by the target repository" } \ No newline at end of file diff --git a/packages/deposit/src/languages/locales/nl/files.json b/packages/deposit/src/languages/locales/nl/files.json index 8bf3594b..0eb93637 100644 --- a/packages/deposit/src/languages/locales/nl/files.json +++ b/packages/deposit/src/languages/locales/nl/files.json @@ -43,5 +43,6 @@ "embargoDate": "Embargodatum", "embargoDateDescription": "Vul dit in als dit bestand pas beschikbaar mag zijn na een bepaalde datum", "minDate": "Moet een toekomstige datum zijn", - "dateInvalid": "Ongeldige datum" + "dateInvalid": "Ongeldige datum", + "fileTooLarge": "Bestanden groter dan {{ size }} GB kunnen niet worden verwerkt door het doelrepository" } \ No newline at end of file diff --git a/packages/deposit/src/types/Metadata.ts b/packages/deposit/src/types/Metadata.ts index a86aed39..331f51cc 100644 --- a/packages/deposit/src/types/Metadata.ts +++ b/packages/deposit/src/types/Metadata.ts @@ -66,5 +66,6 @@ export interface FormConfig { }[]; embargoDateMin?: number; embargoDateMax?: number; + maxSize?: number; }; } From f1c1220aec5a4333289270ca7140febec1819087 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Fri, 12 Jul 2024 11:26:47 +0200 Subject: [PATCH 15/80] added max size to docs --- docs/deposit.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deposit.md b/docs/deposit.md index ea14a41d..650fd8f2 100644 --- a/docs/deposit.md +++ b/docs/deposit.md @@ -91,6 +91,8 @@ import { Deposit } from "@dans-framework/deposit" embargoDate: false, displayRoles: true, displayProcesses: true, + // set an optional maximum file upload size in bytes + maxSize: 10737418240, // e.g. 10 GB }, } From 2af06a4ecc37a9a59395c0f417b0232b0e1401ba Mon Sep 17 00:00:00 2001 From: D-Unit Date: Mon, 15 Jul 2024 08:45:51 +0200 Subject: [PATCH 16/80] wip upload queue --- .../deposit/src/features/submit/Submit.tsx | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 06fc85d6..160df308 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -148,6 +148,7 @@ const Submit = ({ // if metadata has been submitted ok, we start the file submit selectedFiles.map( file => { // only call the upload function if file is not yet uploaded, or is not currently uploading + // TODO: modify this and file upload function to not upload more than X files in parallel const hasStatus = filesSubmitStatus.find( f => f.id === file.id); return !file.submittedFile && !hasStatus && uploadFile(file, sessionId, formConfig.target?.envName); }); @@ -378,10 +379,34 @@ const Submit = ({ t("resubmit") : t("submit")} + ); }; +const FileUploader = () => { + // component that listen to file upload status: + // check files that have status submitting, and start queueing them for upload + const maxConcurrentUploads = 6; + const filesStatus = useAppSelector(getFilesSubmitStatus); + const selectedFiles = useAppSelector(getFiles); + + console.log(filesStatus) + console.log(selectedFiles) + + useEffect(() => { + const currentlyUploading = filesStatus.filter( (file) => file.status === 'submitting'); + if (currentlyUploading.length < maxConcurrentUploads) { + // add first file of selectedFiles that is not currently uploading to queue + // TODO: maybe combine filesSubmitStatus with selectedFiles?? + // const addIdToUpload = selectedFiles + } + }, [filesStatus, selectedFiles]) + + return null; + +}; + export default Submit; From dfe127b569bf612fc47e2d1182dca6c83f27931b Mon Sep 17 00:00:00 2001 From: D-Unit Date: Mon, 15 Jul 2024 08:54:32 +0200 Subject: [PATCH 17/80] wip upload queue --- packages/deposit/src/features/submit/Submit.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 160df308..ac04270c 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -389,6 +389,9 @@ const Submit = ({ const FileUploader = () => { // component that listen to file upload status: // check files that have status submitting, and start queueing them for upload + // note: use filesStatus as the queue, selectedFiles as the queue source. Push one extra to queue when queue has less than X submitting + // or: add extra status type to queue, like readyToSubmit alongside submitting + // TODO refactor main submit code, fileUpload code const maxConcurrentUploads = 6; const filesStatus = useAppSelector(getFilesSubmitStatus); const selectedFiles = useAppSelector(getFiles); From b1acc063398a2432541ba958450874fa24620b8d Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 11:18:54 +0200 Subject: [PATCH 18/80] added upload queue for tus --- .../deposit/src/features/files/FilesTable.tsx | 16 ++++--- .../src/features/files/api/dansFormats.ts | 5 +- .../deposit/src/features/submit/Submit.tsx | 47 +++++++++++-------- packages/deposit/src/types/Submit.ts | 1 + packages/rdt-search-ui | 2 +- pnpm-lock.yaml | 24 ---------- 6 files changed, 42 insertions(+), 53 deletions(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 07e37cc4..87827762 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -27,11 +27,10 @@ import type { FileItemProps, FileActions, } from "../../types/Files"; -import { getSessionId } from "../metadata/metadataSlice"; import LinearProgress from "@mui/material/LinearProgress"; import Box from "@mui/material/Box"; import Stack from "@mui/material/Stack"; -import { getSingleFileSubmitStatus } from "../submit/submitSlice"; +import { getSingleFileSubmitStatus, setFilesSubmitStatus } from "../submit/submitSlice"; import { motion, AnimatePresence, HTMLMotionProps } from "framer-motion"; import { getFormDisabled, getData } from "../../deposit/depositSlice"; import FileStatusIndicator from "./FileStatusIndicator"; @@ -41,7 +40,6 @@ import { findFileGroup } from "./filesHelpers"; import { DatePicker } from '@mui/x-date-pickers/DatePicker'; import moment, { Moment } from "moment"; import type { DateValidationError } from "@mui/x-date-pickers/models"; -import { uploadFile } from "../submit/submitFile"; const FilesTable = () => { const { t } = useTranslation("files"); @@ -367,13 +365,17 @@ const FileTableRow = ({ file }: FileItemProps) => { }; const UploadProgress = ({ file }: FileItemProps) => { + const dispatch = useAppDispatch(); // We handle progress and manually retrying/restarting of file uploads here - const sessionId = useAppSelector(getSessionId); const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); - const formConfig = useAppSelector(getData); const handleSingleFileUpload = () => { - uploadFile(file, sessionId, formConfig.target?.envName) + console.log('retrying') + dispatch(setFilesSubmitStatus({ + id: file.id, + progress: 0, + status: "queued", + })); }; return ( @@ -400,7 +402,7 @@ const UploadProgress = ({ file }: FileItemProps) => { /> - {fileStatus.status === "submitting" && ( + {(fileStatus.status === "submitting" || fileStatus.status === "queued") && ( {`${ fileStatus.progress || 0 }%`} diff --git a/packages/deposit/src/features/files/api/dansFormats.ts b/packages/deposit/src/features/files/api/dansFormats.ts index b7fd826f..e2e13334 100644 --- a/packages/deposit/src/features/files/api/dansFormats.ts +++ b/packages/deposit/src/features/files/api/dansFormats.ts @@ -7,7 +7,10 @@ import type { export const dansFormatsApi = createApi({ reducerPath: "dansFormats", - baseQuery: fetchBaseQuery({ baseUrl: "https://type.labs.dans.knaw.nl" }), + baseQuery: fetchBaseQuery({ + baseUrl: import.meta.env.VITE_FORMATS_API || + "https://type.labs.dans.knaw.nl" + }), endpoints: (build) => ({ fetchDansFormats: build.query({ query: () => ({ diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index ac04270c..ea1397f9 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -24,6 +24,7 @@ import { uploadFile } from "./submitFile"; import { setMetadataSubmitStatus, getMetadataSubmitStatus, + setFilesSubmitStatus, getFilesSubmitStatus, resetFilesSubmitStatus, resetMetadataSubmitStatus, @@ -76,7 +77,7 @@ const Submit = ({ const fileStatusArray = [...new Set(filesSubmitStatus.map((f) => f.status))]; const fileStatus = fileStatusArray.indexOf("error") !== -1 ? "error" - : fileStatusArray.indexOf("submitting") !== -1 ? "submitting" + : fileStatusArray.indexOf("submitting") !== -1 || fileStatusArray.indexOf("queued") !== -1 ? "submitting" : fileStatusArray.indexOf("success") !== -1 ? "success" : ""; @@ -146,12 +147,19 @@ const Submit = ({ }).then((result: { data?: any; error?: any }) => { if (result.data?.status === "OK") { // if metadata has been submitted ok, we start the file submit - selectedFiles.map( file => { + selectedFiles.map( file => + !file.submittedFile && dispatch( + setFilesSubmitStatus({ + id: file.id, + progress: 0, + status: "queued", + }), + ) // only call the upload function if file is not yet uploaded, or is not currently uploading // TODO: modify this and file upload function to not upload more than X files in parallel - const hasStatus = filesSubmitStatus.find( f => f.id === file.id); - return !file.submittedFile && !hasStatus && uploadFile(file, sessionId, formConfig.target?.envName); - }); + // const hasStatus = filesSubmitStatus.find( f => f.id === file.id); + // return !file.submittedFile && !hasStatus && uploadFile(file, sessionId, formConfig.target?.envName); + ); } }), ); @@ -387,26 +395,25 @@ const Submit = ({ }; const FileUploader = () => { - // component that listen to file upload status: - // check files that have status submitting, and start queueing them for upload - // note: use filesStatus as the queue, selectedFiles as the queue source. Push one extra to queue when queue has less than X submitting - // or: add extra status type to queue, like readyToSubmit alongside submitting - // TODO refactor main submit code, fileUpload code - const maxConcurrentUploads = 6; - const filesStatus = useAppSelector(getFilesSubmitStatus); + // component that manages file upload queue + // Check files that have status queued, and start uploading when a spot becomes available in the queue + const maxConcurrentUploads = 3; + const filesSubmitStatus = useAppSelector(getFilesSubmitStatus); const selectedFiles = useAppSelector(getFiles); - - console.log(filesStatus) - console.log(selectedFiles) + const sessionId = useAppSelector(getSessionId); + const formConfig = useAppSelector(getData); useEffect(() => { - const currentlyUploading = filesStatus.filter( (file) => file.status === 'submitting'); + const currentlyUploading = filesSubmitStatus.filter(file => file.status === 'submitting'); if (currentlyUploading.length < maxConcurrentUploads) { - // add first file of selectedFiles that is not currently uploading to queue - // TODO: maybe combine filesSubmitStatus with selectedFiles?? - // const addIdToUpload = selectedFiles + // add first file of selectedFiles that is not currently uploading to the active uploads + selectedFiles.find(file => { + // only call the upload function if file is queued + const hasStatus = filesSubmitStatus.find( f => f.id === file.id); + return hasStatus?.status === "queued" && uploadFile(file, sessionId, formConfig.target?.envName); + }); } - }, [filesStatus, selectedFiles]) + }, [filesSubmitStatus, selectedFiles, sessionId, formConfig]) return null; diff --git a/packages/deposit/src/types/Submit.ts b/packages/deposit/src/types/Submit.ts index bc9f1b5d..d3911040 100644 --- a/packages/deposit/src/types/Submit.ts +++ b/packages/deposit/src/types/Submit.ts @@ -17,6 +17,7 @@ export interface SubmittedFile { export type SubmitStatus = | "" + | "queued" | "submitting" | "submitted" | "error" diff --git a/packages/rdt-search-ui b/packages/rdt-search-ui index 7ec4e2c0..dce9778b 160000 --- a/packages/rdt-search-ui +++ b/packages/rdt-search-ui @@ -1 +1 @@ -Subproject commit 7ec4e2c0d81dea21de8380b7386606b6380553de +Subproject commit dce9778b2e63f276b8a8e639329e7a649076ecda diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6cce9e7b..778613b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -621,9 +621,6 @@ importers: packages/rdt-search-ui: dependencies: - '@dans-framework/utils': - specifier: workspace:* - version: link:../utils '@elastic/elasticsearch': specifier: ^8.10.0 version: 8.10.0 @@ -639,18 +636,9 @@ importers: echarts: specifier: ^5.4.2 version: 5.4.3 - framer-motion: - specifier: ^10.15.0 - version: 10.15.0(react-dom@18.2.0)(react@18.2.0) html-react-parser: specifier: ^4.2.2 version: 4.2.2(react@18.2.0) - i18next: - specifier: ^23.4.1 - version: 23.4.1 - i18next-resources-to-backend: - specifier: ^1.1.4 - version: 1.1.4 lodash.debounce: specifier: ^4.0.8 version: 4.0.8 @@ -660,21 +648,12 @@ importers: ngeohash: specifier: ^0.6.3 version: 0.6.3 - notistack: - specifier: ^3.0.1 - version: 3.0.1(csstype@3.1.2)(react-dom@18.2.0)(react@18.2.0) ol: specifier: ^7.3.0 version: 7.5.2 react: specifier: ^18.2.0 version: 18.2.0 - react-i18next: - specifier: ^13.0.3 - version: 13.0.3(i18next@23.4.1)(react-dom@18.2.0)(react@18.2.0) - react-router-dom: - specifier: ^6.14.2 - version: 6.14.2(react-dom@18.2.0)(react@18.2.0) styled-components: specifier: ^5.3.10 version: 5.3.10(@babel/core@7.22.10)(react-dom@18.2.0)(react-is@18.2.0)(react@18.2.0) @@ -706,9 +685,6 @@ importers: typescript: specifier: ^5.0.3 version: 5.1.6 - vite: - specifier: ^4.4.5 - version: 4.4.5(@types/node@18.0.6) packages/theme: dependencies: From b9c256d4dbe2fc4b33c632ea47024914934dec61 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 11:29:14 +0200 Subject: [PATCH 19/80] added extra var to env --- apps/ohsmart/.env | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/ohsmart/.env b/apps/ohsmart/.env index 98552118..363e3991 100644 --- a/apps/ohsmart/.env +++ b/apps/ohsmart/.env @@ -31,3 +31,4 @@ VITE_FRESHDESK_ASSIGNED_TO = '' ## For dev purposes VITE_DISABLE_AUTOSAVE = '' VITE_DISABLE_API_KEY_MESSAGE = '' +VITE_FORMATS_API = '' From 22e9417fc448dd94a5eb249625eb90bf41079553 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 11:33:50 +0200 Subject: [PATCH 20/80] cleanup comments --- packages/deposit/src/features/submit/Submit.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index ea1397f9..95058d7a 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -155,10 +155,6 @@ const Submit = ({ status: "queued", }), ) - // only call the upload function if file is not yet uploaded, or is not currently uploading - // TODO: modify this and file upload function to not upload more than X files in parallel - // const hasStatus = filesSubmitStatus.find( f => f.id === file.id); - // return !file.submittedFile && !hasStatus && uploadFile(file, sessionId, formConfig.target?.envName); ); } }), @@ -395,8 +391,8 @@ const Submit = ({ }; const FileUploader = () => { - // component that manages file upload queue - // Check files that have status queued, and start uploading when a spot becomes available in the queue + // Component that manages file upload queue. + // Check files that have status queued, and start uploading when a spot becomes available in the queue. const maxConcurrentUploads = 3; const filesSubmitStatus = useAppSelector(getFilesSubmitStatus); const selectedFiles = useAppSelector(getFiles); From b9f5f31805dccb5c739c0ceb0cde13733649b4a2 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 11:36:07 +0200 Subject: [PATCH 21/80] cleanup comments --- packages/deposit/src/features/submit/submitFile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index 28d2d90c..37a1fd88 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -32,7 +32,7 @@ export const uploadFile = async ( sessionId: string, target: string = '' ) => { - // set a loader right away, while we wait for blob to be created + // set file status to submitting, to add it to actual upload queue, while we create the blob store.dispatch( setFilesSubmitStatus({ id: file.id, From cf8e26b38201f1aeb9a19c34d318a01a55ba2f39 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 12:21:15 +0200 Subject: [PATCH 22/80] fixed error response --- apps/ohsmart/index.html | 1 + .../deposit/src/features/submit/submitApi.ts | 17 ++++++----------- .../deposit/src/features/submit/submitFile.ts | 11 +++++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/ohsmart/index.html b/apps/ohsmart/index.html index 17279fb8..29928a39 100644 --- a/apps/ohsmart/index.html +++ b/apps/ohsmart/index.html @@ -42,3 +42,4 @@ --> + diff --git a/packages/deposit/src/features/submit/submitApi.ts b/packages/deposit/src/features/submit/submitApi.ts index 3166b6be..1c1b48d1 100644 --- a/packages/deposit/src/features/submit/submitApi.ts +++ b/packages/deposit/src/features/submit/submitApi.ts @@ -80,18 +80,13 @@ export const submitApi = createApi({ return response; }, transformErrorResponse: (response: FetchBaseQueryError) => { - // enable form again if there's an error, so user can try and resubmit + // flag submit as failed + store.dispatch(setMetadataSubmitStatus("error")); + // enable form again, so user can try and resubmit store.dispatch(setFormDisabled(false)); - return { - error: { - ...response, - data: i18n.t("submitMetadataError", { - ns: "submit", - error: (response.data as any).detail || - (typeof response.data === 'object' ? JSON.stringify(response.data) : response.data), - }), - }, - }; + return ({ + error: i18n.t("submit:submitMetadataError", { error: response.status }) + }); }, }), }), diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index 37a1fd88..20474757 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -43,6 +43,17 @@ export const uploadFile = async ( // convert file url to blob const fetchedFile = await fetch(file.url); + + if (!fetchedFile.ok) { + throw new Error(`Failed to fetch file: ${fetchedFile.statusText}`); + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + status: "error", + }), + ); + } + const fileBlob = await fetchedFile.blob(); // TUS upload logic From fe789eaa85cebe17a36d2500723c9bfd5472f8b8 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 14:12:06 +0200 Subject: [PATCH 23/80] fixed tus upload bug --- packages/deposit/src/features/files/FilesTable.tsx | 6 ++++++ packages/deposit/src/features/submit/Submit.tsx | 8 +++++--- packages/deposit/src/features/submit/submitFile.ts | 13 +++++++++---- .../deposit/src/languages/locales/en/files.json | 1 + .../deposit/src/languages/locales/nl/files.json | 1 + packages/deposit/src/types/Submit.ts | 1 + 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 87827762..64c6a5db 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -28,6 +28,7 @@ import type { FileActions, } from "../../types/Files"; import LinearProgress from "@mui/material/LinearProgress"; +import CircularProgress from "@mui/material/CircularProgress"; import Box from "@mui/material/Box"; import Stack from "@mui/material/Stack"; import { getSingleFileSubmitStatus, setFilesSubmitStatus } from "../submit/submitSlice"; @@ -407,6 +408,11 @@ const UploadProgress = ({ file }: FileItemProps) => { fileStatus.progress || 0 }%`} )} + {fileStatus.status === "finalising" && ( + + + + )} {fileStatus.status === "success" && ( diff --git a/packages/deposit/src/features/submit/Submit.tsx b/packages/deposit/src/features/submit/Submit.tsx index 95058d7a..22094134 100644 --- a/packages/deposit/src/features/submit/Submit.tsx +++ b/packages/deposit/src/features/submit/Submit.tsx @@ -147,15 +147,17 @@ const Submit = ({ }).then((result: { data?: any; error?: any }) => { if (result.data?.status === "OK") { // if metadata has been submitted ok, we start the file submit - selectedFiles.map( file => - !file.submittedFile && dispatch( + selectedFiles.map( file => { + const hasStatus = filesSubmitStatus.find( f => f.id === file.id); + // make sure file is not already submitted or currently submitting + return !file.submittedFile && (!hasStatus || hasStatus?.status === 'error') && dispatch( setFilesSubmitStatus({ id: file.id, progress: 0, status: "queued", }), ) - ); + }); } }), ); diff --git a/packages/deposit/src/features/submit/submitFile.ts b/packages/deposit/src/features/submit/submitFile.ts index 20474757..8d79e689 100644 --- a/packages/deposit/src/features/submit/submitFile.ts +++ b/packages/deposit/src/features/submit/submitFile.ts @@ -25,7 +25,6 @@ const manualError = async (fileName: string, fileId: string, error: any, type: s ); } - // Create a new tus upload export const uploadFile = async ( file: SelectedFile, @@ -67,8 +66,6 @@ export const uploadFile = async ( fileId: file.id, datasetId: sessionId, }, - // no resume after leaving session, since we'd need to load the form first then - storeFingerprintForResuming: false, onError: function (error) { manualError(file.name, file.id, error, 'onError function in TUS upload'); }, @@ -111,7 +108,15 @@ export const uploadFile = async ( const tusId = upload.url?.split('/').pop(); const user = getUser(); // Due to incomplete Python TUS implementation, - // we do an extra api PATCH call to the server to signal succesful upload + // we do an extra api PATCH call to the server to signal succesful upload. + // Response might take a while, so lets display a spinner that informs the user + store.dispatch( + setFilesSubmitStatus({ + id: file.id, + status: "finalising", + }), + ); + try { const response = await fetch(`${import.meta.env.VITE_PACKAGING_TARGET}/inbox/files/${sessionId}/${tusId}`, { method: 'PATCH', diff --git a/packages/deposit/src/languages/locales/en/files.json b/packages/deposit/src/languages/locales/en/files.json index 28275ebd..437f17b3 100644 --- a/packages/deposit/src/languages/locales/en/files.json +++ b/packages/deposit/src/languages/locales/en/files.json @@ -32,6 +32,7 @@ "fileRenamed": "A file with the same name has already been added. We've renamed your file to {{file}}", "fileAlreadyAdded": "File {{file}} has already been added", "fileNotAllowed": "This file name is not allowed. Please rename your file.", + "fileSubmitWaiting": "Server is busy processing file", "fileSubmitSuccess": "Submitted successfully", "uploadFailed": "Failed", "fileSubmitError": "Try uploading again", diff --git a/packages/deposit/src/languages/locales/nl/files.json b/packages/deposit/src/languages/locales/nl/files.json index 0eb93637..03bc2b20 100644 --- a/packages/deposit/src/languages/locales/nl/files.json +++ b/packages/deposit/src/languages/locales/nl/files.json @@ -32,6 +32,7 @@ "fileRenamed": "Een bestand met dezelfde naam is al toegevoegd. We hebben je bestand hernoemd naar {{file}}", "fileAlreadyAdded": "Bestand {{file}} is al toegevoegd", "fileNotAllowed": "Deze bestandsnaam is niet toegestaan. Hernoem je bestand.", + "fileSubmitWaiting": "Server is bezig met verwerken bestand", "fileSubmitSuccess": "Successvol verstuurd", "uploadFailed": "Gefaald", "fileSubmitError": "Probeer opnieuw te uploaden", diff --git a/packages/deposit/src/types/Submit.ts b/packages/deposit/src/types/Submit.ts index d3911040..f87a1d46 100644 --- a/packages/deposit/src/types/Submit.ts +++ b/packages/deposit/src/types/Submit.ts @@ -22,6 +22,7 @@ export type SubmitStatus = | "submitted" | "error" | "saved" + | "finalising" | "success"; export interface ReduxFileSubmitActions { From 439104c70169196b2de14ee797c740f1ef66b247 Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 14:28:03 +0200 Subject: [PATCH 24/80] removed console log --- packages/deposit/src/features/files/FilesTable.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/deposit/src/features/files/FilesTable.tsx b/packages/deposit/src/features/files/FilesTable.tsx index 64c6a5db..13cf3d11 100644 --- a/packages/deposit/src/features/files/FilesTable.tsx +++ b/packages/deposit/src/features/files/FilesTable.tsx @@ -371,7 +371,6 @@ const UploadProgress = ({ file }: FileItemProps) => { const fileStatus = useAppSelector(getSingleFileSubmitStatus(file.id)); const { t } = useTranslation("files"); const handleSingleFileUpload = () => { - console.log('retrying') dispatch(setFilesSubmitStatus({ id: file.id, progress: 0, From 31478aa9c4bfe7822c83aad8300879180bb8de6e Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 15 Jul 2024 14:29:06 +0200 Subject: [PATCH 25/80] deploy ohsmart --- apps/ohsmart/index.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/ohsmart/index.html b/apps/ohsmart/index.html index 29928a39..0887a759 100644 --- a/apps/ohsmart/index.html +++ b/apps/ohsmart/index.html @@ -41,5 +41,4 @@ To create a production bundle, use `npm run build` or `yarn build`. --> - - + \ No newline at end of file From 6b13ed09650457b29c023e6e8ce760a0fa580c4f Mon Sep 17 00:00:00 2001 From: D-Unit Date: Wed, 17 Jul 2024 13:43:22 +0200 Subject: [PATCH 26/80] started work on 4tu app to include repo advisor preform --- apps/4tu/package.json | 1 + apps/4tu/src/App.tsx | 73 +++--- apps/4tu/src/config/pages/RepoAdvisor.tsx | 275 ++++++++++++++++++++++ apps/4tu/src/config/pages/deposit.ts | 2 +- packages/deposit/lib/index.tsx | 1 + packages/rdt-search-ui | 2 +- pnpm-lock.yaml | 29 ++- 7 files changed, 348 insertions(+), 35 deletions(-) create mode 100644 apps/4tu/src/config/pages/RepoAdvisor.tsx diff --git a/apps/4tu/package.json b/apps/4tu/package.json index afa10021..462d1c22 100644 --- a/apps/4tu/package.json +++ b/apps/4tu/package.json @@ -22,6 +22,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.14.2", + "use-debounce": "^9.0.4", "web-vitals": "^3.4.0" }, "devDependencies": { diff --git a/apps/4tu/src/App.tsx b/apps/4tu/src/App.tsx index 26d4eaeb..5007d734 100644 --- a/apps/4tu/src/App.tsx +++ b/apps/4tu/src/App.tsx @@ -1,11 +1,10 @@ -import { Suspense } from "react"; +import { Suspense, useState } from "react"; import { BrowserRouter, Routes, Route } from "react-router-dom"; import Skeleton from "@mui/material/Skeleton"; import Box from "@mui/material/Box"; import { ThemeWrapper } from "@dans-framework/theme"; import { MenuBar, Footer } from "@dans-framework/layout"; import { Deposit } from "@dans-framework/deposit"; -import { Generic, type Page } from "@dans-framework/pages"; import { AuthWrapper, AuthRoute, @@ -13,21 +12,22 @@ import { UserSubmissions, SignInCallback, } from "@dans-framework/user-auth"; +import RepoAdvisor from './config/pages/RepoAdvisor'; // Load config variables import theme from "./config/theme"; import footer from "./config/footer"; -import pages from "./config/pages"; import siteTitle from "./config/siteTitle"; import authProvider from "./config/auth"; import form from "./config/form"; const App = () => { + const [ repoConfig, setRepoConfig ] = useState(); return ( - + {/* Suspense to make sure languages can load first */} { > } /> + {repoConfig ? [ + + + + } + />, + + + + } + />, + + + + } + /> + ] : - + } /> - - - - } - /> - {(pages as Page[]).map((page) => { - return ( - - - - : - } - /> - ); - })} + } diff --git a/apps/4tu/src/config/pages/RepoAdvisor.tsx b/apps/4tu/src/config/pages/RepoAdvisor.tsx new file mode 100644 index 00000000..15db5e0b --- /dev/null +++ b/apps/4tu/src/config/pages/RepoAdvisor.tsx @@ -0,0 +1,275 @@ +import { useState, useEffect, type Dispatch, type SetStateAction } from "react"; +import Stack from '@mui/material/Stack'; +import CircularProgress from '@mui/material/CircularProgress'; +import Paper from '@mui/material/Paper'; +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import Grid from "@mui/material/Unstable_Grid2"; +import Typography from "@mui/material/Typography"; +import Container from "@mui/material/Container"; +import { useDebounce } from "use-debounce"; +import InputLabel from '@mui/material/InputLabel'; +import MenuItem from '@mui/material/MenuItem'; +import FormControl from '@mui/material/FormControl'; +import Select, { SelectChangeEvent } from '@mui/material/Select'; +import TextField from '@mui/material/TextField'; +import Autocomplete from '@mui/material/Autocomplete'; +import type { AutocompleteAPIFieldData } from "@dans-framework/deposit"; + +const RepoAdvisor = ({setRepoConfig}: {setRepoConfig: Dispatch>}) => { + const [recommendations, setRecommendations] = useState([]); + const [ror, setRor] = useState(null); + const [narcis, setNarcis] = useState(null); + const [depositType, setDepositType] = useState(''); + const [fileType, setFileType] = useState(''); + + const fetchRecommendations = () => { + // fetch the recommendations list + const result = ['something', 'other']; + + // set state accordingly + setRecommendations(result); + } + + const resetRecommendations = () => { + setRecommendations([]); + } + + return ( + + + + + + Repository advisor + + + Fill out the form below to get recommendations on where to submit your data. + + 0} + /> + 0} + /> + 0} + /> + {depositType === 'dataset' && + 0} + /> + } + + + + + {recommendations.length > 0 && + + + Recommended repositories + + {recommendations.map( rec => + + + + )} + + + + + } + + + + ); +}; + +type Option = { + value: string; + label: string; +}; + +const SelectField = ({label, value, onChange, options, disabled}: { + label: string; + value: string; + onChange: (s: string) => void; + options: Option[]; + disabled: boolean; +}) => + + + {label} + + + + +// Basic API fetching function, based on the API calls in the Deposit package +const fetchData = async ( + type: string, + setData: (option: AutocompleteAPIFieldData) => void, + debouncedInputValue: string, + setLoading: (b: boolean) => void +) => { + const uri = + type === 'ror' ? + `https://api.ror.org/organizations?query.advanced=name:${debouncedInputValue}*` : + type === 'narcis' ? + `https://vocabs.datastations.nl/rest/v1/NARCIS/search?query=${debouncedInputValue}*&unique=true&lang=en` : + ''; + try { + const result = await fetch(uri, { + headers: { Accept: "application/json" } + }); + const json = await result.json(); + const transformResult = + type === 'ror' ? + (json.number_of_results > 0 ? + { + arg: debouncedInputValue, + response: json.items.map((item: any) => ({ + label: item.name, + value: item.id, + extraLabel: "country", + extraContent: item.country.country_name, + })) + } : + [] + ) : + type === 'narcis' ? + (json.results.length > 0 ? + { + arg: debouncedInputValue, + response: json.results.map((item: any) => ({ + label: item.prefLabel, + value: item.uri, + id: item.localname, + })).filter(Boolean), + } : + [] + ) : + []; + setData(transformResult as AutocompleteAPIFieldData); + } catch (error) { + console.error(error); + } + setLoading(false); +} + +// Derived from the API field in the Deposit package +const ApiField = ({type, label, value, setValue, disabled}: { + type: string; + label: string; + value: Option | null; + setValue: Dispatch>; + disabled: boolean; +}) => { + const [inputValue, setInputValue] = useState(""); + const [data, setData] = useState(); + const [loading, setLoading] = useState(false); + const debouncedInputValue = useDebounce(inputValue, 500)[0]; + + useEffect( () => { + debouncedInputValue && fetchData( + type, + setData, + debouncedInputValue, + setLoading + ); + }, [debouncedInputValue]); + + useEffect( () => { + inputValue && setLoading(true); + }, [inputValue]); + + return ( + + ( + + )} + onChange={(_e, newValue, _reason) => { + setValue(newValue); + }} + filterOptions={(x) => x} + onInputChange={(e, newValue) => { + e && e.type === "change" && setInputValue(newValue); + e && (e.type === "click" || e.type === "blur") && setInputValue(""); + }} + noOptionsText={!inputValue ? "Start typing..." : "No results found"} + loading={loading} + loadingText={ + + Loading... + + } + forcePopupIcon + isOptionEqualToValue={(option, value) => option.value === value.value} + clearOnBlur + disabled={disabled} + /> + + ); +}; + +export default RepoAdvisor; diff --git a/apps/4tu/src/config/pages/deposit.ts b/apps/4tu/src/config/pages/deposit.ts index 7dd528b6..bbe139bb 100644 --- a/apps/4tu/src/config/pages/deposit.ts +++ b/apps/4tu/src/config/pages/deposit.ts @@ -3,7 +3,7 @@ import type { Page } from "@dans-framework/pages"; const page: Page = { id: "deposit", name: "Deposit", - slug: "", + slug: "deposit", template: "deposit", inMenu: false, menuTitle: "Deposit", diff --git a/packages/deposit/lib/index.tsx b/packages/deposit/lib/index.tsx index b91d5ba1..0780e4df 100644 --- a/packages/deposit/lib/index.tsx +++ b/packages/deposit/lib/index.tsx @@ -1,3 +1,4 @@ export { default as Deposit } from "../src"; export { default as i18n } from "../src/languages/i18n"; export type { FormConfig, InitialSectionType } from "../src/types/Metadata"; +export type { AutocompleteAPIFieldData } from "../src/types/MetadataFields"; \ No newline at end of file diff --git a/packages/rdt-search-ui b/packages/rdt-search-ui index dce9778b..7ec4e2c0 160000 --- a/packages/rdt-search-ui +++ b/packages/rdt-search-ui @@ -1 +1 @@ -Subproject commit dce9778b2e63f276b8a8e639329e7a649076ecda +Subproject commit 7ec4e2c0d81dea21de8380b7386606b6380553de diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 778613b1..47ea8ea3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: react-router-dom: specifier: ^6.14.2 version: 6.14.2(react-dom@18.2.0)(react@18.2.0) + use-debounce: + specifier: ^9.0.4 + version: 9.0.4(react@18.2.0) web-vitals: specifier: ^3.4.0 version: 3.4.0 @@ -621,6 +624,9 @@ importers: packages/rdt-search-ui: dependencies: + '@dans-framework/utils': + specifier: workspace:* + version: link:../utils '@elastic/elasticsearch': specifier: ^8.10.0 version: 8.10.0 @@ -636,9 +642,18 @@ importers: echarts: specifier: ^5.4.2 version: 5.4.3 + framer-motion: + specifier: ^10.15.0 + version: 10.15.0(react-dom@18.2.0)(react@18.2.0) html-react-parser: specifier: ^4.2.2 version: 4.2.2(react@18.2.0) + i18next: + specifier: ^23.4.1 + version: 23.4.1 + i18next-resources-to-backend: + specifier: ^1.1.4 + version: 1.1.4 lodash.debounce: specifier: ^4.0.8 version: 4.0.8 @@ -648,12 +663,21 @@ importers: ngeohash: specifier: ^0.6.3 version: 0.6.3 + notistack: + specifier: ^3.0.1 + version: 3.0.1(csstype@3.1.2)(react-dom@18.2.0)(react@18.2.0) ol: specifier: ^7.3.0 version: 7.5.2 react: specifier: ^18.2.0 version: 18.2.0 + react-i18next: + specifier: ^13.0.3 + version: 13.0.3(i18next@23.4.1)(react-dom@18.2.0)(react@18.2.0) + react-router-dom: + specifier: ^6.14.2 + version: 6.14.2(react-dom@18.2.0)(react@18.2.0) styled-components: specifier: ^5.3.10 version: 5.3.10(@babel/core@7.22.10)(react-dom@18.2.0)(react-is@18.2.0)(react@18.2.0) @@ -685,6 +709,9 @@ importers: typescript: specifier: ^5.0.3 version: 5.1.6 + vite: + specifier: ^4.4.5 + version: 4.4.5(@types/node@18.0.6) packages/theme: dependencies: @@ -1977,7 +2004,7 @@ packages: peerDependencies: react: ^17.0.0 || ^18.0.0 dependencies: - '@babel/runtime': 7.22.10 + '@babel/runtime': 7.23.2 '@types/prop-types': 15.7.9 '@types/react-is': 18.2.1 prop-types: 15.8.1 From 767c6e92a46a47652b933eaf8dab1d9939470eb4 Mon Sep 17 00:00:00 2001 From: D-Unit Date: Thu, 18 Jul 2024 10:22:04 +0200 Subject: [PATCH 27/80] 4tu work in progress --- apps/4tu/package.json | 2 + apps/4tu/src/App.tsx | 11 +- apps/4tu/src/config/pages/RepoAdvisor.tsx | 197 ++++++++++------------ apps/4tu/src/config/pages/apiHelpers.ts | 73 ++++++++ pnpm-lock.yaml | 6 + 5 files changed, 181 insertions(+), 108 deletions(-) create mode 100644 apps/4tu/src/config/pages/apiHelpers.ts diff --git a/apps/4tu/package.json b/apps/4tu/package.json index 462d1c22..37e3a6e4 100644 --- a/apps/4tu/package.json +++ b/apps/4tu/package.json @@ -18,7 +18,9 @@ "@dans-framework/utils": "workspace:*", "@fontsource/roboto": "^5.0.7", "@mui/material": "^5.14.3", + "framer-motion": "^10.15.0", "lz-string": "^1.5.0", + "notistack": "^3.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.14.2", diff --git a/apps/4tu/src/App.tsx b/apps/4tu/src/App.tsx index 5007d734..f296aeac 100644 --- a/apps/4tu/src/App.tsx +++ b/apps/4tu/src/App.tsx @@ -22,12 +22,17 @@ import authProvider from "./config/auth"; import form from "./config/form"; const App = () => { - const [ repoConfig, setRepoConfig ] = useState(); + const [ repoConfig, setRepoConfig ] = useState({}); + const configIsSet = repoConfig.hasOwnProperty('form'); return ( - + {/* Suspense to make sure languages can load first */} { > } /> - {repoConfig ? [ + {configIsSet ? [ >}) => { const [recommendations, setRecommendations] = useState([]); - const [ror, setRor] = useState(null); - const [narcis, setNarcis] = useState(null); - const [depositType, setDepositType] = useState(''); - const [fileType, setFileType] = useState(''); + const [ror, setRor] = useState -// Basic API fetching function, based on the API calls in the Deposit package -const fetchData = async ( - type: string, - setData: (option: AutocompleteAPIFieldData) => void, - debouncedInputValue: string, - setLoading: (b: boolean) => void -) => { - const uri = - type === 'ror' ? - `https://api.ror.org/organizations?query.advanced=name:${debouncedInputValue}*` : - type === 'narcis' ? - `https://vocabs.datastations.nl/rest/v1/NARCIS/search?query=${debouncedInputValue}*&unique=true&lang=en` : - ''; - try { - const result = await fetch(uri, { - headers: { Accept: "application/json" } - }); - const json = await result.json(); - const transformResult = - type === 'ror' ? - (json.number_of_results > 0 ? - { - arg: debouncedInputValue, - response: json.items.map((item: any) => ({ - label: item.name, - value: item.id, - extraLabel: "country", - extraContent: item.country.country_name, - })) - } : - [] - ) : - type === 'narcis' ? - (json.results.length > 0 ? - { - arg: debouncedInputValue, - response: json.results.map((item: any) => ({ - label: item.prefLabel, - value: item.uri, - id: item.localname, - })).filter(Boolean), - } : - [] - ) : - []; - setData(transformResult as AutocompleteAPIFieldData); - } catch (error) { - console.error(error); - } - setLoading(false); -} - // Derived from the API field in the Deposit package const ApiField = ({type, label, value, setValue, disabled}: { type: string; @@ -204,16 +186,21 @@ const ApiField = ({type, label, value, setValue, disabled}: { }) => { const [inputValue, setInputValue] = useState(""); const [data, setData] = useState(); - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(false); const debouncedInputValue = useDebounce(inputValue, 500)[0]; + + useEffect(() => { + error && enqueueSnackbar(`Could not reach the ${type} database`, { variant: "customError" }); + }, [error]) useEffect( () => { - debouncedInputValue && fetchData( - type, - setData, - debouncedInputValue, - setLoading - ); + const fetchData = async () => { + const results = await fetchTypeaheadApiData(type, debouncedInputValue); + results ? setData(results) : setError(true); + setLoading(false); + }; + debouncedInputValue && fetchData(); }, [debouncedInputValue]); useEffect( () => { diff --git a/apps/4tu/src/config/pages/apiHelpers.ts b/apps/4tu/src/config/pages/apiHelpers.ts new file mode 100644 index 00000000..9e5a6c03 --- /dev/null +++ b/apps/4tu/src/config/pages/apiHelpers.ts @@ -0,0 +1,73 @@ +import type { AutocompleteAPIFieldData } from "@dans-framework/deposit"; + +export const postRecommendationsApiData = async ( + ror: string | null, + narcis: string | null, + depositType: string, + fileType: string +) => { + try { + /*const result = await fetch(``, { + method: "POST", + headers: { Accept: "application/json" }, + body: JSON.stringify({ + affiliation: ror, + domain: narcis, + depositType: depositType, + fileType: fileType, + }), + }); + const json = await result.json(); + return json;*/ + return ['fake1', 'fake2']; + } catch (error) { + console.error(error); + return false; + } +} + +export const fetchTypeaheadApiData = async ( + type: string, + debouncedInputValue: string, +) => { + const uri = + type === 'ror' ? + `https://api.ror.org/organizations?query.advanced=name:${debouncedInputValue}*` : + type === 'narcis' ? + `https://vocabs.datastations.nl/rest/v1/NARCIS/search?query=${debouncedInputValue}*&unique=true&lang=en` : + ''; + try { + const result = await fetch(uri, { + headers: { Accept: "application/json" } + }); + const json = await result.json(); + const transformResult = + type === 'ror' ? + (json.number_of_results > 0 ? + { + arg: debouncedInputValue, + response: json.items.map((item: any) => ({ + label: item.name, + value: item.id, + })) + } : + [] + ) : + type === 'narcis' ? + (json.results.length > 0 ? + { + arg: debouncedInputValue, + response: json.results.map((item: any) => ({ + label: item.prefLabel, + value: item.uri, + })).filter(Boolean), + } : + [] + ) : + []; + return transformResult as AutocompleteAPIFieldData; + } catch (error) { + console.error(error); + return false; + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47ea8ea3..c673e8e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,9 +56,15 @@ importers: '@mui/material': specifier: ^5.14.3 version: 5.14.3(@emotion/react@11.11.1)(@emotion/styled@11.11.0)(@types/react@18.2.15)(react-dom@18.2.0)(react@18.2.0) + framer-motion: + specifier: ^10.15.0 + version: 10.15.0(react-dom@18.2.0)(react@18.2.0) lz-string: specifier: ^1.5.0 version: 1.5.0 + notistack: + specifier: ^3.0.1 + version: 3.0.1(csstype@3.1.2)(react-dom@18.2.0)(react@18.2.0) react: specifier: ^18.2.0 version: 18.2.0 From f05b73205720bf028b18a241c6e158f55d8d47ef Mon Sep 17 00:00:00 2001 From: Daan Janssen Date: Mon, 22 Jul 2024 13:06:29 +0200 Subject: [PATCH 28/80] 4tu ready to deploy to demo --- apps/4tu/src/App.tsx | 99 +++++++++++++++-------- apps/4tu/src/config/pages.ts | 22 ++++- apps/4tu/src/config/pages/RepoAdvisor.tsx | 82 ++++++++++++++++--- apps/4tu/src/config/pages/apiHelpers.ts | 21 +++-- apps/4tu/src/config/pages/deposit.ts | 12 --- packages/rdt-search-ui | 2 +- pnpm-lock.yaml | 24 ------ 7 files changed, 170 insertions(+), 92 deletions(-) delete mode 100644 apps/4tu/src/config/pages/deposit.ts diff --git a/apps/4tu/src/App.tsx b/apps/4tu/src/App.tsx index f296aeac..8ee9d6a5 100644 --- a/apps/4tu/src/App.tsx +++ b/apps/4tu/src/App.tsx @@ -12,24 +12,39 @@ import { UserSubmissions, SignInCallback, } from "@dans-framework/user-auth"; -import RepoAdvisor from './config/pages/RepoAdvisor'; +import RepoAdvisor, { NoRepoSelected, CurrentlySelected } from './config/pages/RepoAdvisor'; +import type { ExtendedFormConfig } from "./config/pages/apiHelpers"; // Load config variables +import pages from "./config/pages"; import theme from "./config/theme"; import footer from "./config/footer"; import siteTitle from "./config/siteTitle"; import authProvider from "./config/auth"; import form from "./config/form"; +import { AnimatePresence, motion } from "framer-motion"; const App = () => { - const [ repoConfig, setRepoConfig ] = useState({}); - const configIsSet = repoConfig.hasOwnProperty('form'); + const [ repoConfig, setRepoConfig ] = useState(); + const configIsSet = repoConfig?.hasOwnProperty('form') || false; return ( + + { configIsSet && + + + + } + @@ -43,49 +58,63 @@ const App = () => { > } /> - {configIsSet ? [ - + + + + } + /> + + {configIsSet ? - - } - />, - - - - } - />, - + /> : + + } + + } + /> + + {configIsSet ? + : + + } + + } + /> + + {configIsSet ? - - } - /> - ] : + inMenu: true, + }} /> : + + } + + } + /> - + } /> - } diff --git a/apps/4tu/src/config/pages.ts b/apps/4tu/src/config/pages.ts index 050247be..4e9d4ed4 100644 --- a/apps/4tu/src/config/pages.ts +++ b/apps/4tu/src/config/pages.ts @@ -1,5 +1,21 @@ -import deposit from "./pages/deposit"; +import type { Page } from "@dans-framework/pages"; -const pages = [deposit]; +const depositPage: Page = { + id: "deposit", + name: "Deposit", + slug: "deposit", + template: "deposit", + inMenu: true, + menuTitle: "Deposit", +}; -export default pages; +const advisorPage: Page = { + id: "advisor", + name: "Advisor", + slug: "/", + template: "deposit", + inMenu: true, + menuTitle: "Advisor", +}; + +export default [ advisorPage, depositPage ]; diff --git a/apps/4tu/src/config/pages/RepoAdvisor.tsx b/apps/4tu/src/config/pages/RepoAdvisor.tsx index 78bf8e0c..b71e3ca5 100644 --- a/apps/4tu/src/config/pages/RepoAdvisor.tsx +++ b/apps/4tu/src/config/pages/RepoAdvisor.tsx @@ -1,4 +1,5 @@ -import { useState, useEffect, type Dispatch, type SetStateAction } from "react"; +import { useState, useEffect, Fragment, type Dispatch, type SetStateAction } from "react"; +import { useNavigate, Link as RouterLink } from "react-router-dom"; import Stack from '@mui/material/Stack'; import CircularProgress from '@mui/material/CircularProgress'; import Paper from '@mui/material/Paper'; @@ -16,17 +17,23 @@ import TextField from '@mui/material/TextField'; import Autocomplete from '@mui/material/Autocomplete'; import type { AutocompleteAPIFieldData } from "@dans-framework/deposit"; import { AnimatePresence, motion } from "framer-motion"; -import { fetchTypeaheadApiData, postRecommendationsApiData } from "./apiHelpers"; +import { fetchTypeaheadApiData, postRecommendationsApiData, type ExtendedFormConfig } from "./apiHelpers"; import { enqueueSnackbar } from "notistack"; +import Link from '@mui/material/Link'; +import List from '@mui/material/List'; +import ListItem from '@mui/material/ListItem'; +import Divider from '@mui/material/Divider'; +import ListItemText from '@mui/material/ListItemText'; const RepoAdvisor = ({setRepoConfig}: {setRepoConfig: Dispatch>}) => { - const [recommendations, setRecommendations] = useState([]); + const [recommendations, setRecommendations] = useState([]); const [ror, setRor] = useState