diff --git a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt index 119665129b..6439ed30e4 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt @@ -1,5 +1,6 @@ package org.loculus.backend.config +import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.swagger.v3.oas.models.headers.Header @@ -146,7 +147,9 @@ internal fun validateEarliestReleaseDateFields(config: BackendConfig): List(File(configPath)) + val config = objectMapper + .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + .readValue(File(configPath)) logger.info { "Loaded backend config from $configPath" } logger.info { "Config: $config" } val validationErrors = validateEarliestReleaseDateFields(config) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index a56907619b..e6b123c248 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -7,6 +7,7 @@ import org.loculus.backend.api.Organism data class BackendConfig( val organisms: Map, val accessionPrefix: String, + val dataUseTermsEnabled: Boolean, val dataUseTermsUrls: DataUseTermsUrls?, ) { fun getInstanceConfig(organism: Organism) = organisms[organism.name] ?: throw IllegalArgumentException( diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index 88ba74b6ad..ac6a6eb903 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -33,6 +33,7 @@ import org.loculus.backend.api.SubmittedProcessedData import org.loculus.backend.api.UnprocessedData import org.loculus.backend.auth.AuthenticatedUser import org.loculus.backend.auth.HiddenParam +import org.loculus.backend.config.BackendConfig import org.loculus.backend.controller.LoculusCustomHeaders.X_TOTAL_RECORDS import org.loculus.backend.log.REQUEST_ID_MDC_KEY import org.loculus.backend.log.RequestIdContext @@ -76,10 +77,11 @@ open class SubmissionController( private val submissionDatabaseService: SubmissionDatabaseService, private val iteratorStreamer: IteratorStreamer, private val requestIdContext: RequestIdContext, + private val backendConfig: BackendConfig, ) { - @Operation(description = SUBMIT_DESCRIPTION) @ApiResponse(responseCode = "200", description = SUBMIT_RESPONSE_DESCRIPTION) + @ApiResponse(responseCode = "400", description = SUBMIT_ERROR_RESPONSE) @PostMapping("/submit", consumes = ["multipart/form-data"]) fun submit( @PathVariable @Valid organism: Organism, @@ -87,8 +89,10 @@ open class SubmissionController( @Parameter(description = GROUP_ID_DESCRIPTION) @RequestParam groupId: Int, @Parameter(description = METADATA_FILE_DESCRIPTION) @RequestParam metadataFile: MultipartFile, @Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile?, - @Parameter(description = "Data Use terms under which data is released.") @RequestParam dataUseTermsType: - DataUseTermsType, + @Parameter( + description = + "Data Use terms under which data is released. Mandatory when data use terms are enabled for this Instance.", + ) @RequestParam dataUseTermsType: DataUseTermsType?, @Parameter( description = "Mandatory when data use terms are set to 'RESTRICTED'." + @@ -96,13 +100,22 @@ open class SubmissionController( " Format: YYYY-MM-DD", ) @RequestParam restrictedUntil: String?, ): List { + var dataUseTermsKind = DataUseTermsType.OPEN + if (backendConfig.dataUseTermsEnabled) { + if (dataUseTermsType == null) { + throw BadRequestException("the 'dataUseTermsType' needs to be provided.") + } else { + dataUseTermsKind = dataUseTermsType + } + } + val params = SubmissionParams.OriginalSubmissionParams( organism, authenticatedUser, metadataFile, sequenceFile, groupId, - DataUseTerms.fromParameters(dataUseTermsType, restrictedUntil), + DataUseTerms.fromParameters(dataUseTermsKind, restrictedUntil), ) return submitModel.processSubmissions(UUID.randomUUID().toString(), params) } diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt index 66a7711bb1..babbe5c2d8 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt @@ -8,6 +8,10 @@ The accession is the (globally unique) id that the system assigned to the sequen You can use this response to associate the user provided submissionId with the system assigned accession. """ +const val SUBMIT_ERROR_RESPONSE = """ +The data use terms type have not been provided, even though they are enabled for this Loculus instance. +""" + const val METADATA_FILE_DESCRIPTION = """ A TSV (tab separated values) file containing the metadata of the submitted sequence entries. The file may be compressed with zstd, xz, zip, gzip, lzma, bzip2 (with common extensions). diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 3cccc63b84..82d78e61bc 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.TextNode import mu.KotlinLogging import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.GeneticSequence +import org.loculus.backend.api.MetadataMap import org.loculus.backend.api.Organism import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.VersionStatus @@ -94,6 +95,9 @@ open class ReleasedDataModel( return "\"$lastUpdateTime\"" // ETag must be enclosed in double quotes } + private fun conditionalMetadata(condition: Boolean, values: () -> MetadataMap): MetadataMap = + if (condition) values() else emptyMap() + private fun computeAdditionalMetadataFields( rawProcessedData: RawProcessedData, latestVersions: Map, @@ -111,6 +115,13 @@ open class ReleasedDataModel( val earliestReleaseDate = earliestReleaseDateFinder?.calculateEarliestReleaseDate(rawProcessedData) + val dataUseTermsUrl: String? = backendConfig.dataUseTermsUrls?.let { urls -> + when (currentDataUseTerms) { + DataUseTerms.Open -> urls.open + is DataUseTerms.Restricted -> urls.restricted + } + } + var metadata = rawProcessedData.processedData.metadata + mapOf( ("accession" to TextNode(rawProcessedData.accession)), @@ -126,31 +137,42 @@ open class ReleasedDataModel( ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())), ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())), ("versionStatus" to TextNode(versionStatus.name)), - ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)), - ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil), ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), ) + - if (rawProcessedData.isRevocation) { - mapOf("versionComment" to TextNode(rawProcessedData.versionComment)) - } else { - emptyMap() - }.let { - when (backendConfig.dataUseTermsUrls) { - null -> it - else -> { - val url = when (currentDataUseTerms) { - DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open - is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted - } - it + ("dataUseTermsUrl" to TextNode(url)) - } - } - } + - if (earliestReleaseDate != null) { - mapOf("earliestReleaseDate" to TextNode(earliestReleaseDate.toUtcDateString())) - } else { - emptyMap() - } + // TODO add a test for this change + conditionalMetadata( + backendConfig.dataUseTermsEnabled, + { + mapOf( + "dataUseTerms" to TextNode(currentDataUseTerms.type.name), + "dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil, + ) + }, + ) + + conditionalMetadata( + rawProcessedData.isRevocation, + { + mapOf( + "versionComment" to TextNode(rawProcessedData.versionComment), + ) + }, + ) + + conditionalMetadata( + earliestReleaseDate != null, + { + mapOf( + "earliestReleaseDate" to TextNode(earliestReleaseDate!!.toUtcDateString()), + ) + }, + ) + + conditionalMetadata( + dataUseTermsUrl != null, + { + mapOf( + "dataUseTermsUrl" to TextNode(dataUseTermsUrl!!), + ) + }, + ) return ProcessedData( metadata = metadata, diff --git a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt index 8979cbb94b..bcbf292420 100644 --- a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt @@ -70,5 +70,6 @@ fun backendConfig(metadataList: List, earliestReleaseDate: EarliestRel ), ), accessionPrefix = "FOO_", + dataUseTermsEnabled = true, dataUseTermsUrls = null, ) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 57cbddab60..d3875a9ba1 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -184,6 +184,8 @@ class SubmitEndpointTest( organism: Organism, dataUseTerm: DataUseTerms, ) { + println("Data use terms enabled: ") + println(backendConfig.dataUseTermsEnabled) submissionControllerClient.submit( metadataFile, sequencesFile, diff --git a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt index ad3b53fa80..a9a05704df 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt @@ -11,7 +11,12 @@ const val PREFIX = "LOC_" class GenerateAccessionFromNumberServiceTest { private val accessionFromNumberService = GenerateAccessionFromNumberService( - BackendConfig(accessionPrefix = PREFIX, organisms = emptyMap(), dataUseTermsUrls = null), + BackendConfig( + accessionPrefix = PREFIX, + organisms = emptyMap(), + dataUseTermsEnabled = true, + dataUseTermsUrls = null, + ), ) @Test diff --git a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt index 96a0dea239..2849d4682c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt @@ -46,6 +46,7 @@ class EmptyProcessedDataProviderTest { ), ), ), + dataUseTermsEnabled = true, dataUseTermsUrls = null, ), ) diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json index 3729e7a6bf..e63e12667f 100644 --- a/backend/src/test/resources/backend_config.json +++ b/backend/src/test/resources/backend_config.json @@ -225,5 +225,6 @@ ] } } - } + }, + "dataUseTermsEnabled": true } diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json index e61b1c5c72..9f532a9b2c 100644 --- a/backend/src/test/resources/backend_config_single_segment.json +++ b/backend/src/test/resources/backend_config_single_segment.json @@ -1,5 +1,6 @@ { "accessionPrefix": "LOC_", + "dataUseTermsEnabled": true, "organisms": { "dummyOrganism": { "referenceGenomes": { diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 44ae6d38f0..81c0527b3c 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -75,6 +75,7 @@ fields: autocomplete: true displayName: Date released (exact) columnWidth: 100 + # TODO change to be done here - only include these fields if a flag is set - name: dataUseTerms type: string generateIndex: true @@ -160,6 +161,7 @@ enableLoginNavigationItem: {{ $.Values.website.websiteConfig.enableLoginNavigati enableSubmissionNavigationItem: {{ $.Values.website.websiteConfig.enableSubmissionNavigationItem }} enableSubmissionPages: {{ $.Values.website.websiteConfig.enableSubmissionPages }} enableSeqSets: {{ $.Values.seqSets.enabled }} +enableDataUseTerms: {{ $.Values.dataUseTermsEnabled }} accessionPrefix: {{ quote $.Values.accessionPrefix }} {{- $commonMetadata := (include "loculus.commonMetadata" . | fromYaml).fields }} organisms: @@ -289,6 +291,7 @@ fields: {{- define "loculus.generateBackendConfig" }} accessionPrefix: {{ quote $.Values.accessionPrefix }} name: {{ quote $.Values.name }} +dataUseTermsEnabled: {{$.Values.dataUseTermsEnabled }} dataUseTermsUrls: {{$.Values.dataUseTermsUrls | toYaml | nindent 2}} organisms: diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 5c7d4c1b23..a4b9a6ec8b 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -20,6 +20,7 @@ ingestLimitSeconds: 1800 getSubmissionListLimitSeconds: 600 preprocessingTimeout: 600 accessionPrefix: "LOC_" +dataUseTermsEnabled: false dataUseTermsUrls: open: https://#TODO-MVP/open restricted: https://#TODO-MVP/restricted diff --git a/website/src/components/Submission/DataUploadForm.tsx b/website/src/components/Submission/DataUploadForm.tsx index 4ff60fe469..7cfbce904c 100644 --- a/website/src/components/Submission/DataUploadForm.tsx +++ b/website/src/components/Submission/DataUploadForm.tsx @@ -42,6 +42,7 @@ type DataUploadFormProps = { onSuccess: () => void; onError: (message: string) => void; submissionDataTypes: SubmissionDataTypes; + dataUseTermsEnabled: boolean; }; const logger = getClientLogger('DataUploadForm'); @@ -128,6 +129,7 @@ const InnerDataUploadForm = ({ referenceGenomeSequenceNames, metadataTemplateFields, submissionDataTypes, + dataUseTermsEnabled, }: DataUploadFormProps) => { const [metadataFile, setMetadataFile] = useState(undefined); // The columnMapping can be null; if null -> don't apply mapping. @@ -162,12 +164,12 @@ const InnerDataUploadForm = ({ const handleSubmit = async (event: FormEvent) => { event.preventDefault(); - if (!agreedToINSDCUploadTerms) { + if (dataUseTermsEnabled && !agreedToINSDCUploadTerms) { onError('Please tick the box to agree that you will not independently submit these sequences to INSDC'); return; } - if (!confirmedNoPII) { + if (dataUseTermsEnabled && !confirmedNoPII) { onError( 'Please confirm the data you submitted does not include restricted or personally identifiable information.', ); @@ -327,86 +329,87 @@ const InnerDataUploadForm = ({ - {action !== 'revise' && ( + {action === 'submit' && dataUseTermsEnabled && ( )} -
-
-

Acknowledgement

-

Acknowledge submission terms

-
-
-
- {dataUseTermsType === restrictedDataUseTermsOption && ( -

- Your data will be available on Pathoplexus, under the restricted use terms until{' '} - {restrictedUntil.toFormat('yyyy-MM-dd')}. After the restricted period your data will - additionally be made publicly available through the{' '} - - INSDC - {' '} - databases (ENA, DDBJ, NCBI). -

- )} - {dataUseTermsType === openDataUseTermsOption && ( -

- Your data will be available on Pathoplexus under the open use terms. It will - additionally be made publicly available through the{' '} - - INSDC - {' '} - databases (ENA, DDBJ, NCBI). -

- )} -
- -
-
- + {dataUseTermsEnabled && ( +
+
+

Acknowledgement

+

Acknowledge submission terms

+
+
+
+ {dataUseTermsType === restrictedDataUseTermsOption && ( +

+ Your data will be available on Pathoplexus, under the restricted use terms until{' '} + {restrictedUntil.toFormat('yyyy-MM-dd')}. After the restricted period your data + will additionally be made publicly available through the{' '} + + INSDC + {' '} + databases (ENA, DDBJ, NCBI). +

+ )} + {dataUseTermsType === openDataUseTermsOption && ( +

+ Your data will be available on Pathoplexus under the open use terms. It will + additionally be made publicly available through the{' '} + + INSDC + {' '} + databases (ENA, DDBJ, NCBI). +

+ )} +
+ +
+
+ +
-
- -
+ )} +
diff --git a/website/src/types/config.ts b/website/src/types/config.ts index dda387617a..f2a2d4a8c1 100644 --- a/website/src/types/config.ts +++ b/website/src/types/config.ts @@ -144,6 +144,7 @@ export const websiteConfig = z.object({ enableLoginNavigationItem: z.boolean(), enableSubmissionNavigationItem: z.boolean(), enableSubmissionPages: z.boolean(), + enableDataUseTerms: z.boolean(), }); export type WebsiteConfig = z.infer;