From bf66008a24a4dc2b77422e0c985835aef21adc16 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 29 Jan 2025 11:41:46 +0100 Subject: [PATCH 01/56] Add first schema --- kubernetes/loculus/values.schema.json | 319 ++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 kubernetes/loculus/values.schema.json diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json new file mode 100644 index 000000000..3db04e5e0 --- /dev/null +++ b/kubernetes/loculus/values.schema.json @@ -0,0 +1,319 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "Loculus", + "description": "The name of the Loculus instance" + }, + "logo": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL path to the logo image file" + }, + "width": { + "type": "integer", + "default": 100, + "description": "Width of the logo in pixels" + }, + "height": { + "type": "integer", + "default": 100, + "description": "Height of the logo in pixels" + } + } + }, + "accessionPrefix": { + "type": "string", + "default": "LOC_", + "description": "Prefix used for accession numbers" + }, + "environment": { + "type": "string", + "enum": ["local", "server"], + "default": "server", + "description": "Deployment environment. local for development, server for production" + }, + "host": { + "type": "string", + "description": "Hostname where Loculus will be accessible" + }, + "bannerMessage": { + "type": "string", + "description": "Banner message (as HTML) to display at the very top of the page" + }, + "welcomeMessageHTML": { + "type": ["string", "null"], + "description": "A custom welcome message to be shown on the landing page" + }, + "additionalHeadHTML": { + "type": "string", + "description": "Additional HTML to inject into the of pages" + }, + "createTestAccounts": { + "type": "boolean", + "description": "If true, creates the users testuser and superuser" + }, + "robotsNoindexHeader": { + "type": "boolean", + "description": "If true, adds a noindex header to prevent search engine indexing" + }, + "seqSets": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable/disable SeqSets" + }, + "crossRef": { + "type": ["object", "null"], + "properties": { + "DOIPrefix": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "organization": { + "type": "string" + }, + "hostUrl": { + "type": "string" + } + } + } + } + }, + "website": { + "type": "object", + "properties": { + "websiteConfig": { + "type": "object", + "properties": { + "enableLoginNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionPages": { + "type": "boolean", + "default": true + } + } + }, + "runtimeConfig": { + "type": "object", + "properties": { + "public": { + "type": "object", + "properties": { + "backendUrl": { + "type": "string" + }, + "keycloakUrl": { + "type": "string" + }, + "lapisUrlTemplate": { + "type": "string" + } + } + } + } + } + } + }, + "auth": { + "type": "object", + "properties": { + "smtp": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "user": { + "type": "string" + }, + "replyTo": { + "type": "string", + "format": "email" + }, + "from": { + "type": "string", + "format": "email" + }, + "envelopeFrom": { + "type": "string", + "format": "email" + } + } + }, + "verifyEmail": { + "type": "boolean", + "default": true + }, + "resetPasswordAllowed": { + "type": "boolean", + "default": true + }, + "registrationAllowed": { + "type": "boolean", + "default": true + } + } + }, + "organisms": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "object", + "properties": { + "schema": { + "type": "object", + "properties": { + "organismName": { + "type": "string" + }, + "image": { + "type": "string" + }, + "loadSequencesAutomatically": { + "type": "boolean", + "default": true + }, + "metadata": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "displayName": { + "type": "string" + }, + "type": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "definition": { + "type": "string" + }, + "guidance": { + "type": "string" + }, + "generateIndex": { + "type": "boolean" + }, + "autocomplete": { + "type": "boolean" + } + }, + "required": ["name", "displayName", "type"] + } + } + }, + "required": ["organismName"] + }, + "preprocessing": { + "type": "array", + "items": { + "type": "object", + "properties": { + "version": { + "type": "integer" + }, + "image": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "configFile": { + "type": "object" + } + }, + "required": ["version", "image"] + } + }, + "referenceGenomes": { + "type": "object", + "properties": { + "nucleotideSequences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "sequence": { + "type": "string" + }, + "insdcAccessionFull": { + "type": "string" + } + }, + "required": ["name", "sequence"] + } + }, + "genes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "sequence": { + "type": "string" + } + }, + "required": ["name", "sequence"] + } + } + } + } + }, + "required": ["schema"] + } + } + }, + "lineageSystemDefinitions": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "object", + "patternProperties": { + "^[0-9]+$": { + "type": "string", + "format": "uri" + } + } + } + } + } + }, + "required": ["name"] + } \ No newline at end of file From 5523bca7164d29b2fcc604ddd2d470545887b297 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 29 Jan 2025 12:34:58 +0100 Subject: [PATCH 02/56] Add stubs for Schema docs generation --- docs/src/components/SchemaDocs.astro | 27 + .../docs/reference/helm-chart-config.mdx | 5 + kubernetes/loculus/schemas/auth.schema.json | 29 + .../loculus/schemas/website.schema.json | 42 ++ kubernetes/loculus/values.schema.json | 498 ++++++++---------- 5 files changed, 313 insertions(+), 288 deletions(-) create mode 100644 docs/src/components/SchemaDocs.astro create mode 100644 kubernetes/loculus/schemas/auth.schema.json create mode 100644 kubernetes/loculus/schemas/website.schema.json diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro new file mode 100644 index 000000000..ae547d437 --- /dev/null +++ b/docs/src/components/SchemaDocs.astro @@ -0,0 +1,27 @@ +--- +import { readFile, readdir } from 'node:fs/promises'; + +interface Schema { + properties?: any +} + +const { schemaPath } = Astro.props; +let schema: Schema = {}; + +try { + const data = await readFile(schemaPath, 'utf-8'); + schema = JSON.parse(data); +} catch (error) { + console.error('Error reading schema:', error); +} +--- + +
+

Schema Properties

+
    + {schema.properties && + Object.keys(schema.properties).map((key) => ( +
  • {key}
    • + ))} +
+
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 13385ebe7..6f6290ed3 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -2,6 +2,7 @@ title: 'Helm chart config' description: A reference of config fields for the Loculus Helm chart --- +import SchemaDocs from '../../../components/SchemaDocs.astro'; :::note The config structure has not been finalized and may change rapidly until Loculus 1.0 is officially released (see [current state and roadmap](../../introduction/current-state-and-roadmap/)). In the meantime, this reference might also not be fully accurate or complete. @@ -11,6 +12,10 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## General Settings + + + +
diff --git a/kubernetes/loculus/schemas/auth.schema.json b/kubernetes/loculus/schemas/auth.schema.json new file mode 100644 index 000000000..a0e4b2c32 --- /dev/null +++ b/kubernetes/loculus/schemas/auth.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "smtp": { + "type": "object", + "properties": { + "host": { "type": "string" }, + "port": { "type": "integer" }, + "user": { "type": "string" }, + "replyTo": { "type": "string", "format": "email" }, + "from": { "type": "string", "format": "email" }, + "envelopeFrom": { "type": "string", "format": "email" } + } + }, + "verifyEmail": { + "type": "boolean", + "default": true + }, + "resetPasswordAllowed": { + "type": "boolean", + "default": true + }, + "registrationAllowed": { + "type": "boolean", + "default": true + } + } +} diff --git a/kubernetes/loculus/schemas/website.schema.json b/kubernetes/loculus/schemas/website.schema.json new file mode 100644 index 000000000..177c90d21 --- /dev/null +++ b/kubernetes/loculus/schemas/website.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "websiteConfig": { + "type": "object", + "properties": { + "enableLoginNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionPages": { + "type": "boolean", + "default": true + } + } + }, + "runtimeConfig": { + "type": "object", + "properties": { + "public": { + "type": "object", + "properties": { + "backendUrl": { + "type": "string" + }, + "keycloakUrl": { + "type": "string" + }, + "lapisUrlTemplate": { + "type": "string" + } + } + } + } + } + } +} diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 3db04e5e0..3483e1ee7 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1,319 +1,241 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": "string", - "default": "Loculus", - "description": "The name of the Loculus instance" - }, - "logo": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "URL path to the logo image file" - }, - "width": { - "type": "integer", - "default": 100, - "description": "Width of the logo in pixels" - }, - "height": { - "type": "integer", - "default": 100, - "description": "Height of the logo in pixels" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "Loculus", + "description": "The name of the Loculus instance" + }, + "logo": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL path to the logo image file" + }, + "width": { + "type": "integer", + "default": 100, + "description": "Width of the logo in pixels" + }, + "height": { + "type": "integer", + "default": 100, + "description": "Height of the logo in pixels" } - }, - "accessionPrefix": { - "type": "string", - "default": "LOC_", - "description": "Prefix used for accession numbers" - }, - "environment": { - "type": "string", - "enum": ["local", "server"], - "default": "server", - "description": "Deployment environment. local for development, server for production" - }, - "host": { - "type": "string", - "description": "Hostname where Loculus will be accessible" - }, - "bannerMessage": { - "type": "string", - "description": "Banner message (as HTML) to display at the very top of the page" - }, - "welcomeMessageHTML": { - "type": ["string", "null"], - "description": "A custom welcome message to be shown on the landing page" - }, - "additionalHeadHTML": { - "type": "string", - "description": "Additional HTML to inject into the of pages" - }, - "createTestAccounts": { - "type": "boolean", - "description": "If true, creates the users testuser and superuser" - }, - "robotsNoindexHeader": { - "type": "boolean", - "description": "If true, adds a noindex header to prevent search engine indexing" - }, - "seqSets": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean", - "description": "Enable/disable SeqSets" - }, - "crossRef": { - "type": ["object", "null"], - "properties": { - "DOIPrefix": { - "type": "string" - }, - "endpoint": { - "type": "string" - }, - "databaseName": { - "type": "string" - }, - "email": { - "type": "string", - "format": "email" - }, - "organization": { - "type": "string" - }, - "hostUrl": { - "type": "string" - } + } + }, + "accessionPrefix": { + "type": "string", + "default": "LOC_", + "description": "Prefix used for accession numbers" + }, + "environment": { + "type": "string", + "enum": ["local", "server"], + "default": "server", + "description": "Deployment environment. local for development, server for production" + }, + "host": { + "type": "string", + "description": "Hostname where Loculus will be accessible" + }, + "bannerMessage": { + "type": "string", + "description": "Banner message (as HTML) to display at the very top of the page" + }, + "welcomeMessageHTML": { + "type": ["string", "null"], + "description": "A custom welcome message to be shown on the landing page" + }, + "additionalHeadHTML": { + "type": "string", + "description": "Additional HTML to inject into the of pages" + }, + "createTestAccounts": { + "type": "boolean", + "description": "If true, creates the users testuser and superuser" + }, + "robotsNoindexHeader": { + "type": "boolean", + "description": "If true, adds a noindex header to prevent search engine indexing" + }, + "seqSets": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable/disable SeqSets" + }, + "crossRef": { + "type": ["object", "null"], + "properties": { + "DOIPrefix": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "organization": { + "type": "string" + }, + "hostUrl": { + "type": "string" } } } - }, - "website": { - "type": "object", - "properties": { - "websiteConfig": { - "type": "object", - "properties": { - "enableLoginNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionPages": { - "type": "boolean", - "default": true - } - } - }, - "runtimeConfig": { - "type": "object", - "properties": { - "public": { - "type": "object", - "properties": { - "backendUrl": { - "type": "string" - }, - "keycloakUrl": { - "type": "string" - }, - "lapisUrlTemplate": { - "type": "string" + } + }, + "website": { + "$ref": "file://./schemas/website.schema.json" + }, + "auth": { + "$ref": "file://./schemas/auth.schema.json" + }, + "organisms": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "object", + "properties": { + "schema": { + "type": "object", + "properties": { + "organismName": { + "type": "string" + }, + "image": { + "type": "string" + }, + "loadSequencesAutomatically": { + "type": "boolean", + "default": true + }, + "metadata": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "displayName": { + "type": "string" + }, + "type": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "definition": { + "type": "string" + }, + "guidance": { + "type": "string" + }, + "generateIndex": { + "type": "boolean" + }, + "autocomplete": { + "type": "boolean" + } + }, + "required": ["name", "displayName", "type"] } } - } - } - } - } - }, - "auth": { - "type": "object", - "properties": { - "smtp": { - "type": "object", - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "integer" - }, - "user": { - "type": "string" - }, - "replyTo": { - "type": "string", - "format": "email" - }, - "from": { - "type": "string", - "format": "email" }, - "envelopeFrom": { - "type": "string", - "format": "email" - } - } - }, - "verifyEmail": { - "type": "boolean", - "default": true - }, - "resetPasswordAllowed": { - "type": "boolean", - "default": true - }, - "registrationAllowed": { - "type": "boolean", - "default": true - } - } - }, - "organisms": { - "type": "object", - "patternProperties": { - "^.*$": { - "type": "object", - "properties": { - "schema": { + "required": ["organismName"] + }, + "preprocessing": { + "type": "array", + "items": { "type": "object", "properties": { - "organismName": { - "type": "string" + "version": { + "type": "integer" }, "image": { "type": "string" }, - "loadSequencesAutomatically": { - "type": "boolean", - "default": true - }, - "metadata": { + "args": { "type": "array", "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "displayName": { - "type": "string" - }, - "type": { - "type": "string" - }, - "required": { - "type": "boolean" - }, - "definition": { - "type": "string" - }, - "guidance": { - "type": "string" - }, - "generateIndex": { - "type": "boolean" - }, - "autocomplete": { - "type": "boolean" - } - }, - "required": ["name", "displayName", "type"] + "type": "string" } + }, + "configFile": { + "type": "object" } }, - "required": ["organismName"] - }, - "preprocessing": { - "type": "array", - "items": { - "type": "object", - "properties": { - "version": { - "type": "integer" - }, - "image": { - "type": "string" - }, - "args": { - "type": "array", - "items": { + "required": ["version", "image"] + } + }, + "referenceGenomes": { + "type": "object", + "properties": { + "nucleotideSequences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "sequence": { + "type": "string" + }, + "insdcAccessionFull": { "type": "string" } }, - "configFile": { - "type": "object" - } - }, - "required": ["version", "image"] - } - }, - "referenceGenomes": { - "type": "object", - "properties": { - "nucleotideSequences": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "sequence": { - "type": "string" - }, - "insdcAccessionFull": { - "type": "string" - } - }, - "required": ["name", "sequence"] - } - }, - "genes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "sequence": { - "type": "string" - } + "required": ["name", "sequence"] + } + }, + "genes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" }, - "required": ["name", "sequence"] - } + "sequence": { + "type": "string" + } + }, + "required": ["name", "sequence"] } } } - }, - "required": ["schema"] - } + } + }, + "required": ["schema"] } - }, - "lineageSystemDefinitions": { - "type": "object", - "patternProperties": { - "^.*$": { - "type": "object", - "patternProperties": { - "^[0-9]+$": { - "type": "string", - "format": "uri" - } + } + }, + "lineageSystemDefinitions": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "object", + "patternProperties": { + "^[0-9]+$": { + "type": "string", + "format": "uri" } } } } - }, - "required": ["name"] - } \ No newline at end of file + } + }, + "required": ["name"] +} \ No newline at end of file From 3d7655582c7b29d89eee23c9cc4367efa5179461 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 09:17:47 +0100 Subject: [PATCH 03/56] un-split schema --- kubernetes/loculus/schemas/auth.schema.json | 29 --------- .../loculus/schemas/website.schema.json | 42 ------------- kubernetes/loculus/values.schema.json | 61 ++++++++++++++++++- 3 files changed, 59 insertions(+), 73 deletions(-) delete mode 100644 kubernetes/loculus/schemas/auth.schema.json delete mode 100644 kubernetes/loculus/schemas/website.schema.json diff --git a/kubernetes/loculus/schemas/auth.schema.json b/kubernetes/loculus/schemas/auth.schema.json deleted file mode 100644 index a0e4b2c32..000000000 --- a/kubernetes/loculus/schemas/auth.schema.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "smtp": { - "type": "object", - "properties": { - "host": { "type": "string" }, - "port": { "type": "integer" }, - "user": { "type": "string" }, - "replyTo": { "type": "string", "format": "email" }, - "from": { "type": "string", "format": "email" }, - "envelopeFrom": { "type": "string", "format": "email" } - } - }, - "verifyEmail": { - "type": "boolean", - "default": true - }, - "resetPasswordAllowed": { - "type": "boolean", - "default": true - }, - "registrationAllowed": { - "type": "boolean", - "default": true - } - } -} diff --git a/kubernetes/loculus/schemas/website.schema.json b/kubernetes/loculus/schemas/website.schema.json deleted file mode 100644 index 177c90d21..000000000 --- a/kubernetes/loculus/schemas/website.schema.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "websiteConfig": { - "type": "object", - "properties": { - "enableLoginNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionPages": { - "type": "boolean", - "default": true - } - } - }, - "runtimeConfig": { - "type": "object", - "properties": { - "public": { - "type": "object", - "properties": { - "backendUrl": { - "type": "string" - }, - "keycloakUrl": { - "type": "string" - }, - "lapisUrlTemplate": { - "type": "string" - } - } - } - } - } - } -} diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 3483e1ee7..abf793576 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -95,10 +95,67 @@ } }, "website": { - "$ref": "file://./schemas/website.schema.json" + "websiteConfig": { + "type": "object", + "properties": { + "enableLoginNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionNavigationItem": { + "type": "boolean", + "default": true + }, + "enableSubmissionPages": { + "type": "boolean", + "default": true + } + } + }, + "runtimeConfig": { + "type": "object", + "properties": { + "public": { + "type": "object", + "properties": { + "backendUrl": { + "type": "string" + }, + "keycloakUrl": { + "type": "string" + }, + "lapisUrlTemplate": { + "type": "string" + } + } + } + } + } }, "auth": { - "$ref": "file://./schemas/auth.schema.json" + "smtp": { + "type": "object", + "properties": { + "host": { "type": "string" }, + "port": { "type": "integer" }, + "user": { "type": "string" }, + "replyTo": { "type": "string", "format": "email" }, + "from": { "type": "string", "format": "email" }, + "envelopeFrom": { "type": "string", "format": "email" } + } + }, + "verifyEmail": { + "type": "boolean", + "default": true + }, + "resetPasswordAllowed": { + "type": "boolean", + "default": true + }, + "registrationAllowed": { + "type": "boolean", + "default": true + } }, "organisms": { "type": "object", From fc75456d89cd725c2597ede58bcaf5f567275ce2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 10:21:40 +0100 Subject: [PATCH 04/56] Add github linting job --- .github/workflows/helm-schema-lint.yaml | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/helm-schema-lint.yaml diff --git a/.github/workflows/helm-schema-lint.yaml b/.github/workflows/helm-schema-lint.yaml new file mode 100644 index 000000000..7074ffa00 --- /dev/null +++ b/.github/workflows/helm-schema-lint.yaml @@ -0,0 +1,29 @@ +name: helm-schema-lint + +on: + pull_request: + paths: + - "kubernetes/**" + - ".github/workflows/helm-schema-lint.yaml" + push: + branches: + - main + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: latest + + - name: Run Helm lint on values files + run: | + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values_e2e_and_dev.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values_preview_server.yaml From de14186094c8d76497afc1b8b0992ed3e14572b3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 10:50:20 +0100 Subject: [PATCH 05/56] Make version minimum 1 --- kubernetes/loculus/values.schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index abf793576..a0b7ef6b1 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -218,7 +218,8 @@ "type": "object", "properties": { "version": { - "type": "integer" + "type": "integer", + "minimum": 1 }, "image": { "type": "string" From fac6e783e757013d647c68102989b44e6a36bcd0 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 11:29:51 +0100 Subject: [PATCH 06/56] Add grouping and reading of groups in the schema docs generation --- docs/package-lock.json | 16 ++++++ docs/package.json | 3 ++ docs/src/components/SchemaDocs.astro | 51 ++++++++++++------- .../existing-preprocessing-pipelines.md | 8 +-- .../for-administrators/getting-started.md | 14 ++--- .../for-administrators/my-first-loculus.md | 2 - .../docs/for-administrators/schema-designs.md | 20 ++++---- .../setup-with-k3d-and-nginx.mdx | 20 ++++---- .../docs/for-users/approve-submissions.md | 10 ++-- .../docs/for-users/revoke-sequences.md | 12 ++--- .../docs/for-users/submit-sequences.md | 4 +- .../docs/introduction/system-overview.md | 24 ++++----- .../docs/introduction/what-is-loculus.md | 8 +-- .../docs/reference/helm-chart-config.mdx | 49 +++++++++--------- kubernetes/loculus/values.schema.json | 3 ++ 15 files changed, 142 insertions(+), 102 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index a4bc8ddd0..d023e4318 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -19,6 +19,9 @@ "sharp": "^0.33.5", "tailwindcss": "^3.4.16", "typescript": "^5.7.2" + }, + "devDependencies": { + "@types/json-schema": "^7.0.15" } }, "node_modules/@alloc/quick-lru": { @@ -1981,6 +1984,13 @@ "@types/unist": "*" } }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/mdast": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", @@ -9314,6 +9324,12 @@ "@types/unist": "*" } }, + "@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true + }, "@types/mdast": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", diff --git a/docs/package.json b/docs/package.json index 1ad48f509..e0f6e7c42 100644 --- a/docs/package.json +++ b/docs/package.json @@ -22,5 +22,8 @@ "sharp": "^0.33.5", "tailwindcss": "^3.4.16", "typescript": "^5.7.2" + }, + "devDependencies": { + "@types/json-schema": "^7.0.15" } } diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index ae547d437..d40d8863e 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,27 +1,44 @@ --- -import { readFile, readdir } from 'node:fs/promises'; +import { readFile } from 'node:fs/promises'; +import { type JSONSchema7, type JSONSchema7Definition } from 'json-schema'; -interface Schema { - properties?: any -} - -const { schemaPath } = Astro.props; -let schema: Schema = {}; +const { schemaPath, group } = Astro.props; +let schema: JSONSchema7 = {}; try { - const data = await readFile(schemaPath, 'utf-8'); - schema = JSON.parse(data); + const data = await readFile(schemaPath, 'utf-8'); + schema = JSON.parse(data); } catch (error) { - console.error('Error reading schema:', error); + console.error('Error reading schema:', error); +} + +const groupProps: [string, JSONSchema7][] = []; + +if (schema.properties) { + Object.entries(schema.properties).forEach(([key, definition]) => { + console.log(key) + if ( + typeof definition === 'object' && + definition !== null && + 'group' in definition && + definition.group === group + ) { + console.log("FOOO") + groupProps.push([key, definition]); + } + }); } ---
-

Schema Properties

-
    - {schema.properties && - Object.keys(schema.properties).map((key) => ( -
  • {key}
    • - ))} -
+

{group} Schema Properties

+
    + { + groupProps.map(([key, def]) => ( +
  • + {key} - {def.type} - {def.default} - {def.description} +
    • + )) + } +
diff --git a/docs/src/content/docs/for-administrators/existing-preprocessing-pipelines.md b/docs/src/content/docs/for-administrators/existing-preprocessing-pipelines.md index 348b3b6d1..a6047de52 100644 --- a/docs/src/content/docs/for-administrators/existing-preprocessing-pipelines.md +++ b/docs/src/content/docs/for-administrators/existing-preprocessing-pipelines.md @@ -28,12 +28,12 @@ Additionally the pipeline performs checks on the metadata fields. The checks are In the default configuration the pipeline performs: -- **type checks**: Checks that the type of each metadata field corresponds to the expected `type` value seen in the config (default is string). -- **required value checks**: Checks that if a field is required, e.g. `required` field in config is true, that that field is not None. -- **INSDC-accepted country checks**: Using the `process_options` preprocessing function checks that the `geoLocCountry` field is set to an [INSDC-accepted country](https://www.ebi.ac.uk/ena/browser/api/xml/ERC000011) option. +- **type checks**: Checks that the type of each metadata field corresponds to the expected `type` value seen in the config (default is string). +- **required value checks**: Checks that if a field is required, e.g. `required` field in config is true, that that field is not None. +- **INSDC-accepted country checks**: Using the `process_options` preprocessing function checks that the `geoLocCountry` field is set to an [INSDC-accepted country](https://www.ebi.ac.uk/ena/browser/api/xml/ERC000011) option. The pipeline also formats metadata fields: -- **parse timestamp**: Takes an ISO timestamp e.g. `2022-11-01T00:00:00Z` and returns that field in the `%Y-%m-%d` format. +- **parse timestamp**: Takes an ISO timestamp e.g. `2022-11-01T00:00:00Z` and returns that field in the `%Y-%m-%d` format. The code is available on [GitHub](https://github.com/loculus-project/loculus/tree/main/preprocessing/nextclade) under the [AGPL-3.0 license](https://github.com/loculus-project/loculus/blob/main/LICENSE). diff --git a/docs/src/content/docs/for-administrators/getting-started.md b/docs/src/content/docs/for-administrators/getting-started.md index 3eb2da76b..fad6ef931 100644 --- a/docs/src/content/docs/for-administrators/getting-started.md +++ b/docs/src/content/docs/for-administrators/getting-started.md @@ -33,12 +33,12 @@ We do not have a guide to deploy Loculus with Docker Compose at the moment but y You can compile and run Loculus from source code if you do not want to use Docker. We do not have a dedicated guide for this at the moment and recommend reading the [Docker Compose example](#with-docker-compose) to understand how the sub-services should be connected and the (developer) documentation of the individual services for getting them running: -- [Loculus backend](https://github.com/loculus-project/loculus/tree/main/backend) -- [Loculus website](https://github.com/loculus-project/loculus/tree/main/website) -- [PostgreSQL](https://www.postgresql.org/docs/) -- [Keycloak](https://www.keycloak.org/guides) -- [SILO](https://github.com/GenSpectrum/LAPIS-SILO) -- [LAPIS](https://github.com/GenSpectrum/LAPIS) -- Use the [Nextclade preprocessing pipeline](https://github.com/loculus-project/loculus/tree/main/preprocessing/nextclade) or follow the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md) to build your own custom pipeline +- [Loculus backend](https://github.com/loculus-project/loculus/tree/main/backend) +- [Loculus website](https://github.com/loculus-project/loculus/tree/main/website) +- [PostgreSQL](https://www.postgresql.org/docs/) +- [Keycloak](https://www.keycloak.org/guides) +- [SILO](https://github.com/GenSpectrum/LAPIS-SILO) +- [LAPIS](https://github.com/GenSpectrum/LAPIS) +- Use the [Nextclade preprocessing pipeline](https://github.com/loculus-project/loculus/tree/main/preprocessing/nextclade) or follow the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md) to build your own custom pipeline Please let us know if you are interested in using Loculus without Docker or Kubernetes! Your feedback will motivate us to create a guide. You are of course also very welcome to contribute to the documentation if you have successfully deployed a Loculus instance and have written down the steps. diff --git a/docs/src/content/docs/for-administrators/my-first-loculus.md b/docs/src/content/docs/for-administrators/my-first-loculus.md index 64f7539ad..7f55cf031 100644 --- a/docs/src/content/docs/for-administrators/my-first-loculus.md +++ b/docs/src/content/docs/for-administrators/my-first-loculus.md @@ -163,7 +163,6 @@ createTestAccounts: true ``` - Because we have enabled the `createTestAccounts` option, we need to delete the existing keycloak database to ensure that the test users are added. First we need to run `kubectl get pods` to get the name of the keycloak pod, which will be something like `loculus-keycloak-database-665b964c6b-gm9t5` (but with the random string at the end being different). @@ -182,7 +181,6 @@ Now we can upgrade the Loculus installation again: helm upgrade loculus ./kubernetes/loculus --set environment=local --set branch=latest --set disableIngest=true --set disableEnaSubmission=true -f custom_values.yaml ``` - ### Testing it out with some data While that's getting ready, let's create some data to submit. diff --git a/docs/src/content/docs/for-administrators/schema-designs.md b/docs/src/content/docs/for-administrators/schema-designs.md index a16b8e6a9..b9e88649f 100644 --- a/docs/src/content/docs/for-administrators/schema-designs.md +++ b/docs/src/content/docs/for-administrators/schema-designs.md @@ -5,10 +5,10 @@ description: Different ways to design the schema of a Loculus instance Loculus is very flexible in its data model and there are different ways to design the [schema](../../introduction/glossary#schema). Technically, a Loculus instance can have one or multiple organisms and each organism has -- a set of metadata fields -- a set of unaligned nucleotide sequences -- a set of aligned nucleotide sequences -- a set of aligned amino acid sequences +- a set of metadata fields +- a set of unaligned nucleotide sequences +- a set of aligned nucleotide sequences +- a set of aligned amino acid sequences The different nucleotide sequences are called segments and the different amino acid sequences are called genes but they do not need to be biological segments and genes. If there is only one nucleotide sequence, it may but does not need to have a name. If there are multiple nucleotide sequences, they must be named. The amino acid sequences must always be named. @@ -24,9 +24,9 @@ This is the typical model for Loculus. The Loculus instance contains one or more This is a good model if: -- Each sample (taken from the host) only has one (possibly multi-segmented) sequence. -- For each organism, it is clear which reference genome to use. -- Users are expected to analyze the organisms independently (e.g., users don’t desire a table containing sequences from different organisms). +- Each sample (taken from the host) only has one (possibly multi-segmented) sequence. +- For each organism, it is clear which reference genome to use. +- Users are expected to analyze the organisms independently (e.g., users don’t desire a table containing sequences from different organisms). ### One organism for everything @@ -34,9 +34,9 @@ On the opposite end of the spectrum, it is possible to only have one “technica This is a good model if: -- Samples are sequenced with a multi-pathogen panel and may contain sequences from one or multiple pathogens (i.e., co-infections). -- Sequences of different organisms share the same (sampling and host) metadata. -- Users want to see co-infection data (e.g., a sequence details page listing all sequences from the sample). +- Samples are sequenced with a multi-pathogen panel and may contain sequences from one or multiple pathogens (i.e., co-infections). +- Sequences of different organisms share the same (sampling and host) metadata. +- Users want to see co-infection data (e.g., a sequence details page listing all sequences from the sample). ### Multiple references for an organism diff --git a/docs/src/content/docs/for-administrators/setup-with-k3d-and-nginx.mdx b/docs/src/content/docs/for-administrators/setup-with-k3d-and-nginx.mdx index 8f5f1406a..78eabaed2 100644 --- a/docs/src/content/docs/for-administrators/setup-with-k3d-and-nginx.mdx +++ b/docs/src/content/docs/for-administrators/setup-with-k3d-and-nginx.mdx @@ -11,13 +11,13 @@ We will use [k3d](https://k3d.io/) to run a Kubernetes cluster within Docker, [n ## Prerequisites -- A Linux server (e.g. Ubuntu 24.04) with ideally at least 4 CPUs and 8 GB RAM -- A domain -- Install Docker: see [this guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) -- Install k3d: see [this guide](https://k3d.io/stable/#installation) -- Install kubectl: see [this guide](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-kubectl-on-linux) -- Install Helm: see [this guide](https://helm.sh/docs/intro/install/#from-apt-debianubuntu) -- Install nginx: see [this guide](https://ubuntu.com/tutorials/install-and-configure-nginx#2-installing-nginx) +- A Linux server (e.g. Ubuntu 24.04) with ideally at least 4 CPUs and 8 GB RAM +- A domain +- Install Docker: see [this guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) +- Install k3d: see [this guide](https://k3d.io/stable/#installation) +- Install kubectl: see [this guide](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-kubectl-on-linux) +- Install Helm: see [this guide](https://helm.sh/docs/intro/install/#from-apt-debianubuntu) +- Install nginx: see [this guide](https://ubuntu.com/tutorials/install-and-configure-nginx#2-installing-nginx) ## Step 1: Create a k3d cluster @@ -195,9 +195,9 @@ In the output, you should see somewhere: You have to add DNS entries of your domain to point to the server. Please add DNS entries for the following domains and sub-domains: -- `` -- `api.` -- `auth.` +- `` +- `api.` +- `auth.` ## Step 6: Set up nginx diff --git a/docs/src/content/docs/for-users/approve-submissions.md b/docs/src/content/docs/for-users/approve-submissions.md index 50285b395..a02878624 100644 --- a/docs/src/content/docs/for-users/approve-submissions.md +++ b/docs/src/content/docs/for-users/approve-submissions.md @@ -10,9 +10,9 @@ After submitting sequences, you'll be taken to a page showing the progress of pr Sequences that have finished processing will show different icons to indicate whether there were any issues during the processing. -- Green checkmark - indicates that processing was entirely successful -- Yellow checkmark - indicates that processing encountered warnings -- Red label - indicates that there were errors during processing +- Green checkmark - indicates that processing was entirely successful +- Yellow checkmark - indicates that processing encountered warnings +- Red label - indicates that there were errors during processing We highly recommend checking all sequences with warnings to see if they could be rectified or indicate a larger problem with the data. @@ -50,8 +50,8 @@ You can retrieve a list of uploaded but not released sequences by sending a GET The `sequenceEntries` field of the returned object contains a list of sequences with their corresponding `status`: -- Sequence that are in the status `RECEIVED` have not yet been processed. This should usually happen within a few minutes. -- Sequences that are in the status `IN_PROCESSING` are currently being processed, please wait a few more moments. +- Sequence that are in the status `RECEIVED` have not yet been processed. This should usually happen within a few minutes. +- Sequences that are in the status `IN_PROCESSING` are currently being processed, please wait a few more moments. - Sequences that are in the status `PROCESSED` are done processing. If they do not have errors, they can be approved. To find out more about errors, we recommend going to the review page on the website: you can find it by going to the Submission Portal and clicking on "Review". A cURL request could be: diff --git a/docs/src/content/docs/for-users/revoke-sequences.md b/docs/src/content/docs/for-users/revoke-sequences.md index 0187d8d7d..fc572e3a9 100644 --- a/docs/src/content/docs/for-users/revoke-sequences.md +++ b/docs/src/content/docs/for-users/revoke-sequences.md @@ -10,11 +10,11 @@ Note that revoking a sequence **does not delete it**. While it will not turn up Revocation can only be done by members of the group that originally submitted that sequence. -- Find the sequence you would like to revoke via searching the main database, or by viewing your submitted sequences. - (Go to 'Submit' and 'View sequences'). -- Scroll to the bottom of the sequence page and find the 'Revoke this sequence' button in the bottom-left corner. -- Confirm you want to revoke the sequence -- You will be taken to the sequence review page - press 'Release' to release the revoked sequence -- Confirm you want to release the sequence +- Find the sequence you would like to revoke via searching the main database, or by viewing your submitted sequences. + (Go to 'Submit' and 'View sequences'). +- Scroll to the bottom of the sequence page and find the 'Revoke this sequence' button in the bottom-left corner. +- Confirm you want to revoke the sequence +- You will be taken to the sequence review page - press 'Release' to release the revoked sequence +- Confirm you want to release the sequence The sequence is now revoked. diff --git a/docs/src/content/docs/for-users/submit-sequences.md b/docs/src/content/docs/for-users/submit-sequences.md index 284ec65a1..88dec5bf0 100644 --- a/docs/src/content/docs/for-users/submit-sequences.md +++ b/docs/src/content/docs/for-users/submit-sequences.md @@ -8,8 +8,8 @@ Before you begin this process, you should ensure your data is in the correct for Loculus expects: -- Sequence data in `fasta` format with a unique submissionID per sequence. -- Metadata for each sequence. If you upload through the API, only `tsv` is supported. If you upload through the Website, you can also use Excel files (`xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page in each of the supported formats. You can also map columns in your file to the expected upload column names by clicking the 'Add column mapping' button. +- Sequence data in `fasta` format with a unique submissionID per sequence. +- Metadata for each sequence. If you upload through the API, only `tsv` is supported. If you upload through the Website, you can also use Excel files (`xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page in each of the supported formats. You can also map columns in your file to the expected upload column names by clicking the 'Add column mapping' button. ![Metadata template.](../../../assets/MetadataTemplate.png) diff --git a/docs/src/content/docs/introduction/system-overview.md b/docs/src/content/docs/introduction/system-overview.md index 67352cdd5..a69b19a77 100644 --- a/docs/src/content/docs/introduction/system-overview.md +++ b/docs/src/content/docs/introduction/system-overview.md @@ -7,14 +7,14 @@ Below, we provide an overview of the components of Loculus. This is in particula Loculus has a modular architecture and consists of several sub-services: -- **Backend server:** The backend is the central service of Loculus and most other services interact with it. It manages all persistent data (such as the sequence entries and submitting groups) except the data for user authentication which are managed by Keycloak. -- **Backend database:** The backend server stores all the data in a PostgreSQL database and it is the only service that has direct access to this database. -- **Keycloak:** Loculus uses the open-source software [Keycloak](https://github.com/keycloak/keycloak) for identity and access management which supports features such as password reset and two-factor authentication. Each Loculus instance includes a Keycloak instance. (Note: Keycloak is a local/on-premise software, data are not sent to any cloud service!) -- **Keycloak database:** Keycloak also uses a PostgreSQL database to store its data (this may be hosted by the same PostgreSQL server as the backend database). -- **SILO(s):** [SILO](https://github.com/GenSpectrum/LAPIS-SILO) is an open-source query engine for genetic sequences optimized for high performance and supporting alignment-specific queries such as mutation searches. It regularly pulls data from the backend server and indexes them. By default, SILO is not exposed to the users but accessed via LAPIS. For each [organism](../glossary#organism) of a Loculus [instance](../glossary#instance), there is a separate instance of SILO. -- **LAPIS(es):** [LAPIS](https://github.com/GenSpectrum/LAPIS) provides a convenient interface to SILO, offering a lightweight web API and additional data and compression formats. For each SILO instance, there is a corresponding LAPIS instance. -- **Website:** The frontend application of Loculus accesses the APIs of the backend server and LAPIS. It uses the backend server for everything related to data submission and LAPIS for searching and downloading released data. For logins and registrations, users are redirected to Keycloak. -- **Preprocessing pipeline(s):** A preprocessing pipeline fetches [unprocessed/user-submitted data](../glossary#unprocessed-data) from the backend server, processes them (which usually includes cleaning, alignment and adding annotations), and sends [processed data](../glossary#processed-data) back to the backend server. The pipeline contains [organism](../glossary#organism)-specific logic, thus, there is a separate pipeline for each organism. We maintain a customizable preprocessing pipeline that uses [Nextclade](https://github.com/nextstrain/nextclade) for alignment, quality checks and annotations but it is easy to write a new one by following the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md). +- **Backend server:** The backend is the central service of Loculus and most other services interact with it. It manages all persistent data (such as the sequence entries and submitting groups) except the data for user authentication which are managed by Keycloak. +- **Backend database:** The backend server stores all the data in a PostgreSQL database and it is the only service that has direct access to this database. +- **Keycloak:** Loculus uses the open-source software [Keycloak](https://github.com/keycloak/keycloak) for identity and access management which supports features such as password reset and two-factor authentication. Each Loculus instance includes a Keycloak instance. (Note: Keycloak is a local/on-premise software, data are not sent to any cloud service!) +- **Keycloak database:** Keycloak also uses a PostgreSQL database to store its data (this may be hosted by the same PostgreSQL server as the backend database). +- **SILO(s):** [SILO](https://github.com/GenSpectrum/LAPIS-SILO) is an open-source query engine for genetic sequences optimized for high performance and supporting alignment-specific queries such as mutation searches. It regularly pulls data from the backend server and indexes them. By default, SILO is not exposed to the users but accessed via LAPIS. For each [organism](../glossary#organism) of a Loculus [instance](../glossary#instance), there is a separate instance of SILO. +- **LAPIS(es):** [LAPIS](https://github.com/GenSpectrum/LAPIS) provides a convenient interface to SILO, offering a lightweight web API and additional data and compression formats. For each SILO instance, there is a corresponding LAPIS instance. +- **Website:** The frontend application of Loculus accesses the APIs of the backend server and LAPIS. It uses the backend server for everything related to data submission and LAPIS for searching and downloading released data. For logins and registrations, users are redirected to Keycloak. +- **Preprocessing pipeline(s):** A preprocessing pipeline fetches [unprocessed/user-submitted data](../glossary#unprocessed-data) from the backend server, processes them (which usually includes cleaning, alignment and adding annotations), and sends [processed data](../glossary#processed-data) back to the backend server. The pipeline contains [organism](../glossary#organism)-specific logic, thus, there is a separate pipeline for each organism. We maintain a customizable preprocessing pipeline that uses [Nextclade](https://github.com/nextstrain/nextclade) for alignment, quality checks and annotations but it is easy to write a new one by following the [preprocessing pipeline specifications](https://github.com/loculus-project/loculus/blob/main/preprocessing/specification.md). ![Architecture overview](./architectureOverview.svg) @@ -22,7 +22,7 @@ Loculus has a modular architecture and consists of several sub-services: Why did we choose this (rather complex) architecture? Here are some of the key advantages: -- **Performant query engine and existing API:** By using LAPIS and SILO (which were originally developed for [CoV-Spectrum](https://cov-spectrum.org)), Loculus reuses a query engine that has proven to be able to handle datasets with millions of viral sequences and offers a simple and flexible API. -- **Pathogen agnostic:** By keeping the preprocessing pipeline separate from the backend server, it is very simple to develop new and pathogen-specific pipelines. The pipelines do not need to be written in the same programming language as the backend but it is possible to use any language and framework and call existing tools. -- **Secure and convenient authentication:** By using Keycloak, Loculus offers a secure authentication system with support for two-factor authentication and social identity providers such as ORCID. It is also possible to connect it with LDAP or Active Directory servers which might be useful for institution-internal systems. -- **Easy to back up:** Despite having many sub-services, it is simple to create backups because many services do not persist own data. All data are stored in only two PostgreSQL databases: the backend database and the Keycloak database. +- **Performant query engine and existing API:** By using LAPIS and SILO (which were originally developed for [CoV-Spectrum](https://cov-spectrum.org)), Loculus reuses a query engine that has proven to be able to handle datasets with millions of viral sequences and offers a simple and flexible API. +- **Pathogen agnostic:** By keeping the preprocessing pipeline separate from the backend server, it is very simple to develop new and pathogen-specific pipelines. The pipelines do not need to be written in the same programming language as the backend but it is possible to use any language and framework and call existing tools. +- **Secure and convenient authentication:** By using Keycloak, Loculus offers a secure authentication system with support for two-factor authentication and social identity providers such as ORCID. It is also possible to connect it with LDAP or Active Directory servers which might be useful for institution-internal systems. +- **Easy to back up:** Despite having many sub-services, it is simple to create backups because many services do not persist own data. All data are stored in only two PostgreSQL databases: the backend database and the Keycloak database. diff --git a/docs/src/content/docs/introduction/what-is-loculus.md b/docs/src/content/docs/introduction/what-is-loculus.md index c18d79aa3..2e72456a4 100644 --- a/docs/src/content/docs/introduction/what-is-loculus.md +++ b/docs/src/content/docs/introduction/what-is-loculus.md @@ -5,9 +5,9 @@ description: A brief description of Loculus Loculus is a software package to power microbial genomial databases. Major features include: -- Upload and storage of consensus sequences and metadata using a simple web interface or a web API -- Flexible data preprocessing: Loculus comes with a [Nextclade](https://clades.nextstrain.org)-based preprocessing pipeline that is able to align and translate sequences, but it is also easy to implement and plug-in your own pipeline using custom tools. -- Powerful searches: Loculus provides a user-friendly interface to search and view sequences as well as an API to query the data using [LAPIS](https://github.com/GenSpectrum/LAPIS) in the backend. -- Highly configurable: The list of metadata fields is fully configurable and Loculus supports both single- and multi-segmented genomes. +- Upload and storage of consensus sequences and metadata using a simple web interface or a web API +- Flexible data preprocessing: Loculus comes with a [Nextclade](https://clades.nextstrain.org)-based preprocessing pipeline that is able to align and translate sequences, but it is also easy to implement and plug-in your own pipeline using custom tools. +- Powerful searches: Loculus provides a user-friendly interface to search and view sequences as well as an API to query the data using [LAPIS](https://github.com/GenSpectrum/LAPIS) in the backend. +- Highly configurable: The list of metadata fields is fully configurable and Loculus supports both single- and multi-segmented genomes. Loculus aims to be widely useful to groups who need to manage sequencing data. It can be used by small public health or research laboratories with a few members for storing their own, internal data as well as by international databases facilitating global pathogen sequence sharing. diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 6f6290ed3..2c9f258c9 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -2,6 +2,7 @@ title: 'Helm chart config' description: A reference of config fields for the Loculus Helm chart --- + import SchemaDocs from '../../../components/SchemaDocs.astro'; :::note @@ -12,9 +13,7 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## General Settings - - - +
@@ -471,6 +470,7 @@ For production environments, these should always be set to false. Instead, exter +
### Lineage system definitions @@ -479,11 +479,11 @@ Here's an example of a `lineageDefinitions` section: ```yaml lineageSystemDefinitions: - pangoLineage: # Lineage name to use in metadata fields - 1: https://example.org/lineage_defintions_v1.yaml # Definition per pipeline version - 2: https://example.org/lineage_defintions_v2.yaml - myLineage: - 1: ... + pangoLineage: # Lineage name to use in metadata fields + 1: https://example.org/lineage_defintions_v1.yaml # Definition per pipeline version + 2: https://example.org/lineage_defintions_v2.yaml + myLineage: + 1: ... ``` @@ -631,13 +631,12 @@ Each organism object has the following fields: @@ -651,8 +650,8 @@ Each organism object has the following fields: @@ -807,9 +806,9 @@ Each organism object has the following fields: @@ -828,7 +827,10 @@ Each organism object has the following fields: - + @@ -932,8 +934,9 @@ Each organism object has the following fields: diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index a0b7ef6b1..9cbdd3de6 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -3,11 +3,13 @@ "type": "object", "properties": { "name": { + "group": "general", "type": "string", "default": "Loculus", "description": "The name of the Loculus instance" }, "logo": { + "group": "general", "type": "object", "properties": { "url": { @@ -27,6 +29,7 @@ } }, "accessionPrefix": { + "group": "general", "type": "string", "default": "LOC_", "description": "Prefix used for accession numbers" From 9f876b430db3b87fdecbaca841334b451ca2608c Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 11:38:52 +0100 Subject: [PATCH 07/56] Add table --- docs/src/components/SchemaDocs.astro | 31 +++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index d40d8863e..fc8adf178 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -32,13 +32,30 @@ if (schema.properties) {

{group} Schema Properties

+
+
Object - Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. - For each version of an accession, the `earliestReleaseDate` is calculated as the earliest date - of the internal release date, the dates in the configured `externalFields` and the value from the - previous version of the accession (if there is one). - This can be used when having a mix of sequences imported from other databases, as well as sequences - released first in this Loculus instance, to have a field that shows the earliest release date regardless - of where the sequence was first released. + Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. For each + version of an accession, the `earliestReleaseDate` is calculated as the earliest date of the internal + release date, the dates in the configured `externalFields` and the value from the previous version of + the accession (if there is one). This can be used when having a mix of sequences imported from other + databases, as well as sequences released first in this Loculus instance, to have a field that shows the + earliest release date regardless of where the sequence was first released.
Array of strings - Field names to use when calculating the earliest release date. - The fields need to be nullable strings formated with `yyyy-mm-dd`. + Field names to use when calculating the earliest release date. The fields need to be nullable strings + formated with `yyyy-mm-dd`.
Boolean - If true, search results will contain results that contain the given value as a substring. - If false (the default), only exact matches will be returned. - This only works for string fields, and you cannot also enable `autocomplete`. + If true, search results will contain results that contain the given value as a substring. If false (the + default), only exact matches will be returned. This only works for string fields, and you cannot also + enable `autocomplete`.
`rangeOverlapSearch.rangeId` String The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID. + The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the + same range ID. +
`rangeOverlapSearch.rangeDisplayName`String - Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. - The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key. + Use this on string fields that contain lineages, if you want to enable searches that can include + sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` + key.
+ + + + + + + + + + { + groupProps.map(([key, def]) => ( + + + + + + + )) + } + +
FieldTypeDefaultDescription
{key}{def.type}{def.default}{def.description}
+
    - { - groupProps.map(([key, def]) => ( -
  • - {key} - {def.type} - {def.default} - {def.description} -
    • - )) - }
From d5c02c606ffdab987bb1a2aceee2b3df046a207b Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 14:55:13 +0100 Subject: [PATCH 08/56] ... --- docs/src/components/SchemaDocs.astro | 2 +- kubernetes/loculus/values.schema.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index fc8adf178..8ee9d6d82 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -46,7 +46,7 @@ if (schema.properties) { { groupProps.map(([key, def]) => ( - {key} + {key} {def.type} {def.default} {def.description} diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 9cbdd3de6..6b819affc 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -11,6 +11,7 @@ "logo": { "group": "general", "type": "object", + "description": "Configuration for the logo", "properties": { "url": { "type": "string", From c700cd6d9d67310256b41daf0d8740153d51876a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 15:20:28 +0100 Subject: [PATCH 09/56] Make recursive looking up of props possible --- docs/src/components/SchemaDocs.astro | 49 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 8ee9d6d82..764c41a41 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -12,20 +12,37 @@ try { console.error('Error reading schema:', error); } -const groupProps: [string, JSONSchema7][] = []; +const rows: Row[] = []; + +interface Row { + key: string, + type?: string, + default?: string, + description?: string +} + +function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7Definition) { + if ( + typeof definition === 'object' && + definition !== null && + (('group' in definition && + definition.group === group) || prefix !== "") + ) { + rows.push({ + key: `${prefix}${key}`, + type: String(definition.type), + default: definition.default ? String(definition.default) : "", + description: definition.description + }) + if ('properties' in definition && definition.properties) { + Object.entries(definition.properties).forEach(([k, d]) => addSelfAndChildren(`${prefix}${key}.`, k, d)); + } + } +} if (schema.properties) { Object.entries(schema.properties).forEach(([key, definition]) => { - console.log(key) - if ( - typeof definition === 'object' && - definition !== null && - 'group' in definition && - definition.group === group - ) { - console.log("FOOO") - groupProps.push([key, definition]); - } + addSelfAndChildren("", key, definition); }); } --- @@ -44,12 +61,12 @@ if (schema.properties) { { - groupProps.map(([key, def]) => ( + rows.map(row => ( - {key} - {def.type} - {def.default} - {def.description} + {row.key} + {row.type} + {row.default} + {row.description} )) } From 151fccf4ffb2b51782829214912c0652998fe214 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:31:55 +0100 Subject: [PATCH 10/56] general settings complete --- docs/src/components/SchemaDocs.astro | 64 ++++---- .../docs/reference/helm-chart-config.mdx | 138 ------------------ kubernetes/loculus/values.schema.json | 56 +++++-- 3 files changed, 79 insertions(+), 179 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 764c41a41..875f30e5d 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,6 +1,7 @@ --- import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition } from 'json-schema'; +import { defineAction } from 'astro:actions'; const { schemaPath, group } = Astro.props; let schema: JSONSchema7 = {}; @@ -25,8 +26,9 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 if ( typeof definition === 'object' && definition !== null && - (('group' in definition && - definition.group === group) || prefix !== "") + 'groups' in definition && + Array.isArray(definition.groups) && + definition.groups.includes(group) ) { rows.push({ key: `${prefix}${key}`, @@ -37,6 +39,9 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 if ('properties' in definition && definition.properties) { Object.entries(definition.properties).forEach(([k, d]) => addSelfAndChildren(`${prefix}${key}.`, k, d)); } + if ('patternProperties' in definition && definition.patternProperties) { + Object.entries(definition.patternProperties).forEach(([k, d]) => addSelfAndChildren(`${prefix}${key}.`, k, d)); + } } } @@ -45,34 +50,33 @@ if (schema.properties) { addSelfAndChildren("", key, definition); }); } + +// TODO 'desciription' should be markdown processed so `` works as intended +// TODO string defaults should be encapsulated in "" + --- -
-

{group} Schema Properties

-
- - - - - - - - - - - { - rows.map(row => ( - - - - - - - )) - } - -
FieldTypeDefaultDescription
{row.key}{row.type}{row.default}{row.description}
-
-
    -
+
+ + + + + + + + + + + { + rows.map(row => ( + + + + + + + )) + } + +
FieldTypeDefaultDescription
{row.key}{row.type}{row.default}{row.description}
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 2c9f258c9..3a3b8c6e8 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -26,144 +26,6 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll - - `name` - String - "Loculus" - The name of the Loculus instance - - - `logo` - Object - - Configuration for the logo - - - `logo.url` - String - _Loculus logo_ - URL path to the logo image file - - - `logo.width` - Integer - 100 - Width of the logo in pixels - - - `logo.height` - Integer - 100 - Height of the logo in pixels - - - `accessionPrefix` - String - "LOC_" - Prefix used for accession numbers - - - `environment` - String - "server" - Deployment environment. Options: “local” for development, “server” for production. - - - `host` - String - - Hostname where Loculus will be accessible. Used for generating URLs and configuring ingress. - - - `bannerMessage` - String - "This is a demonstration environment. It may contain non-accurate test data and should not be used for real-world applications. Data will be deleted regularly." - - Banner message (as HTML) to display at the very top of the page - - - - `welcomeMessageHTML` - String - - - A custom welcome message to be shown on the landing page - - - - `additionalHeadHTML` - String - - - Additional HTML to inject into the `` of pages - - - - `createTestAccounts` - Boolean - - If true, creates the users _testuser_ and _superuser_ (the username and password are the same). - - - `robotsNoindexHeader` - Boolean - - If true, adds a noindex header to prevent search engine indexing - - - `seqSets.enabled` - Boolean - - Enable/disable SeqSets. If false, `seqSets.crossRef` can be omitted. - - - `seqSets.crossRef` - Object - - Configuration for CrossRef integration. Set to `null` to disable CrossRef integration (you can still use SeqSets without CrossRef DOIs). - - - `seqSets.crossRef.DOIPrefix` - String - - The DOI prefix for SeqSets - - - `seqSets.crossRef.endpoint` - String - - The API endpoint for CrossRef - - - `seqSets.crossRef.databaseName` - String - - The database name for CrossRef - - - `seqSets.crossRef.email` - String - - The email address associated with the CrossRef account - - - `seqSets.crossRef.organization` - String - - The organization name for CrossRef - - - `seqSets.crossRef.hostUrl` - String - - The host URL for CrossRef callbacks - - - `gitHubMainUrl` - String - "https://github.com/loculus-project/loculus" - The link that the GitHub icon in the footer points to - `dataUseTerms.enabled` Boolean diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 6b819affc..9dac72afc 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -3,26 +3,29 @@ "type": "object", "properties": { "name": { - "group": "general", + "groups": ["general"], "type": "string", "default": "Loculus", "description": "The name of the Loculus instance" }, "logo": { - "group": "general", + "groups": ["general"], "type": "object", "description": "Configuration for the logo", "properties": { "url": { + "groups": ["general"], "type": "string", "description": "URL path to the logo image file" }, "width": { + "groups": ["general"], "type": "integer", "default": 100, "description": "Width of the logo in pixels" }, "height": { + "groups": ["general"], "type": "integer", "default": 100, "description": "Height of the logo in pixels" @@ -30,74 +33,105 @@ } }, "accessionPrefix": { + "groups": ["general"], "group": "general", "type": "string", "default": "LOC_", "description": "Prefix used for accession numbers" }, "environment": { + "groups": ["general"], "type": "string", "enum": ["local", "server"], "default": "server", "description": "Deployment environment. local for development, server for production" }, "host": { + "groups": ["general"], "type": "string", "description": "Hostname where Loculus will be accessible" }, "bannerMessage": { + "groups": ["general"], "type": "string", + "default": "This is a demonstration environment. It may contain non-accurate test data and should not be used for real-world applications. Data will be deleted regularly.", "description": "Banner message (as HTML) to display at the very top of the page" }, "welcomeMessageHTML": { - "type": ["string", "null"], + "groups": ["general"], + "type": "string", "description": "A custom welcome message to be shown on the landing page" }, "additionalHeadHTML": { + "groups": ["general"], "type": "string", "description": "Additional HTML to inject into the of pages" }, "createTestAccounts": { + "groups": ["general"], "type": "boolean", "description": "If true, creates the users testuser and superuser" }, "robotsNoindexHeader": { + "groups": ["general"], "type": "boolean", "description": "If true, adds a noindex header to prevent search engine indexing" }, "seqSets": { + "groups": ["general"], "type": "object", "properties": { "enabled": { + "groups": ["general"], "type": "boolean", - "description": "Enable/disable SeqSets" + "description": "Enable/disable SeqSets. If false, `seqSets.crossRef` can be omitted." }, "crossRef": { - "type": ["object", "null"], + "groups": ["general"], + "type": ["object"], + "description": "Configuration for CrossRef integration. Set to `null` to disable CrossRef integration (you can still use SeqSets without CrossRef DOIs).", "properties": { "DOIPrefix": { - "type": "string" + "groups": ["general"], + "type": "string", + "description": "The DOI prefix for SeqSets" }, "endpoint": { - "type": "string" + "groups": ["general"], + "type": "string", + "description": "The API endpoint for CrossRef" }, "databaseName": { - "type": "string" + "groups": ["general"], + "type": "string", + "description": "The database name for CrossRef" }, "email": { + "groups": ["general"], "type": "string", - "format": "email" + "format": "email", + "description": "The email address associated with the CrossRef account" }, "organization": { - "type": "string" + "groups": ["general"], + "type": "string", + "description": "The organization name for CrossRef" }, "hostUrl": { - "type": "string" + "groups": ["general"], + "type": "string", + "description": "The host URL for CrossRef callbacks" } } } } }, + "gitHubMainUrl": { + "groups": ["general"], + "type": "string", + "default": "https://github.com/loculus-project/loculus", + "description": "The link that the GitHub icon in the footer points to" + }, "website": { "websiteConfig": { "type": "object", From 56ec1c89b53d68fe91be228b0362abc74ec9b807 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 15:45:06 +0100 Subject: [PATCH 11/56] Do the website config section --- docs/src/components/SchemaDocs.astro | 25 +++--- .../docs/reference/helm-chart-config.mdx | 75 +---------------- kubernetes/loculus/values.schema.json | 81 ++++++++++++------- 3 files changed, 66 insertions(+), 115 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 875f30e5d..bf6c9debe 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,7 +1,6 @@ --- import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition } from 'json-schema'; -import { defineAction } from 'astro:actions'; const { schemaPath, group } = Astro.props; let schema: JSONSchema7 = {}; @@ -25,17 +24,20 @@ interface Row { function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7Definition) { if ( typeof definition === 'object' && - definition !== null && - 'groups' in definition && - Array.isArray(definition.groups) && - definition.groups.includes(group) + definition !== null ) { - rows.push({ - key: `${prefix}${key}`, - type: String(definition.type), - default: definition.default ? String(definition.default) : "", - description: definition.description - }) + if ( + 'groups' in definition && + Array.isArray(definition.groups) && + definition.groups.includes(group) + ) { + rows.push({ + key: `${prefix}${key}`, + type: String(definition.type), + default: definition.default ? String(definition.default) : "", + description: definition.description + }) + } if ('properties' in definition && definition.properties) { Object.entries(definition.properties).forEach(([k, d]) => addSelfAndChildren(`${prefix}${key}.`, k, d)); } @@ -53,6 +55,7 @@ if (schema.properties) { // TODO 'desciription' should be markdown processed so `` works as intended // TODO string defaults should be encapsulated in "" +// type should be with first letter uppercase --- diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 3a3b8c6e8..16dd4eacb 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -50,80 +50,7 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## Website Settings -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`website`ObjectWebsite specific setting
`website.websiteConfig`ObjectSettings for the `website_config.json`
`website.websiteConfig.enableLoginNavigationItem`BooleantrueWhether the website should show the login button.
`website.websiteConfig.enableSubmissionNavigationItem`BooleantrueWhether the website should show "Submit" link in the top navigation bar.
`website.websiteConfig.enableSubmissionPages`Booleantrue - Whether to completely disable submission related pages. Setting this to false is useful when hosting - Loculus for analysis-only purposes. -
`website.runtimeConfig.public`ObjectSettings for the `public` section of the `runtime_config.json`
`website.runtimeConfig.public.backendUrl`StringtrueOverwrite the URL where the client-side website code expects the Loculus backend
`website.runtimeConfig.public.keycloakUrl`StringtrueOverwrite the URL where the client-side website code expects the Keycloak
`website.runtimeConfig.public.lapisUrlTemplate`Stringtrue - Overwrite the URLs where the client-side website code expects the LAPIS instances. Must contain - `%organism%` as a placeholder. -
-
+ ## User registration and authentication diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 9dac72afc..4bbdd4376 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -133,37 +133,58 @@ "description": "The link that the GitHub icon in the footer points to" }, "website": { - "websiteConfig": { - "type": "object", - "properties": { - "enableLoginNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionNavigationItem": { - "type": "boolean", - "default": true - }, - "enableSubmissionPages": { - "type": "boolean", - "default": true + "groups": ["website"], + "type": "object", + "description": "Website specific setting", + "properties": { + "websiteConfig": { + "groups": ["website"], + "type": "object", + "description": "Settings for the `website_config.json`", + "properties": { + "enableLoginNavigationItem": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether the website should show the login button." + }, + "enableSubmissionNavigationItem": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether the website should show \"Submit\" link in the top navigation bar." + }, + "enableSubmissionPages": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether to completely disable submission related pages. Setting this to false is useful when hosting Loculus for analysis-only purposes." + } } - } - }, - "runtimeConfig": { - "type": "object", - "properties": { - "public": { - "type": "object", - "properties": { - "backendUrl": { - "type": "string" - }, - "keycloakUrl": { - "type": "string" - }, - "lapisUrlTemplate": { - "type": "string" + }, + "runtimeConfig": { + "type": "object", + "properties": { + "public": { + "groups": ["website"], + "type": "object", + "description": "Settings for the `public` section of the `runtime_config.json`", + "properties": { + "backendUrl": { + "groups": ["website"], + "type": "string", + "description": "Overwrite the URL where the client-side website code expects the Loculus backend" + }, + "keycloakUrl": { + "groups": ["website"], + "type": "string", + "description": "Overwrite the URL where the client-side website code expects the Keycloak" + }, + "lapisUrlTemplate": { + "groups": ["website"], + "type": "string", + "description": "Overwrite the URLs where the client-side website code expects the LAPIS instances. Must contain `%organism%` as a placeholder." + } } } } From a5084dbcc1fdcc7879ee7cc4d81600e15cff5806 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:01:43 +0100 Subject: [PATCH 12/56] converted auth settings --- docs/src/components/SchemaDocs.astro | 14 ++- .../docs/reference/helm-chart-config.mdx | 93 +----------------- kubernetes/loculus/values.schema.json | 94 ++++++++++++++----- 3 files changed, 84 insertions(+), 117 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index bf6c9debe..12d07a2f7 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -21,6 +21,10 @@ interface Row { description?: string } +function capitalizeFirst(str: string) { + return str.charAt(0).toUpperCase() + str.slice(1); +} + function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7Definition) { if ( typeof definition === 'object' && @@ -31,10 +35,14 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 Array.isArray(definition.groups) && definition.groups.includes(group) ) { + var def = definition.default !== undefined ? String(definition.default) : ""; + if (definition.type === "string" && def !== "") { + def = `"${def}"` + } rows.push({ key: `${prefix}${key}`, - type: String(definition.type), - default: definition.default ? String(definition.default) : "", + type: capitalizeFirst(String(definition.type)), + default: def, description: definition.description }) } @@ -54,8 +62,6 @@ if (schema.properties) { } // TODO 'desciription' should be markdown processed so `` works as intended -// TODO string defaults should be encapsulated in "" -// type should be with first letter uppercase --- diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 16dd4eacb..e3aa7b2d9 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -54,98 +54,7 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## User registration and authentication - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`auth`ObjectUser authentication (Keycloak settings)
`auth.smtp`ObjectConfiguration for email sending
`auth.smtp.host`StringSMTP server hostname
`auth.smtp.port`IntegerSMTP server port
`auth.smtp.user`StringSMTP username for authentication
`auth.smtp.replyTo`StringReply-to email address for sent emails
`auth.smtp.from`StringFrom email address for sent emails
`auth.smtp.envelopeFrom`StringEnvelope from address for sent emails
`auth.verifyEmail`BooleantrueIf true, requires email verification for new accounts
`auth.resetPasswordAllowed`BooleantrueIf true, allows users to reset their passwords
`auth.registrationAllowed`BooleantrueIf true, allows users to register new accounts in Keycloak.
`insecureCookies`BooleanfalseIf true, allows insecure cookies.
`registrationTermsMessage`String - `"You must agree to the terms of use."` - Message displayed during user registration, typically including terms of service.
+ ## Database deployments diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 4bbdd4376..321361eb8 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -192,30 +192,82 @@ } }, "auth": { - "smtp": { - "type": "object", - "properties": { - "host": { "type": "string" }, - "port": { "type": "integer" }, - "user": { "type": "string" }, - "replyTo": { "type": "string", "format": "email" }, - "from": { "type": "string", "format": "email" }, - "envelopeFrom": { "type": "string", "format": "email" } + "groups": ["auth"], + "type": "object", + "description": "User authentication (Keycloak settings)", + "properties": { + "smtp": { + "groups": ["auth"], + "type": "object", + "description": "Configuration for email sending", + "properties": { + "host": { + "groups": ["auth"], + "type": "string", + "description": "SMTP server hostname" + }, + "port": { + "groups": ["auth"], + "type": "integer", + "description": "SMTP server port" + }, + "user": { + "groups": ["auth"], + "type": "string", + "description": "SMTP username for authentication" + }, + "replyTo": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "Reply-to email address for sent emails" + }, + "from": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "From email address for sent emails" + }, + "envelopeFrom": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "Envelope from address for sent emails" + } + } + }, + "verifyEmail": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, requires email verification for new accounts" + }, + "resetPasswordAllowed": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, allows users to reset their passwords" + }, + "registrationAllowed": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, allows users to register new accounts in Keycloak." } - }, - "verifyEmail": { - "type": "boolean", - "default": true - }, - "resetPasswordAllowed": { - "type": "boolean", - "default": true - }, - "registrationAllowed": { - "type": "boolean", - "default": true } }, + "insecureCookies": { + "groups": ["auth"], + "type": "boolean", + "default": false, + "description": "If true, allows insecure cookies." + }, + "registrationMessage": { + "groups": ["auth"], + "type": "string", + "default": "You must agree to the terms of use.", + "description": "Message displayed during user registration, typically including terms of service." + }, "organisms": { "type": "object", "patternProperties": { From 53b835280ee489728cb9efd98f84704b83030e8b Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:06:35 +0100 Subject: [PATCH 13/56] allow lists of types --- docs/src/components/SchemaDocs.astro | 12 ++++++++++-- kubernetes/loculus/values.schema.json | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 12d07a2f7..4a1a76480 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,6 +1,6 @@ --- import { readFile } from 'node:fs/promises'; -import { type JSONSchema7, type JSONSchema7Definition } from 'json-schema'; +import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; const { schemaPath, group } = Astro.props; let schema: JSONSchema7 = {}; @@ -25,6 +25,14 @@ function capitalizeFirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } +function typeToString(type: JSONSchema7TypeName | JSONSchema7TypeName[] | undefined) { + if (type === undefined) return ""; + if (Array.isArray(type)) { + return type.map(t => capitalizeFirst(String(t))).join(", ") + } + return capitalizeFirst(String(type)); +} + function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7Definition) { if ( typeof definition === 'object' && @@ -41,7 +49,7 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 } rows.push({ key: `${prefix}${key}`, - type: capitalizeFirst(String(definition.type)), + type: typeToString(definition.type), default: def, description: definition.description }) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 321361eb8..aa2dbff09 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -59,7 +59,7 @@ }, "welcomeMessageHTML": { "groups": ["general"], - "type": "string", + "type": ["string", "null"], "description": "A custom welcome message to be shown on the landing page" }, "additionalHeadHTML": { From 1f7b4b9eac076b051885976c056670d12f9bc304 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:17:32 +0100 Subject: [PATCH 14/56] ... --- docs/src/components/SchemaDocs.astro | 6 +-- .../docs/reference/helm-chart-config.mdx | 37 ++----------------- kubernetes/loculus/values.schema.json | 18 +++++++++ 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 4a1a76480..0dd315995 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; -const { schemaPath, group } = Astro.props; +const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; try { @@ -77,9 +77,9 @@ if (schema.properties) { - + - + diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index e3aa7b2d9..947c75942 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -13,7 +13,7 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## General Settings - +
FieldField TypeDefaultDefault Description
@@ -50,44 +50,15 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## Website Settings - + ## User registration and authentication - + ## Database deployments -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`runDevelopmentMainDatabase`BooleantrueIf true, runs a development database within the cluster.
`runDevelopmentKeycloakDatabase`BooleantrueIf true, runs a development Keycloak database within the cluster.
`developmentDatabasePersistence`BooleantrueIf true, makes the database on the argocd preview persistent.
+ For production environments, these should always be set to false. Instead, external managed databases should be used. diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index aa2dbff09..213063140 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -268,6 +268,24 @@ "default": "You must agree to the terms of use.", "description": "Message displayed during user registration, typically including terms of service." }, + "runDevelopmentMainDatabase": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, runs a development database within the cluster." + }, + "runDevelopmentKeycloakDatabase": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, runs a development Keycloak database within the cluster." + }, + "developmentDatabasePersistence": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, makes the database on the argocd preview persistent." + }, "organisms": { "type": "object", "patternProperties": { From 576094fc5c6d5b9d6f3519175ebdba663d951233 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:39:34 +0100 Subject: [PATCH 15/56] services --- .../docs/reference/helm-chart-config.mdx | 49 +------------------ kubernetes/loculus/values.schema.json | 35 +++++++++++++ 2 files changed, 36 insertions(+), 48 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 947c75942..55b12f278 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -64,54 +64,7 @@ For production environments, these should always be set to false. Instead, exter ## Services - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`disableWebsite`BooleanfalseIf true, disables the frontend website deployment.
`disableBackend`BooleanfalseIf true, disables the backend API deployment.
`disablePreprocessing`BooleanfalseIf true, disables preprocessing pipelines.
`disableIngest`BooleanfalseIf true, disables ingestion services.
`disableEnaSubmission`BooleanfalseIf true, disables ENA (European Nucleotide Archive) submission service.
`customWebsiteImage`StringOptional custom Docker image for the website.
+ ## Organism Configuration diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 213063140..c559cd082 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -286,6 +286,41 @@ "default": true, "description": "If true, makes the database on the argocd preview persistent." }, + "disableWebsite": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables the frontend website deployment." + }, + "disableBackend": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables the backend API deployment." + }, + "disablePreprocessing": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables preprocessing pipelines." + }, + "disableIngest": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables ingestion services." + }, + "disableEnaSubmission": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables ENA (European Nucleotide Archive) submission service." + }, + "customWebsiteImage": { + "groups": ["services"], + "type": "string", + "description": "Optional custom Docker image for the website." + }, "organisms": { "type": "object", "patternProperties": { From c5e3d45aa9cc6a49ce7cfc36277c0b1de7a3f082 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:53:14 +0100 Subject: [PATCH 16/56] lineage system defs --- docs/src/components/SchemaDocs.astro | 6 +++++ .../docs/reference/helm-chart-config.mdx | 27 +++---------------- kubernetes/loculus/values.schema.json | 12 ++++++++- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 0dd315995..b020db42c 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,6 +1,7 @@ --- import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; +import { defineAction } from 'astro:actions'; const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; @@ -38,6 +39,9 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 typeof definition === 'object' && definition !== null ) { + if ('placeholder' in definition) { + key = `<${definition.placeholder}>`; + } if ( 'groups' in definition && Array.isArray(definition.groups) && @@ -71,6 +75,8 @@ if (schema.properties) { // TODO 'desciription' should be markdown processed so `` works as intended +// TODO - can we get links to other sections in the descriptions? Would be cool to link to other types. + ---
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 55b12f278..31df3439b 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -68,6 +68,8 @@ For production environments, these should always be set to false. Instead, exter ## Organism Configuration + + @@ -108,30 +110,7 @@ lineageSystemDefinitions: 1: ... ``` -
- - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`lineageSystemDefinitions.`ObjectA map from pipeline versions to file URLs.
`lineageSystemDefinitions..``String`The URL to the lineage defintion file for that lineage system and that pipeline version.
+ ### Organism (type) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index c559cd082..25a08f63e 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -322,7 +322,9 @@ "description": "Optional custom Docker image for the website." }, "organisms": { + "groups": ["organism-conf"], "type": "object", + "description": "An object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)", "patternProperties": { "^.*$": { "type": "object", @@ -445,14 +447,22 @@ } }, "lineageSystemDefinitions": { + "groups": ["organism-conf"], "type": "object", + "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-type))", "patternProperties": { "^.*$": { + "groups": ["lineage-system-definitions"], + "placeholder": "name", "type": "object", + "description": "A map from pipeline versions to file URLs.", "patternProperties": { "^[0-9]+$": { + "groups": ["lineage-system-definitions"], + "placeholder": "pipelineVersion", "type": "string", - "format": "uri" + "format": "uri", + "description": "The URL to the lineage defintion file for that lineage system and that pipeline version." } } } From 62378cc3ef0c1c720ff548f14d83e17767aa877c Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 16:56:56 +0100 Subject: [PATCH 17/56] Add some docs --- kubernetes/loculus/values.schema.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 25a08f63e..6ce4caacf 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1,4 +1,6 @@ { + "title": "values.yaml Schema for the Loculus Helm Chart", + "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation.", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { From 4009c7a6e8a371380bea56c0531a58816b90c740 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 17:07:58 +0100 Subject: [PATCH 18/56] no prefix --- docs/src/components/SchemaDocs.astro | 6 ++++++ .../content/docs/reference/helm-chart-config.mdx | 2 ++ kubernetes/loculus/values.schema.json | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index b020db42c..ee9908adf 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; import { defineAction } from 'astro:actions'; +import { defineCollection } from 'astro:content'; const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; @@ -42,6 +43,9 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 if ('placeholder' in definition) { key = `<${definition.placeholder}>`; } + if ('docsIncludePrefix' in definition && definition.docsIncludePrefix === false) { + prefix = ""; + } if ( 'groups' in definition && Array.isArray(definition.groups) && @@ -77,6 +81,8 @@ if (schema.properties) { // TODO - can we get links to other sections in the descriptions? Would be cool to link to other types. +// TODO improve array handling (i.e. referenceGenomes.nucleotideSequences) + ---
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 31df3439b..5b11aeaed 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -116,6 +116,8 @@ lineageSystemDefinitions: Each organism object has the following fields: + + diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 6ce4caacf..88be8f787 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -332,6 +332,8 @@ "type": "object", "properties": { "schema": { + "groups": ["organism"], + "docsIncludePrefix": false, "type": "object", "properties": { "organismName": { @@ -381,6 +383,8 @@ "required": ["organismName"] }, "preprocessing": { + "groups": ["organism"], + "docsIncludePrefix": false, "type": "array", "items": { "type": "object", @@ -405,11 +409,21 @@ "required": ["version", "image"] } }, + "ingest": { + "groups": ["organism"], + "docsIncludePrefix": false, + "type": "object", + "description": "TODO" + }, "referenceGenomes": { + "groups": ["organism"], + "docsIncludePrefix": false, "type": "object", "properties": { "nucleotideSequences": { + "groups": ["organism"], "type": "array", + "description": "Array of Nucleotide sequence (type)", "items": { "type": "object", "properties": { @@ -427,7 +441,9 @@ } }, "genes": { + "groups": ["organism"], "type": "array", + "description": "Array of Gene (type)", "items": { "type": "object", "properties": { From 83c109b9fb17d4797f61072d530ee5571dcec015 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 17:22:54 +0100 Subject: [PATCH 19/56] progress on the schema type --- .../docs/reference/helm-chart-config.mdx | 101 +----------------- kubernetes/loculus/values.schema.json | 44 +++++++- 2 files changed, 43 insertions(+), 102 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 5b11aeaed..f56785d41 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -118,57 +118,10 @@ Each organism object has the following fields: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`schema`[Schema (type)](#schema-type)
`preprocessing`A list of [Preprocessing (type)](#preprocessing-type)Configuration for the preprocessing pipeline(s)
`ingest`[Ingest (type)](#ingest-type)Configuration for the ingest pipeline(s)
`referenceGenomes`ObjectReference genome sequences for the organism
`referenceGenomes.nucleotideSequences`Array of [NucleotideSequence (type)](#nucleotidesequence-type)
`referenceGenomes.genes`Array of [Gene (type)](#gene-type)
- ### Schema (type) + + @@ -179,56 +132,6 @@ Each organism object has the following fields: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 88be8f787..935bed6a8 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -337,17 +337,46 @@ "type": "object", "properties": { "organismName": { - "type": "string" + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string", + "description": "Display name for the organism" }, "image": { - "type": "string" + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string", + "description": "URL to an image that will be shown on the landing page" }, "loadSequencesAutomatically": { + "groups": ["schema"], + "docsIncludePrefix": false, "type": "boolean", - "default": true + "default": true, + "description": "Whether sequences be loaded automatically on the sequence details page , rather than users having to press a button to do so. (For small genomes, this should probably be true.)" + }, + "submissionDataTypes": { + "docsIncludePrefix": false, + "type": "object", + "properties": { + "consensusSequences": { + "groups": ["schema"], + "type": "boolean", + "default": true, + "description": "If false, the submission form will not allow submission of consensus sequences (i.e. the sequences file must be omitted). All consensus sequence related parts on the website will be hidden." + } + } + }, + "description": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string" }, "metadata": { + "groups": ["schema"], + "docsIncludePrefix": false, "type": "array", + "description": "Array of [Metadata (type)](#metadata-type). Metadata fields associated with the organism.", "items": { "type": "object", "properties": { @@ -378,6 +407,15 @@ }, "required": ["name", "displayName", "type"] } + }, + "metadataTemplate": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "array", + "description": "Array of strings. Which input fields to add to the downloadable metadata template on the submission and revision page.", + "items": { + "type": "string" + } } }, "required": ["organismName"] From c6efac816ab1b9edadf9613370c308303568b72e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 17:39:16 +0100 Subject: [PATCH 20/56] Add more schema stuff --- docs/src/components/SchemaDocs.astro | 4 +- .../docs/reference/helm-chart-config.mdx | 113 ----------- kubernetes/loculus/values.schema.json | 182 +++++++++++++++++- 3 files changed, 176 insertions(+), 123 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index ee9908adf..3e2ca1036 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,8 +1,6 @@ --- import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; -import { defineAction } from 'astro:actions'; -import { defineCollection } from 'astro:content'; const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; @@ -83,6 +81,8 @@ if (schema.properties) { // TODO improve array handling (i.e. referenceGenomes.nucleotideSequences) +// TODO better handling for enums (i.e. string: ascending/descending) + ---
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index f56785d41..df65b3551 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -70,33 +70,6 @@ For production environments, these should always be set to false. Instead, exter -
`organismName`StringDisplay name for the organism
`image`StringURL to an image that will be shown on the landing page
`loadSequencesAutomatically`Booleantrue - Whether sequences be loaded automatically on the sequence details page , rather than users having to - press a button to do so. (For small genomes, this should probably be true.) -
`submissionDataTypes.consensusSequences`Booleantrue - If `false`, the submission form will not allow submission of consensus sequences (i.e. the sequences - file must be omitted). All consensus sequence related parts on the website will be hidden. -
`description`String
`metadata`Array of [Metadata (type)](#metadata-type)Metadata fields associated with the organism.
`metadataTemplate`Array of String - Which input fields to add to the downloadable metadata template on the submission and revision page. -
`earliestReleaseDate` Object
- - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`organisms`ObjectAn object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)
`lineageSystemDefinitions`ObjectAn object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system defintions](#lineage-type))
- ### Lineage system definitions Here's an example of a `lineageDefinitions` section: @@ -122,92 +95,6 @@ Each organism object has the following fields: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`earliestReleaseDate`Object - Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. For each - version of an accession, the `earliestReleaseDate` is calculated as the earliest date of the internal - release date, the dates in the configured `externalFields` and the value from the previous version of - the accession (if there is one). This can be used when having a mix of sequences imported from other - databases, as well as sequences released first in this Loculus instance, to have a field that shows the - earliest release date regardless of where the sequence was first released. -
`earliestReleaseDate.enabled`booleanWhether to enable the `earliestReleaseDate` metadata field.
`earliestReleaseDate.externalFields`Array of strings - Field names to use when calculating the earliest release date. The fields need to be nullable strings - formated with `yyyy-mm-dd`. -
`website`ObjectConfiguration for how the organism data is displayed on the website
`website.tableColumns`Array of StringsColumns to display in the browse table
`website.defaultOrderBy`StringDefault column to sort the browse table
`website.defaultOrder`"ascending" | "descending"Default order direction
`silo`ObjectConfiguration regarding the SILO database engine
`silo.dateToSortBy`String - Name of a field of type date. This might speed up date range searches on this field and improve sequence - compression. -
`silo.partitionBy`String
- ### Metadata (type) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 935bed6a8..ab0b9ef92 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -381,28 +381,129 @@ "type": "object", "properties": { "name": { - "type": "string" + "type": "string", + "description": "Key used across app to refer to this field." }, "displayName": { - "type": "string" + "type": "string", + "description": "Name displayed to users." }, "type": { - "type": "string" + "type": "string" + }, + "header": { + "type": "string", + "description": "Grouping of fields in sequence details UI." }, "required": { - "type": "boolean" + "type": "boolean", + "description": "Whether the field is required by backend." + }, + "desired": { + "type": "boolean", + "description": "Whether the field is a desired input field for submitters." }, "definition": { - "type": "string" + "type": "string", + "description": "Definition of input field for submitters." }, "guidance": { - "type": "string" + "type": "string", + "description": "Guidance for submitters on filling in input field." + }, + "noInput": { + "type": "boolean", + "description": "If true, the field will not be included as input." }, "generateIndex": { - "type": "boolean" + "type": "boolean", + "description": "Whether the field should be indexed for search (only allowed for string fields)." + }, + "columnWidth": { + "type": "number", + "description": "The minimum column width for this field on the search table." + }, + "order": { + "type": "number", + "description": "Order for the column in the search table, lower first." }, "autocomplete": { - "type": "boolean" + "type": "boolean", + "description": "Whether autocomplete should be offered for the field (only for string fields)." + }, + "enableSubstringSearch": { + "type": "boolean", + "description": "If true, search results will match substrings instead of exact values." + }, + "rangeSearch": { + "type": "boolean", + "description": "If true, enables range search for numeric fields." + }, + "rangeOverlapSearch": { + "type": "object", + "description": "Config settings for enabling range overlap search.", + "properties": { + "rangeId": { + "type": "string", + "description": "The range ID that this field belongs to." + }, + "rangeDisplayName": { + "type": "string", + "description": "The display name of the range." + }, + "bound": { + "type": "string", + "enum": ["lower", "upper"], + "description": "Whether this field is the lower or upper bound of the range." + } + } + }, + "initiallyVisible": { + "type": "boolean", + "description": "If true, the field is initially visible in the UI." + }, + "hideOnSequenceDetailsPage": { + "type": "boolean", + "description": "If true, hides the field on the sequence details page." + }, + "hideInSearchResultsTable": { + "type": "boolean", + "description": "If true, hides the field in the search results table." + }, + "perSegment": { + "type": "boolean", + "description": "Whether this metadata field should exist for each segment." + }, + "customDisplay": { + "type": "object", + "description": "Custom display settings for the field on the sequence details page.", + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "ontology_id": { + "type": "string" + }, + "example": { + "type": "string", + "description": "Example value for the field." + }, + "ingest": { + "type": "string", + "description": "Which NCBI field to map to this field." + }, + "preprocessing": { + "type": "object", + "description": "Preprocessing pipeline configuration settings." + }, + "lineageSystem": { + "type": "string", + "description": "Lineage system for searches including sublineages." } }, "required": ["name", "displayName", "type"] @@ -416,6 +517,71 @@ "items": { "type": "string" } + }, + "earliestReleaseDate": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. For each version of an accession, the `earliestReleaseDate` is calculated as the earliest date of the internal release date, the dates in the configured `externalFields` and the value from the previous version of the accession (if there is one). This can be used when having a mix of sequences imported from other databases, as well as sequences released first in this Loculus instance, to have a field that shows the earliest release date regardless of where the sequence was first released.", + "properties": { + "enabled": { + "groups": ["schema"], + "type": "boolean", + "description": "Whether to enable the `earliestReleaseDate` metadata field." + }, + "externalFields": { + "groups": ["schema"], + "type": "array", + "description": "Field names to use when calculating the earliest release date. The fields need to be nullable strings formated with `yyyy-mm-dd`.", + "items": { + "type": "string" + } + } + } + }, + "website": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration for how the organism data is displayed on the website", + "properties": { + "tableColumns": { + "groups": ["schema"], + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns to display in the browse table" + }, + "defaultOrderBy": { + "groups": ["schema"], + "type": "string", + "description": "Default column to sort the browse table" + }, + "defaultOrder": { + "groups": ["schema"], + "type": "string", + "enum": ["ascending", "descending"], + "description": "Default order direction." + } + } + }, + "silo": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration regarding the SILO database engine", + "properties": { + "dateToSortBy": { + "groups": ["schema"], + "type": "string", + "description": "Name of a field of type date. This might speed up date range searches on this field and improve sequence compression." + }, + "partitionBy": { + "groups": ["schema"], + "type": "string" + } + } } }, "required": ["organismName"] From af2a2a154c9441aa9e21a24869fbeccb5346cbf0 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 5 Feb 2025 17:41:56 +0100 Subject: [PATCH 21/56] added more --- kubernetes/loculus/values.schema.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index ab0b9ef92..22e0d9f51 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -617,7 +617,18 @@ "groups": ["organism"], "docsIncludePrefix": false, "type": "object", - "description": "TODO" + "description": "TODO", + "properties": { + "image": { + "type": "string", + "default": "ghcr.io/loculus-project/ingest", + "description": "Docker image for the ingest pipeline" + }, + "configFile": { + "type": "object", + "description": "Fields that should be added to the ingest pipeline config file" + } + } }, "referenceGenomes": { "groups": ["organism"], From 86e0f60cb12c7e99a52855c2951635a4a85f1a70 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 09:53:31 +0100 Subject: [PATCH 22/56] metadata docs --- docs/src/components/SchemaDocs.astro | 7 + .../docs/reference/helm-chart-config.mdx | 238 +----------------- kubernetes/loculus/values.schema.json | 75 +++++- 3 files changed, 74 insertions(+), 246 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 3e2ca1036..d8ff05424 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -66,6 +66,13 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 if ('patternProperties' in definition && definition.patternProperties) { Object.entries(definition.patternProperties).forEach(([k, d]) => addSelfAndChildren(`${prefix}${key}.`, k, d)); } + if ('items' in definition && definition.items !== undefined && typeof definition.items === 'object') { + const items = definition.items; + if ('length' in items) { // filter out arrays + return; + } + addSelfAndChildren(`${prefix}${key}.`, "[]", items) + } } } diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index df65b3551..18b64304a 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -97,243 +97,7 @@ Each organism object has the following fields: ### Metadata (type) -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`name`StringKey used across app to refer to this field
`displayName`StringName displayed to users
`type`
`header`StringGrouping of fields in sequence details UI
`required`BooleanWhether the field is required by backend
`desired`BooleanWhether the field is a desired input field for submitters
`definition`BooleanDefinition of input field for submitters
`guidance`BooleanGuidance for submitters on filling in input field
`noInput`Boolean - Whether a field with this name is expected as possible input, and so should be included in the metadata - template and as a form field. (If set to true it will not be included). -
`generateIndex`Boolean - Whether the field should be indexed for search. This is only allowed for string fields and facilitates - faster filters. It is recommended if the number of different values is rather small. -
`columnWidth`NumberThe minimum column width for this field on the search table.
`order`NumberOrder for the column in the search table, lower first
`autocomplete`Boolean - Whether autocomplete should be offered for the field. This is only allowed for string fields and - probably `generateIndex` should be true. -
`enableSubstringSearch`Boolean - If true, search results will contain results that contain the given value as a substring. If false (the - default), only exact matches will be returned. This only works for string fields, and you cannot also - enable `autocomplete`. -
`rangeSearch`BooleanIf true, enables range search for numeric fields.
`rangeOverlapSearch`ObjectThe config settings for enabling range overlap search.
`rangeOverlapSearch.rangeId`String - The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the - same range ID. -
`rangeOverlapSearch.rangeDisplayName`StringThe display name of the range.
`rangeOverlapSearch.bound`"lower" | "upper"Whether this field is the lower or upper bound of the range.
`initiallyVisible`BooleanIf true, the field is initially visible in the UI.
`hideOnSequenceDetailsPage`BooleanIf true, hides the field on the sequence details page.
`hideInSearchResultsTable`BooleanIf true, hides the field in the search results table (and makes it impossible to show).
`perSegment`Boolean - Whether this is a metadata field that should exist for each segment. If so fields will be created as - `fieldName_A`, `fieldName_B` in the case of an organism with segments `A` and `B`. -
`customDisplay`ObjectCustom display of the field on the sequence details page
`customDisplay.type`
`customDisplay.url`
`ontology_id`String
`definition`String
`guidance`String
`example`StringExample value for the field.
`ingest`StringWhich NCBI field to map to this field
`preprocessing`Object - The values of this field will be added to the preprocessing pipeline config file and the available - values depend on the chosen pipeline. For the Nextclade pipeline, please see - [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions). -
`lineageSystem`String - Use this on string fields that contain lineages, if you want to enable searches that can include - sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` - key. -
+ ### Preprocessing (type) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 22e0d9f51..ce234cfd0 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -381,77 +381,112 @@ "type": "object", "properties": { "name": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Key used across app to refer to this field." }, "displayName": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Name displayed to users." }, "type": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string" }, "header": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Grouping of fields in sequence details UI." }, "required": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", "description": "Whether the field is required by backend." }, "desired": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", "description": "Whether the field is a desired input field for submitters." }, "definition": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Definition of input field for submitters." }, "guidance": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Guidance for submitters on filling in input field." }, "noInput": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "If true, the field will not be included as input." + "description": "Whether a field with this name is expected as possible input, and so should be included in the metadata template and as a form field. (If set to true it will not be included)." }, "generateIndex": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "Whether the field should be indexed for search (only allowed for string fields)." + "description": "Whether the field should be indexed for search. This is only allowed for string fields and facilitates faster filters. It is recommended if the number of different values is rather small." }, "columnWidth": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "number", "description": "The minimum column width for this field on the search table." }, "order": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "number", "description": "Order for the column in the search table, lower first." }, "autocomplete": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "Whether autocomplete should be offered for the field (only for string fields)." + "description": "Whether autocomplete should be offered for the field. This is only allowed for string fields and probably `generateIndex` should be true." }, "enableSubstringSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "If true, search results will match substrings instead of exact values." + "description": "If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. This only works for string fields, and you cannot also enable `autocomplete`." }, "rangeSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", "description": "If true, enables range search for numeric fields." }, "rangeOverlapSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "object", "description": "Config settings for enabling range overlap search.", "properties": { "rangeId": { + "groups": ["metadata"], "type": "string", - "description": "The range ID that this field belongs to." + "description": "The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID." }, "rangeDisplayName": { + "groups": ["metadata"], "type": "string", "description": "The display name of the range." }, "bound": { + "groups": ["metadata"], "type": "string", "enum": ["lower", "upper"], "description": "Whether this field is the lower or upper bound of the range." @@ -459,51 +494,73 @@ } }, "initiallyVisible": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", "description": "If true, the field is initially visible in the UI." }, "hideOnSequenceDetailsPage": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", "description": "If true, hides the field on the sequence details page." }, "hideInSearchResultsTable": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "If true, hides the field in the search results table." + "description": "If true, hides the field in the search results table (and makes it impossible to show)." }, "perSegment": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "description": "Whether this metadata field should exist for each segment." + "description": "Whether this is a metadata field that should exist for each segment. If so fields will be created as fieldName_A, fieldName_B in the case of an organism with segments A and B." }, "customDisplay": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "object", "description": "Custom display settings for the field on the sequence details page.", "properties": { "type": { + "groups": ["metadata"], "type": "string" }, "url": { + "groups": ["metadata"], "type": "string" } } }, "ontology_id": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string" }, "example": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Example value for the field." }, "ingest": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", "description": "Which NCBI field to map to this field." }, "preprocessing": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "object", - "description": "Preprocessing pipeline configuration settings." + "description": "The values of this field will be added to the preprocessing pipeline config file and the available values depend on the chosen pipeline. For the Nextclade pipeline, please see [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions)." }, "lineageSystem": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "string", - "description": "Lineage system for searches including sublineages." + "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." } }, "required": ["name", "displayName", "type"] From efc6c0ebf9d0992b5e02bec7e17a6db1385c1ab5 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 10:55:56 +0100 Subject: [PATCH 23/56] Progress --- .../docs/reference/helm-chart-config.mdx | 60 ++----------------- kubernetes/loculus/values.schema.json | 19 +++++- 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 18b64304a..e2d80b899 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -101,6 +101,8 @@ Each organism object has the following fields: ### Preprocessing (type) + + @@ -142,6 +144,8 @@ The values for `args` and `configFile` depend on the used preprocessing pipeline ### Ingest (type) + +
@@ -171,60 +175,8 @@ The values for `configFile` depend on the used preprocessing pipeline. For [our ### NucleotideSequence (type) -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`name`StringName of the sequence
`sequence`
`insdcAccessionFull`StringINSDC accession of the sequence
+ ### Gene (type) - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`name`StringName of the sequence
`sequence`
+ diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index ce234cfd0..d78cb2961 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -700,13 +700,21 @@ "type": "object", "properties": { "name": { - "type": "string" + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name of the sequence" }, "sequence": { + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, "type": "string" }, "insdcAccessionFull": { - "type": "string" + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, + "type": "string", + "description": "INSDC accession of the sequence" } }, "required": ["name", "sequence"] @@ -720,9 +728,14 @@ "type": "object", "properties": { "name": { - "type": "string" + "groups": ["gene"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name of the sequence." }, "sequence": { + "groups": ["gene"], + "docsIncludePrefix": false, "type": "string" } }, From f03f5eda2f2f1e7c60395211e92979d4ccee9c3d Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 11:02:41 +0100 Subject: [PATCH 24/56] progress --- .../docs/reference/helm-chart-config.mdx | 27 +------------------ kubernetes/loculus/values.schema.json | 4 +++ 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index e2d80b899..f6d7e47d2 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -144,32 +144,7 @@ The values for `args` and `configFile` depend on the used preprocessing pipeline ### Ingest (type) - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`image`"ghcr.io/loculus-project/ingest"Docker image for the ingest pipeline
`configFile`ObjectFields that should be added to the ingest pipeline config file
+ The values for `configFile` depend on the used preprocessing pipeline. For [our ingest pipeline](https://github.com/loculus-project/loculus/tree/main/ingest) which downloads data from NCBI GenBank using NCBI Datasets, the config file needs to contain the taxon_id of the organism and additionally, if the organism is multi-segmented, it requires a list of segment names (nucleotide_sequences) and the nextclade_dataset that can be used for segment identification and alignment. diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index d78cb2961..08c46c2ae 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -677,11 +677,15 @@ "description": "TODO", "properties": { "image": { + "groups": ["ingest"], + "docsIncludePrefix": false, "type": "string", "default": "ghcr.io/loculus-project/ingest", "description": "Docker image for the ingest pipeline" }, "configFile": { + "groups": ["ingest"], + "docsIncludePrefix": false, "type": "object", "description": "Fields that should be added to the ingest pipeline config file" } From 6c9498c689e06ec12fabcdf85a3892c2f85fa177 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 11:05:49 +0100 Subject: [PATCH 25/56] migrated all tables to the config --- .../docs/reference/helm-chart-config.mdx | 39 +------------------ kubernetes/loculus/values.schema.json | 20 ++++++++-- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index f6d7e47d2..4266845d3 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -101,44 +101,7 @@ Each organism object has the following fields: ### Preprocessing (type) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`version`IntegerVersion of the preprocessing pipeline
`image`StringDocker image for the preprocessing pipeline
`args`Array of stringsArguments passed to the preprocessing pipeline
`configFile`ObjectFields that should be added to the preprocessing pipeline config file
+ The values for `args` and `configFile` depend on the used preprocessing pipeline. For the Nextclade preprocessing pipeline, please see [here](../../for-administrators/existing-preprocessing-pipelines/#nextclade-based-pipeline). diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 08c46c2ae..c2143065c 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -651,20 +651,32 @@ "type": "object", "properties": { "version": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, "type": "integer", - "minimum": 1 + "minimum": 1, + "description": "Version of the preprocessing pipeline" }, "image": { - "type": "string" + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "string", + "description": "Docker image for the preprocessing pipeline" }, "args": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, "type": "array", "items": { "type": "string" - } + }, + "description": "Array of Strings. Arguments passed to the preprocessing pipeline." }, "configFile": { - "type": "object" + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "object", + "description": "Fields that should be added to the preprocessing pipeline config file." } }, "required": ["version", "image"] From 54b4cded105a253323a4b7b77fba3b3b33a540b4 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 11:11:19 +0100 Subject: [PATCH 26/56] improve enum handling --- docs/src/components/SchemaDocs.astro | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index d8ff05424..1ff52ec67 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,6 +1,6 @@ --- import { readFile } from 'node:fs/promises'; -import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName } from 'json-schema'; +import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName, type JSONSchema7Type } from 'json-schema'; const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; @@ -25,11 +25,17 @@ function capitalizeFirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } -function typeToString(type: JSONSchema7TypeName | JSONSchema7TypeName[] | undefined) { +function typeToString( + type: JSONSchema7TypeName | JSONSchema7TypeName[] | undefined, + enumvals: JSONSchema7Type[] | undefined +) { if (type === undefined) return ""; if (Array.isArray(type)) { return type.map(t => capitalizeFirst(String(t))).join(", ") } + if (type === "string" && enumvals !== undefined) { + return enumvals?.map(enumval => String(enumval)).join(", ") + } return capitalizeFirst(String(type)); } @@ -55,7 +61,7 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 } rows.push({ key: `${prefix}${key}`, - type: typeToString(definition.type), + type: typeToString(definition.type, definition.enum), default: def, description: definition.description }) @@ -86,10 +92,6 @@ if (schema.properties) { // TODO - can we get links to other sections in the descriptions? Would be cool to link to other types. -// TODO improve array handling (i.e. referenceGenomes.nucleotideSequences) - -// TODO better handling for enums (i.e. string: ascending/descending) - ---
From 22b53e40c61fd2d4ddb0eeee9f9454f73d87cd81 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 11:37:21 +0100 Subject: [PATCH 27/56] Add markdown rendering support --- docs/src/components/MarkdownRenderer.astro | 15 +++++++++++++++ docs/src/components/SchemaDocs.astro | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 docs/src/components/MarkdownRenderer.astro diff --git a/docs/src/components/MarkdownRenderer.astro b/docs/src/components/MarkdownRenderer.astro new file mode 100644 index 000000000..b2da3c155 --- /dev/null +++ b/docs/src/components/MarkdownRenderer.astro @@ -0,0 +1,15 @@ +--- +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import remarkRehype from 'remark-rehype'; +import rehypeStringify from 'rehype-stringify'; + +const { content } = Astro.props; + +const html = String(await unified() + .use(remarkParse) + .use(remarkRehype) + .use(rehypeStringify) + .process(content)); +--- +
diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 1ff52ec67..4accbd267 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,6 +1,7 @@ --- import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName, type JSONSchema7Type } from 'json-schema'; +import MarkdownRenderer from './MarkdownRenderer.astro'; const { schemaPath, group, fieldColumnClass } = Astro.props; let schema: JSONSchema7 = {}; @@ -111,7 +112,7 @@ if (schema.properties) { {row.key} {row.type} {row.default} - {row.description} + )) } From e18b8b5ed4bfdfd936bb566e0b83eac1e3d1b7c8 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 11:45:35 +0100 Subject: [PATCH 28/56] Fix some links --- kubernetes/loculus/values.schema.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index c2143065c..0c0a42fa4 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -335,6 +335,7 @@ "groups": ["organism"], "docsIncludePrefix": false, "type": "object", + "description": "Object of type [Schema](#schema-type)", "properties": { "organismName": { "groups": ["schema"], @@ -647,6 +648,7 @@ "groups": ["organism"], "docsIncludePrefix": false, "type": "array", + "description": "Array of [Preprocessing (type)](#preprocessing-type).", "items": { "type": "object", "properties": { @@ -686,7 +688,7 @@ "groups": ["organism"], "docsIncludePrefix": false, "type": "object", - "description": "TODO", + "description": "Object of type [Ingest](#ingest-type)", "properties": { "image": { "groups": ["ingest"], @@ -704,14 +706,13 @@ } }, "referenceGenomes": { - "groups": ["organism"], "docsIncludePrefix": false, "type": "object", "properties": { "nucleotideSequences": { "groups": ["organism"], "type": "array", - "description": "Array of Nucleotide sequence (type)", + "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", "items": { "type": "object", "properties": { @@ -739,7 +740,7 @@ "genes": { "groups": ["organism"], "type": "array", - "description": "Array of Gene (type)", + "description": "Array of [Gene (type)](#gene-type)", "items": { "type": "object", "properties": { @@ -768,7 +769,7 @@ "lineageSystemDefinitions": { "groups": ["organism-conf"], "type": "object", - "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-type))", + "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-system-definitions))", "patternProperties": { "^.*$": { "groups": ["lineage-system-definitions"], From 3825c31747a06bfef95d3a26435e698f071554d2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 12:01:44 +0100 Subject: [PATCH 29/56] Add docs --- docs/src/components/MarkdownRenderer.astro | 3 ++ docs/src/components/SchemaDocs.astro | 42 +++++++++++----- .../docs/reference/helm-chart-config.mdx | 48 ++++++++++++------- kubernetes/loculus/values.schema.json | 2 +- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/docs/src/components/MarkdownRenderer.astro b/docs/src/components/MarkdownRenderer.astro index b2da3c155..b408a4451 100644 --- a/docs/src/components/MarkdownRenderer.astro +++ b/docs/src/components/MarkdownRenderer.astro @@ -1,4 +1,7 @@ --- +/** + * Pass in a Markdown formatted string as 'content'. + */ import { unified } from 'unified'; import remarkParse from 'remark-parse'; import remarkRehype from 'remark-rehype'; diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 4accbd267..c16c2c5b1 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,4 +1,10 @@ --- +/** + * Render parts of a JSON schema into a table. + * specify the schema file in `schemaPath`, and give a `group` to render. + * Properties in the schema can be assigned to groups by adding a new "groups": ["group1", "group2"] key to them. + * This way, you can split schema definition into semantic groups. + */ import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName, type JSONSchema7Type } from 'json-schema'; import MarkdownRenderer from './MarkdownRenderer.astro'; @@ -13,19 +19,16 @@ try { console.error('Error reading schema:', error); } -const rows: Row[] = []; - -interface Row { - key: string, - type?: string, - default?: string, - description?: string -} - +/** Example: 'boolean' -> 'Boolean'. */ function capitalizeFirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } +/** + * A useful representation of a schema type. + * Plain types are uppercased. Lists of types are joined. + * If type is 'string' but there are also enum vals, the enum vals are returned instead. + */ function typeToString( type: JSONSchema7TypeName | JSONSchema7TypeName[] | undefined, enumvals: JSONSchema7Type[] | undefined @@ -40,6 +43,22 @@ function typeToString( return capitalizeFirst(String(type)); } +/** A row in the table. */ +interface Row { + key: string, + type?: string, + default?: string, + description?: string +} + +const rows: Row[] = []; + +/** + * Recursive function to traverse the schema properties and extract rows for the table. + * @param prefix Accumulates the tree path during recursion, start of with "". + * @param key The key of the property currently observed. + * @param definition The definition of the property currently observed. + */ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7Definition) { if ( typeof definition === 'object' && @@ -83,16 +102,13 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 } } +// start of recursing with the top level properties in the schema. if (schema.properties) { Object.entries(schema.properties).forEach(([key, definition]) => { addSelfAndChildren("", key, definition); }); } -// TODO 'desciription' should be markdown processed so `` works as intended - -// TODO - can we get links to other sections in the descriptions? Would be cool to link to other types. - ---
diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 4266845d3..1088d7ce9 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -5,15 +5,19 @@ description: A reference of config fields for the Loculus Helm chart import SchemaDocs from '../../../components/SchemaDocs.astro'; +const schemaPath = '../kubernetes/loculus/values.schema.json'; + :::note -The config structure has not been finalized and may change rapidly until Loculus 1.0 is officially released (see [current state and roadmap](../../introduction/current-state-and-roadmap/)). In the meantime, this reference might also not be fully accurate or complete. +The config structure has not been finalized and may change rapidly until Loculus 1.0 is +officially released (see [current state and roadmap](../../introduction/current-state-and-roadmap/)). +In the meantime, this reference might also not be fully accurate or complete. ::: The configuration for the Helm chart is provided as a YAML file. It has the following fields: ## General Settings - +
@@ -50,25 +54,26 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## Website Settings - + ## User registration and authentication - + ## Database deployments - + -For production environments, these should always be set to false. Instead, external managed databases should be used. +For production environments, these should always be set to false. +Instead, external managed databases should be used. ## Services - + ## Organism Configuration - + ### Lineage system definitions @@ -83,38 +88,45 @@ lineageSystemDefinitions: 1: ... ``` - + ### Organism (type) Each organism object has the following fields: - + ### Schema (type) - + ### Metadata (type) - + ### Preprocessing (type) - + -The values for `args` and `configFile` depend on the used preprocessing pipeline. For the Nextclade preprocessing pipeline, please see [here](../../for-administrators/existing-preprocessing-pipelines/#nextclade-based-pipeline). +The values for `args` and `configFile` depend on the used preprocessing pipeline. +For the Nextclade preprocessing pipeline, please see +[here](../../for-administrators/existing-preprocessing-pipelines/#nextclade-based-pipeline). ### Ingest (type) - + -The values for `configFile` depend on the used preprocessing pipeline. For [our ingest pipeline](https://github.com/loculus-project/loculus/tree/main/ingest) which downloads data from NCBI GenBank using NCBI Datasets, the config file needs to contain the taxon_id of the organism and additionally, if the organism is multi-segmented, it requires a list of segment names (nucleotide_sequences) and the nextclade_dataset that can be used for segment identification and alignment. +The values for `configFile` depend on the used preprocessing pipeline. +For [our ingest pipeline](https://github.com/loculus-project/loculus/tree/main/ingest) +which downloads data from NCBI GenBank using NCBI Datasets, the config file needs to +contain the taxon_id of the organism and additionally, if the organism is multi-segmented, +it requires a list of segment names (nucleotide_sequences) and the nextclade_dataset that +can be used for segment identification and alignment. ### NucleotideSequence (type) - + ### Gene (type) - + diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 0c0a42fa4..bf3dc46fa 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1,6 +1,6 @@ { "title": "values.yaml Schema for the Loculus Helm Chart", - "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation.", + "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut of a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { From 73c5a129119706bb936e1e73417c2d118d9873cd Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 12:23:14 +0100 Subject: [PATCH 30/56] fix --- docs/src/content/docs/reference/helm-chart-config.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 1088d7ce9..59aa31504 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -5,7 +5,7 @@ description: A reference of config fields for the Loculus Helm chart import SchemaDocs from '../../../components/SchemaDocs.astro'; -const schemaPath = '../kubernetes/loculus/values.schema.json'; +export const schemaPath = '../kubernetes/loculus/values.schema.json'; :::note The config structure has not been finalized and may change rapidly until Loculus 1.0 is From a2386eb3f9774e64b2684a7d6050752eb15cabef Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 18:00:50 +0100 Subject: [PATCH 31/56] additionalProperties: false - for some objects already --- kubernetes/loculus/values.schema.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index bf3dc46fa..385878a9c 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -3,6 +3,7 @@ "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut of a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": false, "properties": { "name": { "groups": ["general"], @@ -602,6 +603,7 @@ "docsIncludePrefix": false, "type": "object", "description": "Configuration for how the organism data is displayed on the website", + "additionalProperties": false, "properties": { "tableColumns": { "groups": ["schema"], @@ -629,6 +631,7 @@ "docsIncludePrefix": false, "type": "object", "description": "Configuration regarding the SILO database engine", + "additionalProperties": false, "properties": { "dateToSortBy": { "groups": ["schema"], @@ -689,6 +692,7 @@ "docsIncludePrefix": false, "type": "object", "description": "Object of type [Ingest](#ingest-type)", + "additionalProperties": false, "properties": { "image": { "groups": ["ingest"], @@ -708,6 +712,7 @@ "referenceGenomes": { "docsIncludePrefix": false, "type": "object", + "additionalProperties": false, "properties": { "nucleotideSequences": { "groups": ["organism"], @@ -715,6 +720,7 @@ "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", "items": { "type": "object", + "additionalProperties": false, "properties": { "name": { "groups": ["nucleotide-sequence"], @@ -743,6 +749,7 @@ "description": "Array of [Gene (type)](#gene-type)", "items": { "type": "object", + "additionalProperties": false, "properties": { "name": { "groups": ["gene"], @@ -770,12 +777,14 @@ "groups": ["organism-conf"], "type": "object", "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-system-definitions))", + "additionalProperties": false, "patternProperties": { "^.*$": { "groups": ["lineage-system-definitions"], "placeholder": "name", "type": "object", "description": "A map from pipeline versions to file URLs.", + "additionalProperties": false, "patternProperties": { "^[0-9]+$": { "groups": ["lineage-system-definitions"], From 285d7cf1f06a272e23aed2fb0eedd501bd71598d Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 6 Feb 2025 18:05:59 +0100 Subject: [PATCH 32/56] Also read additional definitions --- docs/src/components/SchemaDocs.astro | 6 + kubernetes/loculus/values.schema.json | 800 +++++++++++++------------- 2 files changed, 408 insertions(+), 398 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index c16c2c5b1..8b7225aa8 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -102,6 +102,12 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 } } +if (schema.definitions) { + Object.entries(schema.definitions).forEach(([definitionName, definition]) => { + addSelfAndChildren("", definitionName, definition); + }); +} + // start of recursing with the top level properties in the schema. if (schema.properties) { Object.entries(schema.properties).forEach(([key, definition]) => { diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 385878a9c..cafe66295 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -2,329 +2,8 @@ "title": "values.yaml Schema for the Loculus Helm Chart", "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut of a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "groups": ["general"], - "type": "string", - "default": "Loculus", - "description": "The name of the Loculus instance" - }, - "logo": { - "groups": ["general"], - "type": "object", - "description": "Configuration for the logo", - "properties": { - "url": { - "groups": ["general"], - "type": "string", - "description": "URL path to the logo image file" - }, - "width": { - "groups": ["general"], - "type": "integer", - "default": 100, - "description": "Width of the logo in pixels" - }, - "height": { - "groups": ["general"], - "type": "integer", - "default": 100, - "description": "Height of the logo in pixels" - } - } - }, - "accessionPrefix": { - "groups": ["general"], - "group": "general", - "type": "string", - "default": "LOC_", - "description": "Prefix used for accession numbers" - }, - "environment": { - "groups": ["general"], - "type": "string", - "enum": ["local", "server"], - "default": "server", - "description": "Deployment environment. local for development, server for production" - }, - "host": { - "groups": ["general"], - "type": "string", - "description": "Hostname where Loculus will be accessible" - }, - "bannerMessage": { - "groups": ["general"], - "type": "string", - "default": "This is a demonstration environment. It may contain non-accurate test data and should not be used for real-world applications. Data will be deleted regularly.", - "description": "Banner message (as HTML) to display at the very top of the page" - }, - "welcomeMessageHTML": { - "groups": ["general"], - "type": ["string", "null"], - "description": "A custom welcome message to be shown on the landing page" - }, - "additionalHeadHTML": { - "groups": ["general"], - "type": "string", - "description": "Additional HTML to inject into the of pages" - }, - "createTestAccounts": { - "groups": ["general"], - "type": "boolean", - "description": "If true, creates the users testuser and superuser" - }, - "robotsNoindexHeader": { - "groups": ["general"], - "type": "boolean", - "description": "If true, adds a noindex header to prevent search engine indexing" - }, - "seqSets": { - "groups": ["general"], - "type": "object", - "properties": { - "enabled": { - "groups": ["general"], - "type": "boolean", - "description": "Enable/disable SeqSets. If false, `seqSets.crossRef` can be omitted." - }, - "crossRef": { - "groups": ["general"], - "type": ["object"], - "description": "Configuration for CrossRef integration. Set to `null` to disable CrossRef integration (you can still use SeqSets without CrossRef DOIs).", - "properties": { - "DOIPrefix": { - "groups": ["general"], - "type": "string", - "description": "The DOI prefix for SeqSets" - }, - "endpoint": { - "groups": ["general"], - "type": "string", - "description": "The API endpoint for CrossRef" - }, - "databaseName": { - "groups": ["general"], - "type": "string", - "description": "The database name for CrossRef" - }, - "email": { - "groups": ["general"], - "type": "string", - "format": "email", - "description": "The email address associated with the CrossRef account" - }, - "organization": { - "groups": ["general"], - "type": "string", - "description": "The organization name for CrossRef" - }, - "hostUrl": { - "groups": ["general"], - "type": "string", - "description": "The host URL for CrossRef callbacks" - } - } - } - } - }, - "gitHubMainUrl": { - "groups": ["general"], - "type": "string", - "default": "https://github.com/loculus-project/loculus", - "description": "The link that the GitHub icon in the footer points to" - }, - "website": { - "groups": ["website"], - "type": "object", - "description": "Website specific setting", - "properties": { - "websiteConfig": { - "groups": ["website"], - "type": "object", - "description": "Settings for the `website_config.json`", - "properties": { - "enableLoginNavigationItem": { - "groups": ["website"], - "type": "boolean", - "default": true, - "description": "Whether the website should show the login button." - }, - "enableSubmissionNavigationItem": { - "groups": ["website"], - "type": "boolean", - "default": true, - "description": "Whether the website should show \"Submit\" link in the top navigation bar." - }, - "enableSubmissionPages": { - "groups": ["website"], - "type": "boolean", - "default": true, - "description": "Whether to completely disable submission related pages. Setting this to false is useful when hosting Loculus for analysis-only purposes." - } - } - }, - "runtimeConfig": { - "type": "object", - "properties": { - "public": { - "groups": ["website"], - "type": "object", - "description": "Settings for the `public` section of the `runtime_config.json`", - "properties": { - "backendUrl": { - "groups": ["website"], - "type": "string", - "description": "Overwrite the URL where the client-side website code expects the Loculus backend" - }, - "keycloakUrl": { - "groups": ["website"], - "type": "string", - "description": "Overwrite the URL where the client-side website code expects the Keycloak" - }, - "lapisUrlTemplate": { - "groups": ["website"], - "type": "string", - "description": "Overwrite the URLs where the client-side website code expects the LAPIS instances. Must contain `%organism%` as a placeholder." - } - } - } - } - } - } - }, - "auth": { - "groups": ["auth"], - "type": "object", - "description": "User authentication (Keycloak settings)", - "properties": { - "smtp": { - "groups": ["auth"], - "type": "object", - "description": "Configuration for email sending", - "properties": { - "host": { - "groups": ["auth"], - "type": "string", - "description": "SMTP server hostname" - }, - "port": { - "groups": ["auth"], - "type": "integer", - "description": "SMTP server port" - }, - "user": { - "groups": ["auth"], - "type": "string", - "description": "SMTP username for authentication" - }, - "replyTo": { - "groups": ["auth"], - "type": "string", - "format": "email", - "description": "Reply-to email address for sent emails" - }, - "from": { - "groups": ["auth"], - "type": "string", - "format": "email", - "description": "From email address for sent emails" - }, - "envelopeFrom": { - "groups": ["auth"], - "type": "string", - "format": "email", - "description": "Envelope from address for sent emails" - } - } - }, - "verifyEmail": { - "groups": ["auth"], - "type": "boolean", - "default": true, - "description": "If true, requires email verification for new accounts" - }, - "resetPasswordAllowed": { - "groups": ["auth"], - "type": "boolean", - "default": true, - "description": "If true, allows users to reset their passwords" - }, - "registrationAllowed": { - "groups": ["auth"], - "type": "boolean", - "default": true, - "description": "If true, allows users to register new accounts in Keycloak." - } - } - }, - "insecureCookies": { - "groups": ["auth"], - "type": "boolean", - "default": false, - "description": "If true, allows insecure cookies." - }, - "registrationMessage": { - "groups": ["auth"], - "type": "string", - "default": "You must agree to the terms of use.", - "description": "Message displayed during user registration, typically including terms of service." - }, - "runDevelopmentMainDatabase": { - "groups": ["db"], - "type": "boolean", - "default": true, - "description": "If true, runs a development database within the cluster." - }, - "runDevelopmentKeycloakDatabase": { - "groups": ["db"], - "type": "boolean", - "default": true, - "description": "If true, runs a development Keycloak database within the cluster." - }, - "developmentDatabasePersistence": { - "groups": ["db"], - "type": "boolean", - "default": true, - "description": "If true, makes the database on the argocd preview persistent." - }, - "disableWebsite": { - "groups": ["services"], - "type": "boolean", - "default": false, - "description": "If true, disables the frontend website deployment." - }, - "disableBackend": { - "groups": ["services"], - "type": "boolean", - "default": false, - "description": "If true, disables the backend API deployment." - }, - "disablePreprocessing": { - "groups": ["services"], - "type": "boolean", - "default": false, - "description": "If true, disables preprocessing pipelines." - }, - "disableIngest": { - "groups": ["services"], - "type": "boolean", - "default": false, - "description": "If true, disables ingestion services." - }, - "disableEnaSubmission": { - "groups": ["services"], - "type": "boolean", - "default": false, - "description": "If true, disables ENA (European Nucleotide Archive) submission service." - }, - "customWebsiteImage": { - "groups": ["services"], - "type": "string", - "description": "Optional custom Docker image for the website." - }, - "organisms": { + "definitions": { + "organism": { "groups": ["organism-conf"], "type": "object", "description": "An object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)", @@ -687,92 +366,417 @@ "required": ["version", "image"] } }, - "ingest": { - "groups": ["organism"], - "docsIncludePrefix": false, + "ingest": { + "groups": ["organism"], + "docsIncludePrefix": false, + "type": "object", + "description": "Object of type [Ingest](#ingest-type)", + "additionalProperties": false, + "properties": { + "image": { + "groups": ["ingest"], + "docsIncludePrefix": false, + "type": "string", + "default": "ghcr.io/loculus-project/ingest", + "description": "Docker image for the ingest pipeline" + }, + "configFile": { + "groups": ["ingest"], + "docsIncludePrefix": false, + "type": "object", + "description": "Fields that should be added to the ingest pipeline config file" + } + } + }, + "referenceGenomes": { + "docsIncludePrefix": false, + "type": "object", + "additionalProperties": false, + "properties": { + "nucleotideSequences": { + "groups": ["organism"], + "type": "array", + "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name of the sequence" + }, + "sequence": { + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, + "type": "string" + }, + "insdcAccessionFull": { + "groups": ["nucleotide-sequence"], + "docsIncludePrefix": false, + "type": "string", + "description": "INSDC accession of the sequence" + } + }, + "required": ["name", "sequence"] + } + }, + "genes": { + "groups": ["organism"], + "type": "array", + "description": "Array of [Gene (type)](#gene-type)", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "groups": ["gene"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name of the sequence." + }, + "sequence": { + "groups": ["gene"], + "docsIncludePrefix": false, + "type": "string" + } + }, + "required": ["name", "sequence"] + } + } + } + } + }, + "required": ["schema"] + } + } + } + }, + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "groups": ["general"], + "type": "string", + "default": "Loculus", + "description": "The name of the Loculus instance" + }, + "logo": { + "groups": ["general"], + "type": "object", + "description": "Configuration for the logo", + "properties": { + "url": { + "groups": ["general"], + "type": "string", + "description": "URL path to the logo image file" + }, + "width": { + "groups": ["general"], + "type": "integer", + "default": 100, + "description": "Width of the logo in pixels" + }, + "height": { + "groups": ["general"], + "type": "integer", + "default": 100, + "description": "Height of the logo in pixels" + } + } + }, + "accessionPrefix": { + "groups": ["general"], + "group": "general", + "type": "string", + "default": "LOC_", + "description": "Prefix used for accession numbers" + }, + "environment": { + "groups": ["general"], + "type": "string", + "enum": ["local", "server"], + "default": "server", + "description": "Deployment environment. local for development, server for production" + }, + "host": { + "groups": ["general"], + "type": "string", + "description": "Hostname where Loculus will be accessible" + }, + "bannerMessage": { + "groups": ["general"], + "type": "string", + "default": "This is a demonstration environment. It may contain non-accurate test data and should not be used for real-world applications. Data will be deleted regularly.", + "description": "Banner message (as HTML) to display at the very top of the page" + }, + "welcomeMessageHTML": { + "groups": ["general"], + "type": ["string", "null"], + "description": "A custom welcome message to be shown on the landing page" + }, + "additionalHeadHTML": { + "groups": ["general"], + "type": "string", + "description": "Additional HTML to inject into the of pages" + }, + "createTestAccounts": { + "groups": ["general"], + "type": "boolean", + "description": "If true, creates the users testuser and superuser" + }, + "robotsNoindexHeader": { + "groups": ["general"], + "type": "boolean", + "description": "If true, adds a noindex header to prevent search engine indexing" + }, + "seqSets": { + "groups": ["general"], + "type": "object", + "properties": { + "enabled": { + "groups": ["general"], + "type": "boolean", + "description": "Enable/disable SeqSets. If false, `seqSets.crossRef` can be omitted." + }, + "crossRef": { + "groups": ["general"], + "type": ["object"], + "description": "Configuration for CrossRef integration. Set to `null` to disable CrossRef integration (you can still use SeqSets without CrossRef DOIs).", + "properties": { + "DOIPrefix": { + "groups": ["general"], + "type": "string", + "description": "The DOI prefix for SeqSets" + }, + "endpoint": { + "groups": ["general"], + "type": "string", + "description": "The API endpoint for CrossRef" + }, + "databaseName": { + "groups": ["general"], + "type": "string", + "description": "The database name for CrossRef" + }, + "email": { + "groups": ["general"], + "type": "string", + "format": "email", + "description": "The email address associated with the CrossRef account" + }, + "organization": { + "groups": ["general"], + "type": "string", + "description": "The organization name for CrossRef" + }, + "hostUrl": { + "groups": ["general"], + "type": "string", + "description": "The host URL for CrossRef callbacks" + } + } + } + } + }, + "gitHubMainUrl": { + "groups": ["general"], + "type": "string", + "default": "https://github.com/loculus-project/loculus", + "description": "The link that the GitHub icon in the footer points to" + }, + "website": { + "groups": ["website"], + "type": "object", + "description": "Website specific setting", + "properties": { + "websiteConfig": { + "groups": ["website"], + "type": "object", + "description": "Settings for the `website_config.json`", + "properties": { + "enableLoginNavigationItem": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether the website should show the login button." + }, + "enableSubmissionNavigationItem": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether the website should show \"Submit\" link in the top navigation bar." + }, + "enableSubmissionPages": { + "groups": ["website"], + "type": "boolean", + "default": true, + "description": "Whether to completely disable submission related pages. Setting this to false is useful when hosting Loculus for analysis-only purposes." + } + } + }, + "runtimeConfig": { + "type": "object", + "properties": { + "public": { + "groups": ["website"], "type": "object", - "description": "Object of type [Ingest](#ingest-type)", - "additionalProperties": false, + "description": "Settings for the `public` section of the `runtime_config.json`", "properties": { - "image": { - "groups": ["ingest"], - "docsIncludePrefix": false, + "backendUrl": { + "groups": ["website"], "type": "string", - "default": "ghcr.io/loculus-project/ingest", - "description": "Docker image for the ingest pipeline" + "description": "Overwrite the URL where the client-side website code expects the Loculus backend" }, - "configFile": { - "groups": ["ingest"], - "docsIncludePrefix": false, - "type": "object", - "description": "Fields that should be added to the ingest pipeline config file" - } - } - }, - "referenceGenomes": { - "docsIncludePrefix": false, - "type": "object", - "additionalProperties": false, - "properties": { - "nucleotideSequences": { - "groups": ["organism"], - "type": "array", - "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name of the sequence" - }, - "sequence": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string" - }, - "insdcAccessionFull": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string", - "description": "INSDC accession of the sequence" - } - }, - "required": ["name", "sequence"] - } + "keycloakUrl": { + "groups": ["website"], + "type": "string", + "description": "Overwrite the URL where the client-side website code expects the Keycloak" }, - "genes": { - "groups": ["organism"], - "type": "array", - "description": "Array of [Gene (type)](#gene-type)", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "groups": ["gene"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name of the sequence." - }, - "sequence": { - "groups": ["gene"], - "docsIncludePrefix": false, - "type": "string" - } - }, - "required": ["name", "sequence"] - } + "lapisUrlTemplate": { + "groups": ["website"], + "type": "string", + "description": "Overwrite the URLs where the client-side website code expects the LAPIS instances. Must contain `%organism%` as a placeholder." } } } - }, - "required": ["schema"] + } + } + } + }, + "auth": { + "groups": ["auth"], + "type": "object", + "description": "User authentication (Keycloak settings)", + "properties": { + "smtp": { + "groups": ["auth"], + "type": "object", + "description": "Configuration for email sending", + "properties": { + "host": { + "groups": ["auth"], + "type": "string", + "description": "SMTP server hostname" + }, + "port": { + "groups": ["auth"], + "type": "integer", + "description": "SMTP server port" + }, + "user": { + "groups": ["auth"], + "type": "string", + "description": "SMTP username for authentication" + }, + "replyTo": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "Reply-to email address for sent emails" + }, + "from": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "From email address for sent emails" + }, + "envelopeFrom": { + "groups": ["auth"], + "type": "string", + "format": "email", + "description": "Envelope from address for sent emails" + } + } + }, + "verifyEmail": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, requires email verification for new accounts" + }, + "resetPasswordAllowed": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, allows users to reset their passwords" + }, + "registrationAllowed": { + "groups": ["auth"], + "type": "boolean", + "default": true, + "description": "If true, allows users to register new accounts in Keycloak." } } }, + "insecureCookies": { + "groups": ["auth"], + "type": "boolean", + "default": false, + "description": "If true, allows insecure cookies." + }, + "registrationMessage": { + "groups": ["auth"], + "type": "string", + "default": "You must agree to the terms of use.", + "description": "Message displayed during user registration, typically including terms of service." + }, + "runDevelopmentMainDatabase": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, runs a development database within the cluster." + }, + "runDevelopmentKeycloakDatabase": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, runs a development Keycloak database within the cluster." + }, + "developmentDatabasePersistence": { + "groups": ["db"], + "type": "boolean", + "default": true, + "description": "If true, makes the database on the argocd preview persistent." + }, + "disableWebsite": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables the frontend website deployment." + }, + "disableBackend": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables the backend API deployment." + }, + "disablePreprocessing": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables preprocessing pipelines." + }, + "disableIngest": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables ingestion services." + }, + "disableEnaSubmission": { + "groups": ["services"], + "type": "boolean", + "default": false, + "description": "If true, disables ENA (European Nucleotide Archive) submission service." + }, + "customWebsiteImage": { + "groups": ["services"], + "type": "string", + "description": "Optional custom Docker image for the website." + }, + "organisms": { "$ref": "#/definitions/organism" }, + "defaultOrganisms": { "$ref": "#/definitions/organism" }, "lineageSystemDefinitions": { "groups": ["organism-conf"], "type": "object", From 6f04209a4f01752291f74939fad7c87aadb6b623 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 14:34:03 +0100 Subject: [PATCH 33/56] Add undocumented props --- kubernetes/loculus/values.schema.json | 118 +++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index cafe66295..a83810335 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -776,7 +776,6 @@ "description": "Optional custom Docker image for the website." }, "organisms": { "$ref": "#/definitions/organism" }, - "defaultOrganisms": { "$ref": "#/definitions/organism" }, "lineageSystemDefinitions": { "groups": ["organism-conf"], "type": "object", @@ -800,6 +799,123 @@ } } } + }, + "enaDbName": { + "type": "string", + "default": "Loculus", + "description": "TODO" + }, + "enforceHTTPS": { + "type": "boolean", + "default": true, + "description": "TODO" + }, + "ingestLimitSeconds": { + "type": "integer", + "default": 1800, + "description": "TODO" + }, + "dataUseTermsUrls": { + "type": "object", + "additionalProperties": false, + "properties": { + "open": { + "type": "string" + }, + "restricted": { + "type": "string" + } + } + }, + "subdomainSeparator": { + "type": "string", + "default": "-" + }, + "getSubmissionListLimitSeconds": { + "type": "integer", + "default": 600 + }, + "replicas": { + "type": "object", + "properties": { + "website": { + "type": "integer", + "description": "TODO", + "default": 1 + }, + "backend": { + "type": "integer", + "description": "TODO", + "default": 1 + } + }, + "additionalProperties": false + }, + "preprocessingTimeout": { + "type": "integer", + "description": "TODO", + "default": 600 + }, + "enaUniqueSuffix": { + "type": "string", + "description": "TODO", + "default": "Loculus" + }, + "enaIsBroker": { + "type": "boolean", + "description": "TODO", + "default": false + }, + "submitToEnaProduction": { + "type": "boolean", + "description": "TODO", + "default": false + }, + "images": { + "type": "object", + "properties": { + "lapisSilo": { + "type": "string", + "description": "TODO", + "default": "ghcr.io/genspectrum/lapis-silo:0.5.2" + }, + "lapis": { + "type": "string", + "description": "TODO", + "default": "ghcr.io/genspectrum/lapis:0.3.13" + } + }, + "additionalProperties": false + }, + "registrationTermsMessage": { + "type": "string", + "description": "TODO", + "default": "You must agree to the terms of use." + }, + "siloImportLimitSeconds": { + "type": "integer", + "description": "TODO", + "default": 3600 + }, + "gitHubEditLink": { + "type": "string", + "description": "TODO", + "default": "https://github.com/loculus-project/loculus/edit/main/monorepo/website/" + }, + "defaultResources": { + "description": "TODO", + }, + "resources": { + "description": "TODO", + }, + "secrets": { + "description": "TODO", + }, + "defaultOrganismConfig": { + "description": "TODO", + }, + "defaultOrganisms": { + "description": "TODO", } }, "required": ["name"] From 2452157ba3452912ac8d327fa10fa08505dbc6fb Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 14:44:54 +0100 Subject: [PATCH 34/56] some minor refinements --- kubernetes/loculus/values.schema.json | 17 ++++++++--------- kubernetes/loculus/values.yaml | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index a83810335..886b0cbef 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -76,7 +76,8 @@ "type": { "groups": ["metadata"], "docsIncludePrefix": false, - "type": "string" + "type": "string", + "default": "string" }, "header": { "groups": ["metadata"], @@ -244,7 +245,7 @@ "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." } }, - "required": ["name", "displayName", "type"] + "required": ["name"] } }, "metadataTemplate": { @@ -776,6 +777,7 @@ "description": "Optional custom Docker image for the website." }, "organisms": { "$ref": "#/definitions/organism" }, + "defaultOrganisms": { "$ref": "#/definitions/organism" }, "lineageSystemDefinitions": { "groups": ["organism-conf"], "type": "object", @@ -903,19 +905,16 @@ "default": "https://github.com/loculus-project/loculus/edit/main/monorepo/website/" }, "defaultResources": { - "description": "TODO", + "description": "TODO" }, "resources": { - "description": "TODO", + "description": "TODO" }, "secrets": { - "description": "TODO", + "description": "TODO" }, "defaultOrganismConfig": { - "description": "TODO", - }, - "defaultOrganisms": { - "description": "TODO", + "description": "TODO" } }, "required": ["name"] diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 56872eb3d..4ad3a1161 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -943,7 +943,7 @@ defaultOrganismConfig: &defaultOrganismConfig ontology_id: GENEPIO:0001474 definition: The average number of reads representing a given nucleotide in the reconstructed sequence. guidance: Provide value as a fold of coverage (as a number). - example: 400 + example: "400" displayName: Depth of coverage type: int header: Sequencing @@ -952,7 +952,7 @@ defaultOrganismConfig: &defaultOrganismConfig ontology_id: GENEPIO:0001475 definition: The threshold used as a cut-off for the depth of coverage. guidance: Provide the threshold fold coverage (as a number) - example: 100 + example: "100" displayName: Breadth of coverage type: int header: Sequencing From cf03bf49180285a736944423b3a1c742583643a6 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 15:14:31 +0100 Subject: [PATCH 35/56] Add resources schema --- kubernetes/loculus/values.schema.json | 58 ++++++++++++++++++++++++++- kubernetes/loculus/values.yaml | 2 +- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 886b0cbef..77dd01d5d 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -452,6 +452,25 @@ "required": ["schema"] } } + }, + "resourceSpec": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { "type": "string" }, + "cpu": { "type": "string" } + } + }, + "limits": { + "type": "object", + "properties": { + "memory": { "type": "string" }, + "cpu": { "type": "string" } + } + } + } } }, "type": "object", @@ -905,10 +924,45 @@ "default": "https://github.com/loculus-project/loculus/edit/main/monorepo/website/" }, "defaultResources": { - "description": "TODO" + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { "type": "string" }, + "cpu": { "type": "string" } + }, + "required": ["memory", "cpu"], + "additionalProperties": false + }, + "limits": { + "type": "object", + "properties": { + "memory": { "type": "string" }, + "cpu": { "type": "string" } + }, + "required": ["memory", "cpu"], + "additionalProperties": false + } + }, + "required": ["requests", "limits"], + "additionalProperties": false }, "resources": { - "description": "TODO" + "type": "object", + "properties": { + "website": { "$ref": "#/definitions/resourceSpec" }, + "docs": { "$ref": "#/definitions/resourceSpec" }, + "ena-submission": { "$ref": "#/definitions/resourceSpec" }, + "ena-submission-list-cronjob": { "$ref": "#/definitions/resourceSpec" }, + "ingest": { "$ref": "#/definitions/resourceSpec" }, + "keycloak": { "$ref": "#/definitions/resourceSpec" }, + "silo": { "$ref": "#/definitions/resourceSpec" }, + "lapis": { "$ref": "#/definitions/resourceSpec" }, + "silo-preprocessing": { "$ref": "#/definitions/resourceSpec" }, + "backend": { "$ref": "#/definitions/resourceSpec" }, + "preprocessing": { "$ref": "#/definitions/resourceSpec" } + } }, "secrets": { "description": "TODO" diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 4ad3a1161..a55e08aa5 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1712,7 +1712,7 @@ resources: cpu: "10m" limits: memory: "10Gi" - ingest: + ingest: # also ingest-init requests: memory: "1Gi" cpu: "200m" From 681f6a659e3e89261cfb3f359213984b584c38f6 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 15:19:47 +0100 Subject: [PATCH 36/56] more progress --- kubernetes/loculus/values.schema.json | 805 +++++++++++++------------- 1 file changed, 404 insertions(+), 401 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 77dd01d5d..04c5d7e6f 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -4,454 +4,447 @@ "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "organism": { - "groups": ["organism-conf"], "type": "object", - "description": "An object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)", - "patternProperties": { - "^.*$": { + "properties": { + "schema": { + "groups": ["organism"], + "docsIncludePrefix": false, "type": "object", + "description": "Object of type [Schema](#schema-type)", "properties": { - "schema": { - "groups": ["organism"], + "organismName": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string", + "description": "Display name for the organism" + }, + "image": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string", + "description": "URL to an image that will be shown on the landing page" + }, + "loadSequencesAutomatically": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "boolean", + "default": true, + "description": "Whether sequences be loaded automatically on the sequence details page , rather than users having to press a button to do so. (For small genomes, this should probably be true.)" + }, + "submissionDataTypes": { "docsIncludePrefix": false, "type": "object", - "description": "Object of type [Schema](#schema-type)", "properties": { - "organismName": { - "groups": ["schema"], - "docsIncludePrefix": false, - "type": "string", - "description": "Display name for the organism" - }, - "image": { - "groups": ["schema"], - "docsIncludePrefix": false, - "type": "string", - "description": "URL to an image that will be shown on the landing page" - }, - "loadSequencesAutomatically": { + "consensusSequences": { "groups": ["schema"], - "docsIncludePrefix": false, "type": "boolean", "default": true, - "description": "Whether sequences be loaded automatically on the sequence details page , rather than users having to press a button to do so. (For small genomes, this should probably be true.)" - }, - "submissionDataTypes": { - "docsIncludePrefix": false, - "type": "object", - "properties": { - "consensusSequences": { - "groups": ["schema"], + "description": "If false, the submission form will not allow submission of consensus sequences (i.e. the sequences file must be omitted). All consensus sequence related parts on the website will be hidden." + } + } + }, + "description": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "string" + }, + "metadata": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "array", + "description": "Array of [Metadata (type)](#metadata-type). Metadata fields associated with the organism.", + "items": { + "type": "object", + "properties": { + "name": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Key used across app to refer to this field." + }, + "displayName": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name displayed to users." + }, + "type": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "default": "string" + }, + "header": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Grouping of fields in sequence details UI." + }, + "required": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "Whether the field is required by backend." + }, + "desired": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "Whether the field is a desired input field for submitters." + }, + "definition": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Definition of input field for submitters." + }, + "guidance": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Guidance for submitters on filling in input field." + }, + "noInput": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "Whether a field with this name is expected as possible input, and so should be included in the metadata template and as a form field. (If set to true it will not be included)." + }, + "generateIndex": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "Whether the field should be indexed for search. This is only allowed for string fields and facilitates faster filters. It is recommended if the number of different values is rather small." + }, + "columnWidth": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "number", + "description": "The minimum column width for this field on the search table." + }, + "order": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "number", + "description": "Order for the column in the search table, lower first." + }, + "autocomplete": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "Whether autocomplete should be offered for the field. This is only allowed for string fields and probably `generateIndex` should be true." + }, + "enableSubstringSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. This only works for string fields, and you cannot also enable `autocomplete`." + }, + "rangeSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "If true, enables range search for numeric fields." + }, + "rangeOverlapSearch": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "object", + "description": "Config settings for enabling range overlap search.", + "properties": { + "rangeId": { + "groups": ["metadata"], + "type": "string", + "description": "The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID." + }, + "rangeDisplayName": { + "groups": ["metadata"], + "type": "string", + "description": "The display name of the range." + }, + "bound": { + "groups": ["metadata"], + "type": "string", + "enum": ["lower", "upper"], + "description": "Whether this field is the lower or upper bound of the range." + } + } + }, + "initiallyVisible": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "If true, the field is initially visible in the UI." + }, + "hideOnSequenceDetailsPage": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "If true, hides the field on the sequence details page." + }, + "hideInSearchResultsTable": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "boolean", + "description": "If true, hides the field in the search results table (and makes it impossible to show)." + }, + "perSegment": { + "groups": ["metadata"], + "docsIncludePrefix": false, "type": "boolean", - "default": true, - "description": "If false, the submission form will not allow submission of consensus sequences (i.e. the sequences file must be omitted). All consensus sequence related parts on the website will be hidden." - } + "description": "Whether this is a metadata field that should exist for each segment. If so fields will be created as fieldName_A, fieldName_B in the case of an organism with segments A and B." + }, + "customDisplay": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "object", + "description": "Custom display settings for the field on the sequence details page.", + "properties": { + "type": { + "groups": ["metadata"], + "type": "string" + }, + "url": { + "groups": ["metadata"], + "type": "string" + } + } + }, + "ontology_id": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string" + }, + "example": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Example value for the field." + }, + "ingest": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Which NCBI field to map to this field." + }, + "preprocessing": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "object", + "description": "The values of this field will be added to the preprocessing pipeline config file and the available values depend on the chosen pipeline. For the Nextclade pipeline, please see [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions)." + }, + "lineageSystem": { + "groups": ["metadata"], + "docsIncludePrefix": false, + "type": "string", + "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." } }, - "description": { + "required": ["name"] + } + }, + "metadataTemplate": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "array", + "description": "Array of strings. Which input fields to add to the downloadable metadata template on the submission and revision page.", + "items": { + "type": "string" + } + }, + "earliestReleaseDate": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. For each version of an accession, the `earliestReleaseDate` is calculated as the earliest date of the internal release date, the dates in the configured `externalFields` and the value from the previous version of the accession (if there is one). This can be used when having a mix of sequences imported from other databases, as well as sequences released first in this Loculus instance, to have a field that shows the earliest release date regardless of where the sequence was first released.", + "properties": { + "enabled": { "groups": ["schema"], - "docsIncludePrefix": false, - "type": "string" + "type": "boolean", + "description": "Whether to enable the `earliestReleaseDate` metadata field." }, - "metadata": { + "externalFields": { "groups": ["schema"], - "docsIncludePrefix": false, "type": "array", - "description": "Array of [Metadata (type)](#metadata-type). Metadata fields associated with the organism.", + "description": "Field names to use when calculating the earliest release date. The fields need to be nullable strings formated with `yyyy-mm-dd`.", "items": { - "type": "object", - "properties": { - "name": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Key used across app to refer to this field." - }, - "displayName": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name displayed to users." - }, - "type": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "default": "string" - }, - "header": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Grouping of fields in sequence details UI." - }, - "required": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field is required by backend." - }, - "desired": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field is a desired input field for submitters." - }, - "definition": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Definition of input field for submitters." - }, - "guidance": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Guidance for submitters on filling in input field." - }, - "noInput": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether a field with this name is expected as possible input, and so should be included in the metadata template and as a form field. (If set to true it will not be included)." - }, - "generateIndex": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field should be indexed for search. This is only allowed for string fields and facilitates faster filters. It is recommended if the number of different values is rather small." - }, - "columnWidth": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "number", - "description": "The minimum column width for this field on the search table." - }, - "order": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "number", - "description": "Order for the column in the search table, lower first." - }, - "autocomplete": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether autocomplete should be offered for the field. This is only allowed for string fields and probably `generateIndex` should be true." - }, - "enableSubstringSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. This only works for string fields, and you cannot also enable `autocomplete`." - }, - "rangeSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, enables range search for numeric fields." - }, - "rangeOverlapSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "Config settings for enabling range overlap search.", - "properties": { - "rangeId": { - "groups": ["metadata"], - "type": "string", - "description": "The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID." - }, - "rangeDisplayName": { - "groups": ["metadata"], - "type": "string", - "description": "The display name of the range." - }, - "bound": { - "groups": ["metadata"], - "type": "string", - "enum": ["lower", "upper"], - "description": "Whether this field is the lower or upper bound of the range." - } - } - }, - "initiallyVisible": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, the field is initially visible in the UI." - }, - "hideOnSequenceDetailsPage": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, hides the field on the sequence details page." - }, - "hideInSearchResultsTable": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, hides the field in the search results table (and makes it impossible to show)." - }, - "perSegment": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether this is a metadata field that should exist for each segment. If so fields will be created as fieldName_A, fieldName_B in the case of an organism with segments A and B." - }, - "customDisplay": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "Custom display settings for the field on the sequence details page.", - "properties": { - "type": { - "groups": ["metadata"], - "type": "string" - }, - "url": { - "groups": ["metadata"], - "type": "string" - } - } - }, - "ontology_id": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string" - }, - "example": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Example value for the field." - }, - "ingest": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Which NCBI field to map to this field." - }, - "preprocessing": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "The values of this field will be added to the preprocessing pipeline config file and the available values depend on the chosen pipeline. For the Nextclade pipeline, please see [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions)." - }, - "lineageSystem": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." - } - }, - "required": ["name"] + "type": "string" } - }, - "metadataTemplate": { + } + } + }, + "website": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration for how the organism data is displayed on the website", + "additionalProperties": false, + "properties": { + "tableColumns": { "groups": ["schema"], - "docsIncludePrefix": false, "type": "array", - "description": "Array of strings. Which input fields to add to the downloadable metadata template on the submission and revision page.", "items": { "type": "string" - } + }, + "description": "Columns to display in the browse table" }, - "earliestReleaseDate": { + "defaultOrderBy": { "groups": ["schema"], - "docsIncludePrefix": false, - "type": "object", - "description": "Configuration object for enabling and configuring the `earliestReleaseDate` metadata field. For each version of an accession, the `earliestReleaseDate` is calculated as the earliest date of the internal release date, the dates in the configured `externalFields` and the value from the previous version of the accession (if there is one). This can be used when having a mix of sequences imported from other databases, as well as sequences released first in this Loculus instance, to have a field that shows the earliest release date regardless of where the sequence was first released.", - "properties": { - "enabled": { - "groups": ["schema"], - "type": "boolean", - "description": "Whether to enable the `earliestReleaseDate` metadata field." - }, - "externalFields": { - "groups": ["schema"], - "type": "array", - "description": "Field names to use when calculating the earliest release date. The fields need to be nullable strings formated with `yyyy-mm-dd`.", - "items": { - "type": "string" - } - } - } + "type": "string", + "description": "Default column to sort the browse table" }, - "website": { + "defaultOrder": { "groups": ["schema"], - "docsIncludePrefix": false, - "type": "object", - "description": "Configuration for how the organism data is displayed on the website", - "additionalProperties": false, - "properties": { - "tableColumns": { - "groups": ["schema"], - "type": "array", - "items": { - "type": "string" - }, - "description": "Columns to display in the browse table" - }, - "defaultOrderBy": { - "groups": ["schema"], - "type": "string", - "description": "Default column to sort the browse table" - }, - "defaultOrder": { - "groups": ["schema"], - "type": "string", - "enum": ["ascending", "descending"], - "description": "Default order direction." - } - } + "type": "string", + "enum": ["ascending", "descending"], + "description": "Default order direction." + } + } + }, + "silo": { + "groups": ["schema"], + "docsIncludePrefix": false, + "type": "object", + "description": "Configuration regarding the SILO database engine", + "additionalProperties": false, + "properties": { + "dateToSortBy": { + "groups": ["schema"], + "type": "string", + "description": "Name of a field of type date. This might speed up date range searches on this field and improve sequence compression." }, - "silo": { + "partitionBy": { "groups": ["schema"], - "docsIncludePrefix": false, - "type": "object", - "description": "Configuration regarding the SILO database engine", - "additionalProperties": false, - "properties": { - "dateToSortBy": { - "groups": ["schema"], - "type": "string", - "description": "Name of a field of type date. This might speed up date range searches on this field and improve sequence compression." - }, - "partitionBy": { - "groups": ["schema"], - "type": "string" - } - } + "type": "string" } + } + } + }, + "required": ["organismName"] + }, + "preprocessing": { + "groups": ["organism"], + "docsIncludePrefix": false, + "type": "array", + "description": "Array of [Preprocessing (type)](#preprocessing-type).", + "items": { + "type": "object", + "properties": { + "version": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "integer", + "minimum": 1, + "description": "Version of the preprocessing pipeline" + }, + "image": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "string", + "description": "Docker image for the preprocessing pipeline" }, - "required": ["organismName"] + "args": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "array", + "items": { + "type": "string" + }, + "description": "Array of Strings. Arguments passed to the preprocessing pipeline." + }, + "configFile": { + "groups": ["preprocessing"], + "docsIncludePrefix": false, + "type": "object", + "description": "Fields that should be added to the preprocessing pipeline config file." + } }, - "preprocessing": { - "groups": ["organism"], + "required": ["version", "image"] + } + }, + "ingest": { + "groups": ["organism"], + "docsIncludePrefix": false, + "type": "object", + "description": "Object of type [Ingest](#ingest-type)", + "additionalProperties": false, + "properties": { + "image": { + "groups": ["ingest"], "docsIncludePrefix": false, + "type": "string", + "default": "ghcr.io/loculus-project/ingest", + "description": "Docker image for the ingest pipeline" + }, + "configFile": { + "groups": ["ingest"], + "docsIncludePrefix": false, + "type": "object", + "description": "Fields that should be added to the ingest pipeline config file" + } + } + }, + "referenceGenomes": { + "docsIncludePrefix": false, + "type": "object", + "additionalProperties": false, + "properties": { + "nucleotideSequences": { + "groups": ["organism"], "type": "array", - "description": "Array of [Preprocessing (type)](#preprocessing-type).", + "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", "items": { "type": "object", + "additionalProperties": false, "properties": { - "version": { - "groups": ["preprocessing"], - "docsIncludePrefix": false, - "type": "integer", - "minimum": 1, - "description": "Version of the preprocessing pipeline" - }, - "image": { - "groups": ["preprocessing"], + "name": { + "groups": ["nucleotide-sequence"], "docsIncludePrefix": false, "type": "string", - "description": "Docker image for the preprocessing pipeline" + "description": "Name of the sequence" }, - "args": { - "groups": ["preprocessing"], + "sequence": { + "groups": ["nucleotide-sequence"], "docsIncludePrefix": false, - "type": "array", - "items": { - "type": "string" - }, - "description": "Array of Strings. Arguments passed to the preprocessing pipeline." + "type": "string" }, - "configFile": { - "groups": ["preprocessing"], + "insdcAccessionFull": { + "groups": ["nucleotide-sequence"], "docsIncludePrefix": false, - "type": "object", - "description": "Fields that should be added to the preprocessing pipeline config file." + "type": "string", + "description": "INSDC accession of the sequence" } }, - "required": ["version", "image"] + "required": ["name", "sequence"] } }, - "ingest": { + "genes": { "groups": ["organism"], - "docsIncludePrefix": false, - "type": "object", - "description": "Object of type [Ingest](#ingest-type)", - "additionalProperties": false, - "properties": { - "image": { - "groups": ["ingest"], - "docsIncludePrefix": false, - "type": "string", - "default": "ghcr.io/loculus-project/ingest", - "description": "Docker image for the ingest pipeline" - }, - "configFile": { - "groups": ["ingest"], - "docsIncludePrefix": false, - "type": "object", - "description": "Fields that should be added to the ingest pipeline config file" - } - } - }, - "referenceGenomes": { - "docsIncludePrefix": false, - "type": "object", - "additionalProperties": false, - "properties": { - "nucleotideSequences": { - "groups": ["organism"], - "type": "array", - "description": "Array of [Nucleotide sequence (type)](#nucleotidesequence-type)", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name of the sequence" - }, - "sequence": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string" - }, - "insdcAccessionFull": { - "groups": ["nucleotide-sequence"], - "docsIncludePrefix": false, - "type": "string", - "description": "INSDC accession of the sequence" - } - }, - "required": ["name", "sequence"] + "type": "array", + "description": "Array of [Gene (type)](#gene-type)", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "groups": ["gene"], + "docsIncludePrefix": false, + "type": "string", + "description": "Name of the sequence." + }, + "sequence": { + "groups": ["gene"], + "docsIncludePrefix": false, + "type": "string" } }, - "genes": { - "groups": ["organism"], - "type": "array", - "description": "Array of [Gene (type)](#gene-type)", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "groups": ["gene"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name of the sequence." - }, - "sequence": { - "groups": ["gene"], - "docsIncludePrefix": false, - "type": "string" - } - }, - "required": ["name", "sequence"] - } - } + "required": ["name", "sequence"] } } - }, - "required": ["schema"] + } } - } + }, + "required": ["schema"] }, "resourceSpec": { "type": "object", @@ -795,8 +788,21 @@ "type": "string", "description": "Optional custom Docker image for the website." }, - "organisms": { "$ref": "#/definitions/organism" }, - "defaultOrganisms": { "$ref": "#/definitions/organism" }, + "organisms": { + "groups": ["organism-conf"], + "type": "object", + "description": "An object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)", + "patternProperties": { + "^.*$": { "$ref": "#/definitions/organism" } + } + }, + "defaultOrganismConfig": { "$ref": "#/definitions/organism" }, + "defaultOrganisms": { + "type": "object", + "patternProperties": { + "^.*$": { "$ref": "#/definitions/organism" } + } + }, "lineageSystemDefinitions": { "groups": ["organism-conf"], "type": "object", @@ -966,9 +972,6 @@ }, "secrets": { "description": "TODO" - }, - "defaultOrganismConfig": { - "description": "TODO" } }, "required": ["name"] From c6fb3c24697ca2d2398fae9512fcfa48f6498620 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 15:53:39 +0100 Subject: [PATCH 37/56] more progress --- kubernetes/loculus/values.schema.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 04c5d7e6f..17f10ee56 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -972,6 +972,19 @@ }, "secrets": { "description": "TODO" + }, + "backendExtraArgs": { + "type": "array", + "description": "TODO", + "items": { + "type": "string" + } + }, + "previewDocs": { + "description": "TODO" + }, + "reduceResourceRequest": { + "description": "TODO" } }, "required": ["name"] From b343516b889ffaf31021f282fbdb8156fed3b682 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 16:08:25 +0100 Subject: [PATCH 38/56] symlink schema --- docs/src/assets/values.schema.json | 1 + docs/src/content/docs/reference/helm-chart-config.mdx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 docs/src/assets/values.schema.json diff --git a/docs/src/assets/values.schema.json b/docs/src/assets/values.schema.json new file mode 120000 index 000000000..2557e7846 --- /dev/null +++ b/docs/src/assets/values.schema.json @@ -0,0 +1 @@ +../../../kubernetes/loculus/values.schema.json \ No newline at end of file diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 59aa31504..9321a2eaa 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -5,7 +5,7 @@ description: A reference of config fields for the Loculus Helm chart import SchemaDocs from '../../../components/SchemaDocs.astro'; -export const schemaPath = '../kubernetes/loculus/values.schema.json'; +export const schemaPath = 'src/assets/values.schema.json'; :::note The config structure has not been finalized and may change rapidly until Loculus 1.0 is From 7ece52ffcc467bd1c6a71e244fc13e549df3c698 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:24:09 +0100 Subject: [PATCH 39/56] change how schema file is handled in the docs --- .github/workflows/docs-deploy.yml | 4 +++ .github/workflows/docs-image.yml | 5 +++- .github/workflows/docs-test.yml | 3 ++ docs/.gitignore | 2 ++ docs/package.json | 5 ++-- docs/src/assets/values.schema.json | 1 - docs/src/components/SchemaDocs.astro | 7 +++-- .../docs/reference/helm-chart-config.mdx | 30 +++++++++---------- 8 files changed, 35 insertions(+), 22 deletions(-) delete mode 120000 docs/src/assets/values.schema.json diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index db9447faf..0abf805a9 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -22,6 +22,10 @@ jobs: - name: Navigate to docs directory run: cd docs + - name: Copy Kubernetes schema to docs + run: npm run copy-schema + working-directory: ./docs + - name: Install Dependencies run: npm install working-directory: ./docs diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index e7da1bae7..9695bcb46 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -42,7 +42,7 @@ jobs: - name: Generate files hash id: files-hash run: | - DIR_HASH=$(echo -n ${{ hashFiles('docs/**', '.github/workflows/docs-image.yml') }}) + DIR_HASH=$(echo -n ${{ hashFiles('docs/**', '.github/workflows/docs-image.yml', 'kubernetes/loculus/values.schema.json') }}) echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV - name: Setup Docker metadata id: dockerMetadata @@ -73,6 +73,9 @@ jobs: run: | NODE_VERSION=$(cat docs/.nvmrc | tr -cd [:digit:].) echo "NODE_VERSION=$NODE_VERSION" >> $GITHUB_ENV + - name: Copy Kubernetes schema to docs + run: npm run copy-schema + working-directory: ./docs - name: Build and push image if: env.CACHE_HIT == 'false' uses: docker/build-push-action@v6 diff --git a/.github/workflows/docs-test.yml b/.github/workflows/docs-test.yml index a1394bad9..af8b354bb 100644 --- a/.github/workflows/docs-test.yml +++ b/.github/workflows/docs-test.yml @@ -22,6 +22,9 @@ jobs: uses: actions/setup-node@v4 with: node-version-file: ./docs/.nvmrc + - name: Copy Kubernetes schema to docs + run: npm run copy-schema + working-directory: ./docs - name: Install Dependencies run: npm install working-directory: ./docs diff --git a/docs/.gitignore b/docs/.gitignore index 6240da8b1..319dba4e4 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -12,6 +12,8 @@ yarn-debug.log* yarn-error.log* pnpm-debug.log* +# +src/values.schema.json # environment variables .env diff --git a/docs/package.json b/docs/package.json index e0f6e7c42..c36b012b1 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,8 +3,9 @@ "type": "module", "version": "0.0.1", "scripts": { - "dev": "astro dev", - "start": "astro dev", + "copy-schema": "cp ../kubernetes/loculus/values.schema.json src/", + "dev": "npm run copy-schema && astro dev", + "start": "npm run copy-schema && astro dev", "build": "astro check && astro build", "preview": "astro preview", "astro": "astro", diff --git a/docs/src/assets/values.schema.json b/docs/src/assets/values.schema.json deleted file mode 120000 index 2557e7846..000000000 --- a/docs/src/assets/values.schema.json +++ /dev/null @@ -1 +0,0 @@ -../../../kubernetes/loculus/values.schema.json \ No newline at end of file diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 8b7225aa8..62151be4a 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -9,11 +9,14 @@ import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName, type JSONSchema7Type } from 'json-schema'; import MarkdownRenderer from './MarkdownRenderer.astro'; -const { schemaPath, group, fieldColumnClass } = Astro.props; +const { group, fieldColumnClass } = Astro.props; + let schema: JSONSchema7 = {}; +// TODO - schema currently needs to be read like this and cannot be imported, +// because it actually doesn't conform to the JSONSchema7 type. try { - const data = await readFile(schemaPath, 'utf-8'); + const data = await readFile('src/values.schema.json', 'utf-8'); schema = JSON.parse(data); } catch (error) { console.error('Error reading schema:', error); diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 9321a2eaa..6ec35dd2f 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -5,8 +5,6 @@ description: A reference of config fields for the Loculus Helm chart import SchemaDocs from '../../../components/SchemaDocs.astro'; -export const schemaPath = 'src/assets/values.schema.json'; - :::note The config structure has not been finalized and may change rapidly until Loculus 1.0 is officially released (see [current state and roadmap](../../introduction/current-state-and-roadmap/)). @@ -17,7 +15,7 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## General Settings - +
@@ -54,26 +52,26 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll ## Website Settings - + ## User registration and authentication - + ## Database deployments - + For production environments, these should always be set to false. Instead, external managed databases should be used. ## Services - + ## Organism Configuration - + ### Lineage system definitions @@ -88,25 +86,25 @@ lineageSystemDefinitions: 1: ... ``` - + ### Organism (type) Each organism object has the following fields: - + ### Schema (type) - + ### Metadata (type) - + ### Preprocessing (type) - + The values for `args` and `configFile` depend on the used preprocessing pipeline. For the Nextclade preprocessing pipeline, please see @@ -114,7 +112,7 @@ For the Nextclade preprocessing pipeline, please see ### Ingest (type) - + The values for `configFile` depend on the used preprocessing pipeline. For [our ingest pipeline](https://github.com/loculus-project/loculus/tree/main/ingest) @@ -125,8 +123,8 @@ can be used for segment identification and alignment. ### NucleotideSequence (type) - + ### Gene (type) - + From 8ec26d4fdb49526e61cc228aef9ecded1b13b2f5 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:38:48 +0100 Subject: [PATCH 40/56] Update schema --- kubernetes/loculus/values.schema.json | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 17f10ee56..e6bf73e04 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -842,17 +842,28 @@ "default": 1800, "description": "TODO" }, - "dataUseTermsUrls": { + "dataUseTerms": { "type": "object", - "additionalProperties": false, "properties": { - "open": { - "type": "string" + "enabled": { + "type": "boolean" }, - "restricted": { - "type": "string" + "urls": { + "type": "object", + "properties": { + "open": { + "type": "string" + }, + "restricted": { + "type": "string" + } + }, + "required": ["open", "restricted"], + "additionalProperties": false } - } + }, + "required": ["enabled", "urls"], + "additionalProperties": false }, "subdomainSeparator": { "type": "string", @@ -985,6 +996,15 @@ }, "reduceResourceRequest": { "description": "TODO" + }, + "ingest": { + "type": "object", + "properties": { + "ncbiGatewayUrl": { + "type": ["string", "null"] + } + }, + "additionalProperties": false } }, "required": ["name"] From 1e847bab2548fb7ad627617769a44aceb3389a4a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:48:11 +0100 Subject: [PATCH 41/56] add 'branch' and 'sha' --- kubernetes/loculus/values.schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index e6bf73e04..9a6089127 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1005,6 +1005,14 @@ } }, "additionalProperties": false + }, + "branch": { + "type": "string", + "description": "TODO" + }, + "sha": { + "type": "string", + "description": "TODO" } }, "required": ["name"] From 231164715906954b6fe396952cf96ca4b0295587 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:57:08 +0100 Subject: [PATCH 42/56] Some cleanup and documenting --- docs/.gitignore | 2 +- docs/src/components/SchemaDocs.astro | 20 ++++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index 319dba4e4..48416f538 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -12,7 +12,7 @@ yarn-debug.log* yarn-error.log* pnpm-debug.log* -# +# this file is only copied over from /kubernetes/loculus src/values.schema.json # environment variables diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index 62151be4a..c355f9545 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -1,26 +1,18 @@ --- /** - * Render parts of a JSON schema into a table. - * specify the schema file in `schemaPath`, and give a `group` to render. + * Render parts of the JSON schema into a table, given a `group` to render. * Properties in the schema can be assigned to groups by adding a new "groups": ["group1", "group2"] key to them. * This way, you can split schema definition into semantic groups. */ -import { readFile } from 'node:fs/promises'; import { type JSONSchema7, type JSONSchema7Definition, type JSONSchema7TypeName, type JSONSchema7Type } from 'json-schema'; import MarkdownRenderer from './MarkdownRenderer.astro'; +import rawSchema from '../values.schema.json'; -const { group, fieldColumnClass } = Astro.props; - -let schema: JSONSchema7 = {}; +// the 'as any' isn't pretty but needed because our schema doesn't actually conform to the type. +// it's still nice to use the type to get at least some type hinting when developing. +let schema: JSONSchema7 = rawSchema as any; -// TODO - schema currently needs to be read like this and cannot be imported, -// because it actually doesn't conform to the JSONSchema7 type. -try { - const data = await readFile('src/values.schema.json', 'utf-8'); - schema = JSON.parse(data); -} catch (error) { - console.error('Error reading schema:', error); -} +const { group, fieldColumnClass } = Astro.props; /** Example: 'boolean' -> 'Boolean'. */ function capitalizeFirst(str: string) { From ba536c9db92196f0acf0dadae696f8a03a1e09f2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 17:58:25 +0100 Subject: [PATCH 43/56] remove outdated comment --- kubernetes/loculus/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index a55e08aa5..4ad3a1161 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1712,7 +1712,7 @@ resources: cpu: "10m" limits: memory: "10Gi" - ingest: # also ingest-init + ingest: requests: memory: "1Gi" cpu: "200m" From 97ed86984fe4c4ac5b137150552d5ba49f1d5bed Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 18:13:36 +0100 Subject: [PATCH 44/56] progress --- docs/src/components/SchemaDocs.astro | 8 +- kubernetes/loculus/values.schema.json | 353 ++++++++++++-------------- 2 files changed, 170 insertions(+), 191 deletions(-) diff --git a/docs/src/components/SchemaDocs.astro b/docs/src/components/SchemaDocs.astro index c355f9545..f9b2bfac9 100644 --- a/docs/src/components/SchemaDocs.astro +++ b/docs/src/components/SchemaDocs.astro @@ -98,8 +98,12 @@ function addSelfAndChildren(prefix: string, key: string, definition: JSONSchema7 } if (schema.definitions) { - Object.entries(schema.definitions).forEach(([definitionName, definition]) => { - addSelfAndChildren("", definitionName, definition); + Object.entries(schema.definitions).forEach(([_, definition]) => { + if (typeof definition === 'object' && definition.properties) { + Object.entries(definition.properties).forEach(([key, definition]) => { + addSelfAndChildren("", key, definition); + }); + } }); } diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 9a6089127..51b9f3d23 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -3,6 +3,169 @@ "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut of a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "metadata": { + "type": "object", + "properties": { + "name": { + "groups": ["metadata"], + "type": "string", + "description": "Key used across app to refer to this field." + }, + "displayName": { + "groups": ["metadata"], + "type": "string", + "description": "Name displayed to users." + }, + "type": { + "groups": ["metadata"], + "type": "string", + "default": "string" + }, + "header": { + "groups": ["metadata"], + "type": "string", + "description": "Grouping of fields in sequence details UI." + }, + "required": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether the field is required by backend." + }, + "desired": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether the field is a desired input field for submitters." + }, + "definition": { + "groups": ["metadata"], + "type": "string", + "description": "Definition of input field for submitters." + }, + "guidance": { + "groups": ["metadata"], + "type": "string", + "description": "Guidance for submitters on filling in input field." + }, + "noInput": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether a field with this name is expected as possible input, and so should be included in the metadata template and as a form field. (If set to true it will not be included)." + }, + "generateIndex": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether the field should be indexed for search. This is only allowed for string fields and facilitates faster filters. It is recommended if the number of different values is rather small." + }, + "columnWidth": { + "groups": ["metadata"], + "type": "number", + "description": "The minimum column width for this field on the search table." + }, + "order": { + "groups": ["metadata"], + "type": "number", + "description": "Order for the column in the search table, lower first." + }, + "autocomplete": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether autocomplete should be offered for the field. This is only allowed for string fields and probably `generateIndex` should be true." + }, + "enableSubstringSearch": { + "groups": ["metadata"], + "type": "boolean", + "description": "If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. This only works for string fields, and you cannot also enable `autocomplete`." + }, + "rangeSearch": { + "groups": ["metadata"], + "type": "boolean", + "description": "If true, enables range search for numeric fields." + }, + "rangeOverlapSearch": { + "groups": ["metadata"], + "type": "object", + "description": "Config settings for enabling range overlap search.", + "properties": { + "rangeId": { + "groups": ["metadata"], + "type": "string", + "description": "The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID." + }, + "rangeDisplayName": { + "groups": ["metadata"], + "type": "string", + "description": "The display name of the range." + }, + "bound": { + "groups": ["metadata"], + "type": "string", + "enum": ["lower", "upper"], + "description": "Whether this field is the lower or upper bound of the range." + } + } + }, + "initiallyVisible": { + "groups": ["metadata"], + "type": "boolean", + "description": "If true, the field is initially visible in the UI." + }, + "hideOnSequenceDetailsPage": { + "groups": ["metadata"], + "type": "boolean", + "description": "If true, hides the field on the sequence details page." + }, + "hideInSearchResultsTable": { + "groups": ["metadata"], + "type": "boolean", + "description": "If true, hides the field in the search results table (and makes it impossible to show)." + }, + "perSegment": { + "groups": ["metadata"], + "type": "boolean", + "description": "Whether this is a metadata field that should exist for each segment. If so fields will be created as fieldName_A, fieldName_B in the case of an organism with segments A and B." + }, + "customDisplay": { + "groups": ["metadata"], + "type": "object", + "description": "Custom display settings for the field on the sequence details page.", + "properties": { + "type": { + "groups": ["metadata"], + "type": "string" + }, + "url": { + "groups": ["metadata"], + "type": "string" + } + } + }, + "ontology_id": { + "groups": ["metadata"], + "type": "string" + }, + "example": { + "groups": ["metadata"], + "type": "string", + "description": "Example value for the field." + }, + "ingest": { + "groups": ["metadata"], + "type": "string", + "description": "Which NCBI field to map to this field." + }, + "preprocessing": { + "groups": ["metadata"], + "type": "object", + "description": "The values of this field will be added to the preprocessing pipeline config file and the available values depend on the chosen pipeline. For the Nextclade pipeline, please see [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions)." + }, + "lineageSystem": { + "groups": ["metadata"], + "type": "string", + "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." + } + }, + "required": ["name"] + }, "organism": { "type": "object", "properties": { @@ -53,195 +216,7 @@ "docsIncludePrefix": false, "type": "array", "description": "Array of [Metadata (type)](#metadata-type). Metadata fields associated with the organism.", - "items": { - "type": "object", - "properties": { - "name": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Key used across app to refer to this field." - }, - "displayName": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Name displayed to users." - }, - "type": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "default": "string" - }, - "header": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Grouping of fields in sequence details UI." - }, - "required": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field is required by backend." - }, - "desired": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field is a desired input field for submitters." - }, - "definition": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Definition of input field for submitters." - }, - "guidance": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Guidance for submitters on filling in input field." - }, - "noInput": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether a field with this name is expected as possible input, and so should be included in the metadata template and as a form field. (If set to true it will not be included)." - }, - "generateIndex": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether the field should be indexed for search. This is only allowed for string fields and facilitates faster filters. It is recommended if the number of different values is rather small." - }, - "columnWidth": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "number", - "description": "The minimum column width for this field on the search table." - }, - "order": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "number", - "description": "Order for the column in the search table, lower first." - }, - "autocomplete": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether autocomplete should be offered for the field. This is only allowed for string fields and probably `generateIndex` should be true." - }, - "enableSubstringSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, search results will contain results that contain the given value as a substring. If false (the default), only exact matches will be returned. This only works for string fields, and you cannot also enable `autocomplete`." - }, - "rangeSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, enables range search for numeric fields." - }, - "rangeOverlapSearch": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "Config settings for enabling range overlap search.", - "properties": { - "rangeId": { - "groups": ["metadata"], - "type": "string", - "description": "The range that this field belongs to. Two fields (the upper and lower bound) need to be defined with the same range ID." - }, - "rangeDisplayName": { - "groups": ["metadata"], - "type": "string", - "description": "The display name of the range." - }, - "bound": { - "groups": ["metadata"], - "type": "string", - "enum": ["lower", "upper"], - "description": "Whether this field is the lower or upper bound of the range." - } - } - }, - "initiallyVisible": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, the field is initially visible in the UI." - }, - "hideOnSequenceDetailsPage": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, hides the field on the sequence details page." - }, - "hideInSearchResultsTable": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "If true, hides the field in the search results table (and makes it impossible to show)." - }, - "perSegment": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "boolean", - "description": "Whether this is a metadata field that should exist for each segment. If so fields will be created as fieldName_A, fieldName_B in the case of an organism with segments A and B." - }, - "customDisplay": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "Custom display settings for the field on the sequence details page.", - "properties": { - "type": { - "groups": ["metadata"], - "type": "string" - }, - "url": { - "groups": ["metadata"], - "type": "string" - } - } - }, - "ontology_id": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string" - }, - "example": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Example value for the field." - }, - "ingest": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Which NCBI field to map to this field." - }, - "preprocessing": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "object", - "description": "The values of this field will be added to the preprocessing pipeline config file and the available values depend on the chosen pipeline. For the Nextclade pipeline, please see [here](https://github.com/loculus-project/loculus/blob/main/preprocessing/nextclade/README.md#custom-preprocessing-functions)." - }, - "lineageSystem": { - "groups": ["metadata"], - "docsIncludePrefix": false, - "type": "string", - "description": "Use this on string fields that contain lineages, if you want to enable searches that can include sublineages. The value needs to be a lineage system that is defined under the `lineageSystemDefinitions` key." - } - }, - "required": ["name"] - } + "items": { "$ref": "#/definitions/metadata" } }, "metadataTemplate": { "groups": ["schema"], From c6fc5aeb3034acea587e50f90438f6efd089840a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 7 Feb 2025 18:24:16 +0100 Subject: [PATCH 45/56] ArgoCD fix attempt --- kubernetes/loculus/values.schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 51b9f3d23..65eef8c13 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -988,6 +988,10 @@ "sha": { "type": "string", "description": "TODO" + }, + "shortbranch": { + "type": "string", + "description": "TODO - this is required/set by ArgoCD" } }, "required": ["name"] From 2d42d1c2a9d9a8435e92dbec620333de48d77433 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 11 Feb 2025 11:23:24 +0100 Subject: [PATCH 46/56] Add restriction on organism names --- kubernetes/loculus/values.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 65eef8c13..ff7f3fc67 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -768,14 +768,14 @@ "type": "object", "description": "An object where the keys are the organism IDs and values are an [Organism (type)](#organism-type)", "patternProperties": { - "^.*$": { "$ref": "#/definitions/organism" } + "^[a-z0-9-]+$": { "$ref": "#/definitions/organism" } } }, "defaultOrganismConfig": { "$ref": "#/definitions/organism" }, "defaultOrganisms": { "type": "object", "patternProperties": { - "^.*$": { "$ref": "#/definitions/organism" } + "^[a-z0-9-]+$": { "$ref": "#/definitions/organism" } } }, "lineageSystemDefinitions": { @@ -784,7 +784,7 @@ "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-system-definitions))", "additionalProperties": false, "patternProperties": { - "^.*$": { + "^[a-z0-9-]+$": { "groups": ["lineage-system-definitions"], "placeholder": "name", "type": "object", From c0ba68bfee4c672d898768b828a5a9e4d6b9b94a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 11 Feb 2025 11:25:54 +0100 Subject: [PATCH 47/56] foo --- kubernetes/loculus/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index ff7f3fc67..fc20ac5ec 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -784,7 +784,7 @@ "description": "An object where the keys are the lineage system names and values are links to lineage system definition files per pipeline version (See [Lineage system definitions](#lineage-system-definitions))", "additionalProperties": false, "patternProperties": { - "^[a-z0-9-]+$": { + "^[a-zA-Z0-9-]+$": { "groups": ["lineage-system-definitions"], "placeholder": "name", "type": "object", From 25b6594ca1c5b189237a1afb3be00ccd99e24663 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 09:55:46 +0000 Subject: [PATCH 48/56] Update kubernetes/loculus/values.schema.json Co-authored-by: Chaoran Chen --- kubernetes/loculus/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index fc20ac5ec..16dafe3ae 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1,6 +1,6 @@ { "title": "values.yaml Schema for the Loculus Helm Chart", - "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut of a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", + "description": "Also has 'groups' and 'placeholder' keys in some places, which are used during docs generation. You can set 'docsIncludePrefix' to false to cut off a path when generating docs. Have a look at the SchemaDocs.astro component in the Loculus docs.", "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "metadata": { From a9d66e263b13e666d0dc1127342f81db5a9012ec Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 10:57:59 +0100 Subject: [PATCH 49/56] data use terms --- .../docs/reference/helm-chart-config.mdx | 33 ------------------- kubernetes/loculus/values.schema.json | 9 +++-- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 6ec35dd2f..0b80fd743 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -17,39 +17,6 @@ The configuration for the Helm chart is provided as a YAML file. It has the foll -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldTypeDefaultDescription
`dataUseTerms.enabled`BooleantrueWhether this Loculus instance handles data use terms.
`dataUseTerms.urls.open`StringA URL describing the open data use terms.
`dataUseTerms.urls.restricted`StringA URL describing the restricted data use terms.
-
- ## Website Settings diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 16dafe3ae..1e93ab7c1 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -821,16 +821,19 @@ "type": "object", "properties": { "enabled": { - "type": "boolean" + "type": "boolean", + "description": "Whether this Loculus instance handles data use terms." }, "urls": { "type": "object", "properties": { "open": { - "type": "string" + "type": "string", + "description": "A URL describing the open data use terms." }, "restricted": { - "type": "string" + "type": "string", + "description": "A URL describing the restricted data use terms." } }, "required": ["open", "restricted"], From e6c41b7ca396a65a09d1b97991ee41956539a913 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 11:37:17 +0100 Subject: [PATCH 50/56] refine some of the text --- docs/src/content/docs/reference/helm-chart-config.mdx | 6 ++++-- kubernetes/loculus/values.schema.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/reference/helm-chart-config.mdx b/docs/src/content/docs/reference/helm-chart-config.mdx index 0b80fd743..e5b79270d 100644 --- a/docs/src/content/docs/reference/helm-chart-config.mdx +++ b/docs/src/content/docs/reference/helm-chart-config.mdx @@ -61,11 +61,13 @@ Each organism object has the following fields: -### Schema (type) +### Organism schema (type) -### Metadata (type) +### Metadata field (type) + +Definition of metadata fields for sequence entries of an organism, for example the collection date and location of a sample. diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 1e93ab7c1..95f0cbcea 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -215,7 +215,7 @@ "groups": ["schema"], "docsIncludePrefix": false, "type": "array", - "description": "Array of [Metadata (type)](#metadata-type). Metadata fields associated with the organism.", + "description": "Array of [Metadata fields (type)](#metadata-type) associated with the organism.", "items": { "$ref": "#/definitions/metadata" } }, "metadataTemplate": { From 8a0ed3e03073134b5f869d1426d6851b3051d842 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 11:39:37 +0100 Subject: [PATCH 51/56] organism schema --- kubernetes/loculus/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 95f0cbcea..652e13373 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -173,7 +173,7 @@ "groups": ["organism"], "docsIncludePrefix": false, "type": "object", - "description": "Object of type [Schema](#schema-type)", + "description": "Object of type [organism schema](#schema-type)", "properties": { "organismName": { "groups": ["schema"], From 0da588b82f40c9d3988ccf45acdf6b86fbcc58da Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 11:52:50 +0100 Subject: [PATCH 52/56] make the type an enum --- kubernetes/loculus/values.schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 652e13373..c5f9b922d 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -19,6 +19,7 @@ "type": { "groups": ["metadata"], "type": "string", + "enum": ["string", "int", "float", "number", "date", "boolean", "authors"], "default": "string" }, "header": { From 31e79619b9817e63405040f2a88e7c842173bb27 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 12:03:57 +0100 Subject: [PATCH 53/56] Also allow numbers as examples --- kubernetes/loculus/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index c5f9b922d..c5c28ed99 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -146,7 +146,7 @@ }, "example": { "groups": ["metadata"], - "type": "string", + "type": ["string", "number"], "description": "Example value for the field." }, "ingest": { From 7a1007bba231e135bd0d00743424dd8b1e6b28d3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 12 Feb 2025 12:31:56 +0100 Subject: [PATCH 54/56] Add DUT to general section --- kubernetes/loculus/values.schema.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index c5c28ed99..0d1f19d35 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -822,6 +822,7 @@ "type": "object", "properties": { "enabled": { + "groups": ["general"], "type": "boolean", "description": "Whether this Loculus instance handles data use terms." }, @@ -829,10 +830,12 @@ "type": "object", "properties": { "open": { + "groups": ["general"], "type": "string", "description": "A URL describing the open data use terms." }, "restricted": { + "groups": ["general"], "type": "string", "description": "A URL describing the restricted data use terms." } From 030bcd25c434048345bc96f6e50dccbfee3919e2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 13 Feb 2025 10:21:58 +0100 Subject: [PATCH 55/56] Add organismSpecific resources --- kubernetes/loculus/values.schema.json | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 0d1f19d35..9558e9d6d 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -440,7 +440,7 @@ } } } - } + } }, "type": "object", "additionalProperties": false, @@ -960,7 +960,29 @@ "lapis": { "$ref": "#/definitions/resourceSpec" }, "silo-preprocessing": { "$ref": "#/definitions/resourceSpec" }, "backend": { "$ref": "#/definitions/resourceSpec" }, - "preprocessing": { "$ref": "#/definitions/resourceSpec" } + "preprocessing": { "$ref": "#/definitions/resourceSpec" }, + "organismSpecific": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "website": { "$ref": "#/definitions/resourceSpec" }, + "ena-submission": { "$ref": "#/definitions/resourceSpec" }, + "ena-submission-list-cronjob": { "$ref": "#/definitions/resourceSpec" }, + "ingest": { "$ref": "#/definitions/resourceSpec" }, + "keycloak": { "$ref": "#/definitions/resourceSpec" }, + "silo": { "$ref": "#/definitions/resourceSpec" }, + "lapis": { "$ref": "#/definitions/resourceSpec" }, + "silo-preprocessing": { "$ref": "#/definitions/resourceSpec" }, + "backend": { "$ref": "#/definitions/resourceSpec" }, + "preprocessing": { "$ref": "#/definitions/resourceSpec" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } } }, "secrets": { From 801a6ebc70d960a63df8652d86002773a8381e9f Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 13 Feb 2025 10:23:51 +0100 Subject: [PATCH 56/56] Add podPriorityClassName --- kubernetes/loculus/values.schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubernetes/loculus/values.schema.json b/kubernetes/loculus/values.schema.json index 9558e9d6d..1b485c84e 100644 --- a/kubernetes/loculus/values.schema.json +++ b/kubernetes/loculus/values.schema.json @@ -1021,6 +1021,10 @@ "shortbranch": { "type": "string", "description": "TODO - this is required/set by ArgoCD" + }, + "podPriorityClassName": { + "type": "string", + "description": "TODO - found in PPX production.yaml" } }, "required": ["name"]