diff --git a/gradle.properties b/gradle.properties index 82f6b539a0..2a29ae6eb5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 GROUP=com.atlan -VERSION_NAME=4.2.4-SNAPSHOT +VERSION_NAME=4.3.0-SNAPSHOT POM_URL=https://github.com/atlanhq/atlan-java POM_SCM_URL=git@github.com:atlanhq/atlan-java.git diff --git a/package-toolkit/config/src/main/resources/Framework.pkl b/package-toolkit/config/src/main/resources/Framework.pkl new file mode 100644 index 0000000000..51d6f22f05 --- /dev/null +++ b/package-toolkit/config/src/main/resources/Framework.pkl @@ -0,0 +1,1190 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ + +/// Template for defining configuration for a custom package in Atlan. +/// +/// | Variable | | Usage | Default | +/// |---|---|---|---| +/// | **packageId** | | Unique identifier for the package, including its namespace. | | +/// | **packageName** | | Display name for the package, as it should be shown in the UI. | | +/// | **version** | | Version of this package, following semantic versioning. | | +/// | **description** | | Description for the package, as it should be shown in the UI. | | +/// | **iconUrl** | | Link to an icon to use for the package, as it should be shown in the UI. | | +/// | **docsUrl** | | Link to an online document describing the package. | | +/// | **uiConfig** | | Configuration for the UI of the custom package. | | +/// | **implementationLanguage** | | Coding language the package is implemented in. | | +/// | **containerImage** | | Container image to run the logic of the custom package. | | +/// | **containerCommand** | | Full command to run in the container image, as a list rather than spaced. | | +/// | containerImagePullPolicy | | (Optional) Override the default IfNotPresent policy. | `IfNotPresent` | +/// | outputs | | (Optional) Any outputs that the custom package logic is expected to produce. | | +/// | keywords | | (Optional) Any keyword labels to apply to the package. | | +/// | allowSchedule | | (Optional) Whether to allow the package to be scheduled (true) or only run immediately (false). | `true` | +/// | certified | | (Optional) Whether the package should be listed as certified (true) or not (false). | `true` | +/// | preview | | (Optional) Whether the package should be labeled as an early preview in the UI (true) or not (false). | `false` | +/// | connectorType | | (Optional) If the package needs to configure a connector, specify its type here. | | +/// | category | | Name of the pill under which the package should be categorized in the marketplace in the UI. | `custom` | +@ModuleInfo { minPklVersion = "0.27.2" } +open module com.atlan.pkg.Framework + +import "pkl:semver" +import "FrameworkRenderer.pkl" +import "Connectors.pkl" + +/// Unique identifier for the package, including its namespace. +/// For example: @csa/open-api-spec-loader +packageId: String(matches(Regex("@[a-z0-9-]+/[a-z0-9-]+"))) + +/// Display name for the package, as it should be shown in the UI. +packageName: String + +/// Version of this package, following semantic versioning. +version: semver.Version + +/// Description for the package, as it should be shown in the UI. +description: String + +/// Link to an icon to use for the package, as it should be shown in the UI. +iconUrl: String + +/// Link to an online document describing the package. +docsUrl: String + +/// Configuration for an independent publish step. +/// This is necessary for any package intended to run in a secure mode, where the extraction can +/// run outside the Atlan tenant and the results be transferred (pushed) to the Atlan tenant to be +/// loaded. +publishConfig: PublishConfig? + +/// Configuration for the UI of the custom package. +/// +/// | Variable | | Usage | +/// |---|---|---| +/// | **[tasks][UIStep]** | | Mapping of top-level tasks to be configured, keyed by the name of the task as it should appear in the UI. | +/// | **[rules][UIRule]** | | Listing of rules to control which inputs appear based on values selected in other inputs. | +uiConfig: UIConfig + +/// Coding language the package is implemented in. +/// This will control what (if any) strongly-typed configuration hand-over classes are generated by the toolkit. +/// (Note: if using 'Other', no strongly-typed configuration hand-over classes will be generated.) +implementationLanguage: CodeLanguage + +/// Container image to run the logic of the custom package. +containerImage: String + +/// Full command to run in the container image, as a list rather than spaced. +containerCommand: Listing + +/// (Optional) Override the default IfNotPresent policy. +containerImagePullPolicy: ImagePullPolicy = "IfNotPresent" + +/// (Optional) Any outputs that the custom package logic is expected to produce. +outputs: WorkflowOutputs? + +/// (Optional) Any keyword labels to apply to the package. +keywords: Listing = new Listing {} + +/// (Optional) Whether to allow the package to be scheduled (default, true) or only run immediately (false). +allowSchedule: Boolean = true + +/// (Optional) Whether the package should be listed as certified (default, true) or not (false). +certified: Boolean = true + +/// (Optional) Whether the package should be labeled as an early preview in the UI (true) or not (default, false). +preview: Boolean = false + +/// (Optional) If the package needs to configure a connector, specify its type here. +connectorType: Connectors.Type? + +/// Name of the pill under which the package should be categorized in the marketplace in the UI. +category: String = "custom" + +// --- CONTENT ABOVE THIS LINE IS INTENDED INPUT --- +// --- CONTENT BELOW THIS LINE IS USED TO GENERATE OUTPUTS --- + +/// (Generated) Command to run in the container image. +fixed command: List = List(containerCommand[0]) + +/// (Generated) Arguments to provide to the command that runs in the container image. +fixed args: List = containerCommand.toList().sublist(1, containerCommand.length) + +abstract class PublishConfig { + /// Image tag for the version of the publish package to use. + hidden versionTag: String + + /// Container image to run the logic of the publish step. + fixed containerImage: String + + /// Full command to run for the publish step, as a list rather than spaced. + fixed containerCommand: Listing + + /// (Optional) Override the default IfNotPresent policy. + fixed containerImagePullPolicy: ImagePullPolicy = "IfNotPresent" + + /// (Generated) Command to run in the container image. + fixed command: List = List(containerCommand[0]) + + /// (Generated) Arguments to provide to the command that runs in the container image. + fixed args: List = containerCommand.toList().sublist(1, containerCommand.length) + + /// (Optional) Input artifacts that the publish package will consume. + fixed inputArtifacts: Listing + + /// (Optional) Outputs produced by the publish package. + fixed outputs: WorkflowOutputs? + + /// (Optional) Parameters to pass through to the publish package. + fixed parameters: Map? + + /// Transfer a value from a UI configuration input as the setting for this publish config. + /// - `uiConfig` the package's UI configuration + /// - `name` the name of the property in the package's UI configuration to transfer + function transferConfigInput(uiConfig: UIConfig, name: String): String = + if (!uiConfig.properties.containsKey(name)) + throw("Invalid UI configuration property when trying to transfer a config input -- ensure the name matches a UI input: " + name) + else + "{{inputs.parameters." + name + "}}" + + /// Transfer an output file produced by the package as the setting for this publish config. + /// - `outputs` the package's outputs + /// - `name` the name (key) of the output file produced by the package + /// - `rename` (optional) new name to give the output file after transferring it + function transferFile(outputs: WorkflowOutputs?, name: String, rename: String?): FrameworkRenderer.NamePathPair = + if (outputs == null || !outputs.files.containsKey(name)) + throw("Invalid file reference when trying to transfer a file -- ensure the name matches an output from the package: " + name) + else + let (nm = name) + new FrameworkRenderer.NamePathPair { + name = nm + path = if (rename != null) "/tmp/\(rename)" else outputs.files.getOrNull(name)!! + } +} + +/// Use an asset import step as the final publish phase for the package. +class AssetImport extends PublishConfig { + hidden assetsFile: FrameworkRenderer.NamePathPair? = null + hidden assetsUpsertSemantic: ImportSemantic?|String? = null + hidden assetsConfig: ConfigType?|String? = null + hidden assetsAttrToOverwrite: Listing?|String? = null + hidden assetsFailOnErrors: Boolean?|String? = null + hidden assetsCaseSensitive: Boolean?|String? = null + hidden assetsTableViewAgnostic: Boolean?|String? = null + hidden assetsFieldSeparator: String? = null + hidden assetsBatchSize: Int?|String? = null + hidden trackBatches: Boolean?|String? = null + hidden glossariesFile: FrameworkRenderer.NamePathPair? = null + hidden glossariesUpsertSemantic: ImportSemantic?|String? = null + hidden glossariesConfig: ConfigType?|String? = null + hidden glossariesAttrToOverwrite: Listing?|String? = null + hidden glossariesFailOnErrors: Boolean?|String? = null + hidden glossariesFieldSeparator: String? = null + hidden glossariesBatchSize: Int?|String? = null + hidden dataProductsFile: FrameworkRenderer.NamePathPair? = null + hidden dataProductsUpsertSemantic: ImportSemantic?|String? = null + hidden dataProductsConfig: ConfigType?|String? = null + hidden dataProductsAttrToOverwrite: Listing?|String? = null + hidden dataProductsFailOnErrors: Boolean?|String? = null + hidden dataProductsFieldSeparator: String? = null + hidden dataProductsBatchSize: Int?|String? = null + + fixed inputArtifacts { + when (assetsFile != null) { assetsFile } + when (glossariesFile != null) { glossariesFile } + when (dataProductsFile != null) { dataProductsFile } + } + fixed outputs { + files { + ["debug-logs"] = "/tmp/debug.log" + // TODO: add error file, once we produce it + } + } + fixed containerImage = "ghcr.io/atlanhq/csa-asset-import:\(versionTag)" + fixed containerCommand { + "/dumb-init" + "--" + "java" + "com.atlan.pkg.aim.Importer" + } + + fixed parameters: Map = new Mapping { + ["import_type"] = "DIRECT" + when (assetsFile != null) { ["assets_file"] = assetsFile.path } + when (assetsUpsertSemantic != null) { ["assets_upsert_semantic"] = assetsUpsertSemantic } + when (assetsConfig != null) { ["assets_config"] = assetsConfig } + when (assetsAttrToOverwrite != null) { + when (assetsAttrToOverwrite is Listing) { + ["assets_attr_to_overwrite"] = "[\"" + assetsAttrToOverwrite.join("\",\"") + "\"]" + } else { + ["assets_attr_to_overwrite"] = assetsAttrToOverwrite + } + } + when (assetsFailOnErrors != null) { ["assets_fail_on_errors"] = assetsFailOnErrors.toString() } + when (assetsCaseSensitive != null) { ["assets_case_sensitive"] = assetsCaseSensitive.toString() } + when (assetsTableViewAgnostic != null) { ["assets_table_view_agnostic"] = assetsTableViewAgnostic.toString() } + when (assetsFieldSeparator != null) { ["assets_field_separator"] = assetsFieldSeparator } + when (assetsBatchSize != null) { ["assets_batch_size"] = assetsBatchSize.toString() } + when (trackBatches != null) { ["track_batches"] = trackBatches.toString() } + + when (glossariesFile != null) { ["glossaries_file"] = glossariesFile.path } + when (glossariesUpsertSemantic != null) { ["glossaries_upsert_semantic"] = glossariesUpsertSemantic } + when (glossariesConfig != null) { ["glossaries_config"] = glossariesConfig } + when (glossariesAttrToOverwrite != null) { + when (glossariesAttrToOverwrite is Listing) { + ["glossaries_attr_to_overwrite"] = "[\"" + glossariesAttrToOverwrite.join("\",\"") + "\"]" + } else { + ["glossaries_attr_to_overwrite"] = glossariesAttrToOverwrite + } + } + when (glossariesFailOnErrors != null) { ["glossaries_fail_on_errors"] = glossariesFailOnErrors.toString() } + when (glossariesFieldSeparator != null) { ["glossaries_field_separator"] = glossariesFieldSeparator } + when (glossariesBatchSize != null) { ["glossaries_batch_size"] = glossariesBatchSize.toString() } + + when (dataProductsFile != null) { ["data_products_file"] = dataProductsFile.path } + when (dataProductsUpsertSemantic != null) { ["data_products_upsert_semantic"] = dataProductsUpsertSemantic } + when (dataProductsConfig != null) { ["data_products_config"] = dataProductsConfig } + when (dataProductsAttrToOverwrite != null) { + when (dataProductsAttrToOverwrite is Listing) { + ["data_products_attr_to_overwrite"] = "[\"" + dataProductsAttrToOverwrite.join("\",\"") + "\"]" + } else { + ["data_products_attr_to_overwrite"] = dataProductsAttrToOverwrite + } + } + when (dataProductsFailOnErrors != null) { ["data_products_fail_on_errors"] = dataProductsFailOnErrors.toString() } + when (dataProductsFieldSeparator != null) { ["data_products_field_separator"] = dataProductsFieldSeparator } + when (dataProductsBatchSize != null) { ["data_products_batch_size"] = dataProductsBatchSize.toString() } + }.toMap() +} + +/// Configuration for the UI of the custom package. +/// +/// | Variable | | Usage | +/// |---|---|---| +/// | **[tasks][UIStep]** | | Mapping of top-level tasks to be configured, keyed by the name of the task as it should appear in the UI. | +/// | **[rules][UIRule]** | | Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. | +class UIConfig { + + /// Mapping of top-level tasks to be configured, keyed by the name of the task as it should appear in the UI. + /// + /// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces: + /// ``` + /// tasks { + /// ["Input"] { + /// ... + /// } + /// ["Delivery"] { + /// ... + /// } + /// ... + /// } + /// ``` + hidden tasks: Mapping + + /// (Generated) Details of all inputs across all steps for the UI. + fixed properties: Map = new Mapping { + for (_, step in tasks) { + // Note: using spread syntax to error on any duplicates across steps + ...?step.inputs + } + }.toMap() + + /// Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. + /// + /// Remember in Pkl to define a listing, use curly braces and create new elements within: + /// ``` + /// rules { + /// new UIRule { + /// whenInputs { ["export_scope"] = "ENRICHED_ONLY" } + /// required = { "qn_prefix" } + /// } + /// new UIRule { + /// ... + /// } + /// ... + /// } + /// ``` + hidden rules: Listing = new Listing {} + + /// (Generated) Details of all UI rules to use to control the UI. + fixed anyOf: List? = if (rules.isEmpty) null else rules.toList() + + /// (Generated) Details about each task (top-level step) to use in the UI. + fixed steps: List = tasks.fold(List(), (acc: List, key, value) -> + acc.add(resolveStep(key, value)) + ) +} + +const function resolveInputs(inputs: Mapping): Mapping = inputs.fold(new Mapping {}, (acc: Mapping, inputKey, input) -> + (acc) { + [inputKey] = input + } +) + +const function resolveStep(stepName: String, step: UIStep): UIStep = (step) { + title = stepName +} + +/// Configure a top-level step in the UI for the package. +/// +/// | Field | | Description | +/// |---|---|---| +/// | **`title`** | | name to show in the UI for the step | +/// | `description` | | description to show in the UI for the step | +/// | **[inputs][UIElement]** | | mapping of inputs to be configured for the step, keyed by a unique (variable) name for the input | +class UIStep { + + /// Name to show in the UI for the step. + /// This will be set automatically from the key of the outer map in which this step is defined. + title: String + + /// Description to show in the UI for the step. + description: String = "" + + /// Mapping of [inputs][UIElement] to be configured for the step, keyed by a unique (variable) name for the input. + /// Note: the name of the variable (key of the map) should be in lower_snake_case. + /// + /// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces: + /// ``` + /// inputs { + /// ["export_scope"] = new Radio { + /// ... + /// } + /// ["qn_prefix"] = new TextInput { + /// ... + /// } + /// ... + /// } + /// ``` + hidden inputs: Mapping + + /// (Generated) Unique identifier for the step. + fixed id = title.replaceAll(" ", "_").toLowerCase() + + /// (Generated) List of inputs contained within the step. + fixed properties: List = inputs.fold(List(), (acc, key, _) -> acc.add(key)) +} + +/// Validates the provided string value is in snake_case. +local const lower_snake_case = (str) -> str.matches(Regex("[a-z0-9_]*")) + +/// Configure basic UI rules that when the specified inputs have specified values, certain +/// other fields become required. +/// +/// | Field | | Description | +/// |---|---|---| +/// | **`whenInputs`** | | mapping from input ID to value for the step | +/// | **`required`** | | list of input IDs that should become required when the inputs all match | +class UIRule { + fixed properties: Mapping> = new Mapping { + for (k, v in whenInputs) { + [k] = new Mapping { + ["const"] = v + } + } + } + + /// Mapping from input ID (variable name for the input) to value for the step + hidden whenInputs: Mapping + + /// List of input IDs (variable names) that should become required when the inputs all match + required: Listing +} + +/// Class defining any outputs the package's logic will produce. +class WorkflowOutputs { + /// Files the package will produce in the local filesystem. + hidden files: Mapping? + + /// Outputs the package will produce into S3. + s3Objects: Listing? + + /// (Generated) List of all artifacts the package will produce. + fixed artifacts: List = (s3Objects?.toList() ?? List()) + + (files?.fold(List(), (acc: List, key, value) -> + acc.add(new FrameworkRenderer.NamePathPair { + name=key + path=value + })) ?? List()) +} + +class S3Artifact extends FrameworkRenderer.NamedPair { + /// Name of the object in S3 + name: String + + /// Path on the container's local filesystem for the object + path: String = "/tmp/\(name).json" + + /// (Optional) Configuration for how the S3 object should be archived. + archive: Mapping> = new Mapping { + ["none"] = new Mapping {} + } + + /// (Optional) Configuration for how the S3 object should be mapped. + s3: Mapping = new Mapping { + ["key"] = "{{inputs.parameters.\(FrameworkRenderer.S3_CONFIG_PREFIX)}}/\(name).json" + } + + /// (Optional) Configuration for how the S3 object should be garbage collected. + artifactGC: Mapping = new Mapping { + ["strategy"] = "OnWorkflowDeletion" + } +} + +/// Image pull policy for the container image. +/// For more details on behavior, see: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy +typealias ImagePullPolicy = "Always"|"IfNotPresent"|"Never" + +/// Coding languages the toolkit can generate strongly-typed configuration hand-over classes for. +typealias CodeLanguage = "Java"|"Kotlin"|"Python"|"Other" + +/// Options for the semantic to use when importing (publishing) assets in Atlan. +typealias ImportSemantic = "upsert"|"update"|"partial" + +/// Configuration type for a package. +typealias ConfigType = "default"|"advanced" + +/// (Generated) Unique name to use for file-based objects for the package. +hidden fixed name = packageId.replaceAll("@", "").replaceAll("/", "-") + +const function getPythonPkgName(m): String = getLowerSnakeCase(getPascalCase(m.packageName)) + +/// Set up multiple outputs for the module, one for each configuration file. +/// - `m` the package config to generate outputs for +const function getOutputs(m): Mapping = new Mapping { + ["build/package/\(m.name)/configmaps/default.yaml"] = FrameworkRenderer.getConfigMap(m) + ["build/package/\(m.name)/templates/default.yaml"] = FrameworkRenderer.getWorkflowTemplate(m) + ["build/package/\(m.name)/index.js"] = FrameworkRenderer.getIndexJs() + ["build/package/\(m.name)/package.json"] = FrameworkRenderer.getPackageJson(m) + when (m.implementationLanguage == "Kotlin" || m.implementationLanguage == "Java") { + ["src/main/kotlin/\(getClassName(m.packageName)).kt"] = FrameworkRenderer.getConfigClassKt(m, getClassName(m.packageName)) + } + when (m.implementationLanguage == "Python") { + ["\(getPythonPkgName(m))/\(getPythonPkgName(m))_cfg.py"] = FrameworkRenderer.getConfigClassPy(m) + ["\(getPythonPkgName(m))/__init__.py"] = FrameworkRenderer.getBlankFile() + ["\(getPythonPkgName(m))/main.py.example"] = FrameworkRenderer.getMainPy(getPythonPkgName(m)) + ["\(getPythonPkgName(m))/logging.conf"] = FrameworkRenderer.getLoggingConfPy() + ["requirements.txt.example"] = FrameworkRenderer.getRequirementsPy() + ["requirements-dev.txt.example"] = FrameworkRenderer.getRequirementsDevPy() + ["version.txt"] = FrameworkRenderer.getVersionPy(m) + ["Dockerfile"] = FrameworkRenderer.getDockerfilePy(getPythonPkgName(m)) + } +} + +/// Translate the model content into a set of files for both type definitions (JSON) +/// and UI configuration (TypeScript). +const function getModuleOutput(m): ModuleOutput = new ModuleOutput { + files = getOutputs(m) +} + +/// Set the output of the module to be separate files for each custom type definition in the model. +output = getModuleOutput(this) + +/// Turn the provided name into a PascalCase name. +const function getPascalCase(name: String): String = + name.split(Regex("[\\W_]+")).map((word) -> word.capitalize()).join("") + +/// Turn the provided name into a lower_snake_case_name. +const function getLowerSnakeCase(text: String): String = getSnakeCase(text).toLowerCase() +local const function getSnakeCase(text: String): String = + text.replaceAll("_", "").replaceAll(Regex("([A-Z0-9]+)([A-Z][a-z][0-9])"), "$1_$2").replaceAll(Regex("([a-z0-9])([A-Z0-9])"), "$1_$2") + +const function getClassName(name: String): String = "\(getPascalCase(name))Cfg" + +// --- WIDGETS --- +/// Base class for all UI elements that can be used in the configuration of a package. +/// +/// You will not use this class directly, but will use one of its subclasses: +/// +/// | Widget | | Usage | +/// |---|---|---| +/// | [APITokenSelector] | | select an existing API token from a drop-down list, and returns the GUID of the selected API token | +/// | [BooleanInput] | | choose either "Yes" or "No", and returns the value that was selected | +/// | [ConnectionCreator] | | create a new connection by providing a name and list of admins, and returns a string representation of the connection object that should be created | +/// | [ConnectionSelector] | | select an existing connection from a drop-down list, and returns the qualified name of the selected connection | +/// | [ConnectorTypeSelector] | | select from the types of connectors that exist in the tenant (for example "Snowflake"), without needing to select a specific instance of a connection (for example, the "production" connection for Snowflake); will return a string-encoded object giving the connection type that was selected and a list of all connections in the tenant that have that type | +/// | [CredentialInput] | | enter sensitive credential information that will be encrypted and protected in Atlan's Vault, and returns the GUID of the entry in the Vault | +/// | [DateInput] | | enter or select a date (not including time) from a calendar, and returns the epoch-based number representing that selected date in seconds | +/// | [DropDown] | | select from a drop-down of provided options | +/// | [FileCopier] | | copy a file from the tenant's S3 bucket, and returns the name of the file (as it is named in S3) | +/// | [FileUploader] | | upload a file, and returns the GUID-based name of the file (as it is renamed after upload) | +/// | [KeygenInput] | | generate a unique key that could be used for securing an exchange or other unique identification purposes, and provides buttons to regenerate the key or copy its text; will return the generated key as clear text | +/// | [MultipleGroups] | | choose multiple groups, and returns an array of group names that were selected | +/// | [MultipleUsers] | | choose multiple users, and returns an array of usernames that were selected | +/// | [NumericInput] | | enter an arbitrary number into a single-line text input field, and returns the value of the number that was entered | +/// | [PasswordInput] | | enter arbitrary text, but the text will be shown as dots when entered rather than being displayed in clear text; will return the entered text in clear text | +/// | [Radio] | | select just one option from a set of options, and returns the key of the selected option (typically, this is used to control mutually exclusive options in the UI) | +/// | [SingleGroup] | | select a single group, and returns the group name of the selected group | +/// | [SingleUser] | | select a single user, and returns the username of the selected user | +/// | [TextInput] | | enter arbitrary text into a single-line text input field, and returns the value of the text that was entered | +abstract class UIElement { + /// Type of the element, which can be used to determine how to render it in the UI. + fixed type: String + + /// Name to show in the UI for the widget. + hidden title: String + + /// Whether a value must be selected to proceed with the UI setup. + required: Boolean = false + + /// Whether the widget will be shown in the UI (false) or not (true). + hidden hide: Boolean = false + + /// Informational text to place in a hover-over to describe the use of the input. + hidden helpText: String = "" + + /// Sizing of the input on the UI (8 is full-width, 4 is half-width). + hidden width: Int = 8 + + /// Default fallback value to use for the element if nothing is selected or entered in the UI. + hidden fallback: Any? = null + + /// (Generated) UI configuration for the element. + fixed ui: Widget = new Widget { + label = title + `hidden` = hide + help = helpText + grid = width + } +} + +/// Base class for all UI elements that use some enumeration of valid values. +abstract class UIElementWithEnum extends UIElement { + + /// Mapping of possible values for the input, keyed by a unique (variable) name for the value. + hidden possibleValues: Mapping + + /// Default value to select in the input. + default: String? + + /// (Generated) Listing of the unique variable names for the possible values. + fixed enum: List = possibleValues.fold(List(), (acc: List, key, _) -> acc.add(key)) + + /// (Generated) Listing of the display values for each possible value. + fixed enumNames: List = possibleValues.fold(List(), (acc: List, _, value) -> acc.add(value)) +} + +/// Base class for all generated elements of the UI. +class Widget { + + /// Internal name of this widget, that controls what kind of widget is rendered in the UI. + widget: String + + /// Label to show in the UI for the widget. + label: String + + /// Whether the widget will be shown in the UI (false) or not (true). + `hidden`: Boolean = false + + /// Informational text to place in a hover-over to describe the use of the input. + help: String = "" + + /// Example text to place within the widget to exemplify its use. + placeholder: String? + + /// Sizing of the input on the UI (8 is full-width, 4 is half-width). + grid: Int = 8 + + /// Mode of the widget, if applicable (usually whether something allows multiple selections or only a single selection). + mode: String? + + /// TBC + start: Int? + + /// List of the mime-types of files that should be accepted. + accept: List? + + /// Whether the widget should return file metadata. + fileMetadata: Boolean? + + /// Most distant past value that can be selected through the widget. + min: Int? + + /// Most distant future value that can be selected through the widget. + max: Int? + + /// Default value to select in the widget. + default: Any? + + /// Type of credential to be nested within the widget. + credentialType: String? + + /// TBC + nestedValue: Boolean? + + /// Fixed string that should appear (immutable) before the input box. + addonBefore: String? + + /// Whether the value can be changed (false) or is immutable (true). + disabled: Boolean? + + /// SQL query to run to determine the widget's selectable content. + query: String? + + /// Whether to prevent the Test Authentication button from showing (true) or show it (false) for credential inputs. + isTestAuthenticationDisabled: Boolean? +} + +/// Widget that allows you to enter arbitrary text into a single-line text input field, +/// and returns the value of the text that was entered. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `placeholderText` | | example text to place within the widget to exemplify its use | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +/// | `defaultValue` | | default value to use in the widget | | +/// | `prepend` | | fixed text to prepend to the value entered by the user | | +/// | `enabled` | | whether the value can be changed (true) or is immutable (false) | | +class TextInput extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Example text to place within the widget to exemplify its use. + hidden placeholderText: String = "" + + /// (Optional) Default value to use in the widget. + hidden defaultValue: String? + + /// (Optional) Fixed text to prepend to the value entered by the user. + hidden prepend: String? + + /// (Optional) Whether the value can be changed (true) or is immutable (false). + hidden enabled: Boolean? + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "input" + placeholder = placeholderText + addonBefore = prepend + disabled = if (enabled != null) !enabled else null + default = defaultValue + } +} + +/// Widget that allows you to enter arbitrary text into a multi-line text input box, +/// and returns the value of the text that was entered. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `placeholderText` | | example text to place within the widget to exemplify its use | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class TextBoxInput extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Example text to place within the widget to exemplify its use. + hidden placeholderText: String = "" + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "TextInput" + placeholder = placeholderText + } +} + +/// Widget that allows you to select just one option from a set of options, and returns the key of +/// the selected option. +/// Typically, this is used to control mutually exclusive options in the UI. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`possibleValues`** | | possible values that can be selected in the radio button | | +/// | **`default`** | | default value to select in the radio button (its key) | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +class Radio extends UIElementWithEnum { + fixed type = "string" + + /// Possible values that can be selected in the radio button. + /// The key is the value that will be returned, and the value is the text to display. + hidden possibleValues: Mapping + + /// Default value to select in the radio button (its key). + default: String?(if (this != null) possibleValues.keys.contains(this) else true) + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "radio" + } +} + +/// Widget that allows you to create a new connection by providing a name and list of admins, +/// and returns a string representation of the connection object that should be created. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +class ConnectionCreator extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "connection" + } +} + +/// Widget that allows you to select an existing connection from a drop-down list, +/// and returns the qualified name of the selected connection. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `multiSelect` | | whether multiple connections can be selected (true) or only a single connection (false) | `false` | +/// | `begin` | | TBC | `1` | +class ConnectionSelector extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Whether multiple connections can be selected (true) or only a single connection (false). + hidden multiSelect: Boolean = false + + /// TBC + hidden begin: Int = 1 + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "connectionSelector" + mode = if (multiSelect) "multiple" else "" + start = begin + } +} + +/// Widget that allows you to upload a file, and returns the GUID-based name of the file (as it is +/// renamed after upload). +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`fileTypes`** | | list of the mime-types of files that should be accepted | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `placeholderText` | | example text to place within the widget to exemplify its use | `""` | +class FileUploader extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// List of the mime-types of files that should be accepted. + hidden fileTypes: Listing + + /// Example text to place within the widget to exemplify its use. + hidden placeholderText: String = "" + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "fileUpload" + placeholder = placeholderText + accept = fileTypes.toList() + fileMetadata = true + } +} + +/// Widget that allows you to copy a file from the tenant's S3 bucket, and returns the name of the file +/// (as it is named in S3). +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be entered to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `placeholderText` | | example text to place within the widget to exemplify its use | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class FileCopier extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Example text to place within the widget to exemplify its use. + hidden placeholderText: String = "" + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "input" + placeholder = placeholderText + } +} + +/// Widget that allows you to enter an arbitrary number into a single-line text input field, +/// and returns the value of the number that was entered. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `placeholderValue` | | example value to place within the widget to exemplify its use | | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +/// | `enabled` | | whether the value can be changed (true) or is immutable (false) | | +class NumericInput extends UIElement { + fixed type = "number" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Example text to place within the widget to exemplify its use. + hidden placeholderValue: Number? + + /// Default number to use in the widget. + default: Number? + + /// (Optional) Whether the value can be changed (true) or is immutable (false). + hidden enabled: Boolean? + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "inputNumber" + placeholder = placeholderValue?.toString() + disabled = if (enabled != null) !enabled else null + } +} + +/// Widget that allows you to enter or select a date (not including time) from a calendar, +/// and returns the epoch-based number representing that selected date in seconds. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `past` | | an offset from today (0) that indicates how far back in the calendar can be selected (-1 is yesterday, 1 is tomorrow, and so on) | `-14` | +/// | `future` | | an offset from today (0) that indicates how far forward in the calendar can be selected (-1 is yesterday, 1 is tomorrow, and so on) | `0` | +/// | `defaultDay` | | an offset from today that indicates the default date that should be selected in the calendar (0 is today, -1 is yesterday, 1 is tomorrow, and so on) | `0` | +/// | `begin` | | TBC | `1` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class DateInput extends UIElement { + fixed type = "number" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// An offset from today (0) that indicates how far back in the calendar can be selected (-1 is yesterday, 1 is tomorrow, and so on). + hidden past: Int = -14 + + /// An offset from today (0) that indicates how far forward in the calendar can be selected (-1 is yesterday, 1 is tomorrow, and so on). + hidden future: Int = 0 + + /// An offset from today that indicates the default date that should be selected in the calendar (0 is today, -1 is yesterday, 1 is tomorrow, and so on). + hidden defaultDay: Int = 0 + + /// TBC + hidden begin: Int = 1 + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "date" + start = begin + min = past + max = future + default = defaultDay + } +} + +/// Widget that allows you to choose either "Yes" or "No", +/// and returns the value that was selected. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `defaultSelection` | | the default value to use for this boolean input | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +class BooleanInput extends UIElement { + fixed type = "boolean" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// The default value to use for this boolean input. + hidden defaultSelection: Boolean = false + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "boolean" + default = defaultSelection + } +} + +/// Widget that allows you to select from a drop-down of provided options. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`possibleValues`** | | map of option keys to the value that will be display for each option in the drop-down | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `multiSelect` | | whether multiple options can be selected (true) or only a single option (false) | `false` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class DropDown extends UIElementWithEnum { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Whether multiple values can be selected (true) or only a single value (false). + hidden multiSelect: Boolean = false + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "select" + mode = if (multiSelect) "multiple" else "" + } +} + +/// Widget that allows you to choose multiple groups, +/// and returns an array of group names that were selected. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class MultipleGroups extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "groupMultiple" + } +} + +/// Widget that allows you to select a single group, +/// and returns the group name of the selected group. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class SingleGroup extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "groups" + } +} + +/// Widget that allows you to choose multiple users, +/// and returns an array of usernames that were selected. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class MultipleUsers extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "userMultiple" + } +} + +/// Widget that allows you to select a single user, +/// and returns the username of the selected user. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class SingleUser extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "users" + } +} + +/// Widget that allows you to select from the types of connectors that exist in the tenant +/// (for example "Snowflake"), without needing to select a specific instance of a connection +/// (for example, the "production" connection for Snowflake). Will return a string-encoded +/// object giving the connection type that was selected and a list of all connections in the tenant +/// that have that type. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +/// | `begin` | | TBC | `1` | +class ConnectorTypeSelector extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// TBC + hidden begin: Int = 1 + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "sourceConnectionSelector" + start = begin + } +} + +/// Widget that allows you to select an existing API token from a drop-down list, +/// and returns the GUID of the selected API token. +/// Note: currently only API tokens that were created by the user configuring the workflow +/// will appear in the drop-down list. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +class APITokenSelector extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "apiTokenSelect" + } +} + +/// Widget that allows you to generate a unique key that could be used for securing an exchange +/// or other unique identification purposes, and provides buttons to regenerate the key or copy its +/// text. Will return the generated key as clear text. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class KeygenInput extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "keygen" + } +} + +/// Widget that allows you to enter arbitrary text, but the text will be shown as dots when entered +/// rather than being displayed in clear text. Will return the entered text in clear text. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class PasswordInput extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "password" + } +} + +/// Widget that allows you to enter sensitive credential information that will be encrypted and protected +/// in Atlan's Vault and not be visible to anyone. Will return the GUID of the entry in the Vault. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`credType`** | | type of credential to be nested within the widget | | +/// | `required` | | whether a value must be selected to proceed with the UI setup | `false` | +/// | `hide` | | whether the widget will be shown in the UI (false) or not (true) | `false` | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +class CredentialInput extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Type of credential to be nested within the widget. + hidden credType: String + + /// Whether to show the button for testing authentication. (Default: true, show it) + hidden allowTestAuthentication: Boolean = true + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "credential" + credentialType = credType + isTestAuthenticationDisabled = !allowTestAuthentication + } +} + +/// Widget that allows you to configure multiple sub-elements all grouped together under a single parent +/// variable. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`inputs`** | | map of the sub-elements that should be nested within this input | | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +open class NestedInput extends UIElement { + fixed type = "object" + + /// Whether the widget will be shown in the UI (false) or not (true). + hidden hide: Boolean = false // Necessary for generated Kotlin copy method override + + /// Map of the sub-elements that should be nested within this input, keyed by unique lower_snake_case variable name. + hidden inputs: Mapping + + /// (Generated) Map of properties nested within this input. + fixed properties: Map = inputs.toMap() + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "nested" + nestedValue = false + `hidden` = hide + } +} + +/// Widget that executes arbitrary SQL against the connector that is being configured. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`sqlQuery`** | | query to run in the SQL executor | | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class SQLExecutor extends UIElement { + fixed type = "string" + required: Boolean = false // Necessary for generated Kotlin copy method override + + /// Query to run in the SQL executor. + hidden sqlQuery: String + + /// (Generated) Internal configuration for the UI's rendering. + fixed ui { + widget = "sql" + query = sqlQuery + } +} diff --git a/package-toolkit/config/src/main/resources/FrameworkRenderer.pkl b/package-toolkit/config/src/main/resources/FrameworkRenderer.pkl new file mode 100644 index 0000000000..c214656faa --- /dev/null +++ b/package-toolkit/config/src/main/resources/FrameworkRenderer.pkl @@ -0,0 +1,1150 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ + +/// Module for rendering the outputs for a custom package's configuration in Atlan. +@ModuleInfo { minPklVersion = "0.25.1" } +module com.atlan.pkg.FrameworkRenderer + +import "Framework.pkl" +import "Connectors.pkl" +import "Credential.pkl" + +/// Render the configmap YAML file. +const function getConfigMap(m: Framework): FileOutput = new FileOutput { + value = new ConfigMap { + name = m.name + config = m.uiConfig + } + renderer = new YamlRenderer {} +} + +/// Render the connector-specific credential configmap YAML file. +const function getCredentialConfigMap(m: Credential): FileOutput = new FileOutput { + value = new ConnectorConfigMap { + name = m.name + config = m + } + renderer = new YamlRenderer {} +} + +local const function getProcessTemplate( + templateName: String, + uiCfg: Framework.UIConfig, + publish: Framework.PublishConfig?, + package: String, + outs: Framework.WorkflowOutputs?, + img: String, + pullPolicy: Framework.ImagePullPolicy, + cmd: List, + arguments: List, + params: Map?, + inputFiles: Boolean, + inputArtifacts: List +) = new WorkflowTemplateDefinition { + name = templateName + cfg = uiCfg + pub = publish + f = inputFiles + directInputFiles = inputArtifacts + container = new WorkflowContainer { + config = cfg + image = img + command = cmd + args = arguments + imagePullPolicy = pullPolicy + volumeMounts = if (credVariable != null) new Listing { + new VolumeMount { + name = "credentials" + mountPath = "/tmp/credentials" + } + } else null + passthroughParams = params + } + outputs = outs + pkg = package + credVariable = let (creds = cfg.properties.filter((_, u) -> u is Framework.CredentialInput)) + if (creds.isEmpty) null else creds.keys.first + volumes = if (credVariable != null) new Listing { + new EmptyDirVolume { + name = "credentials" + } + } else null + initContainers = if (credVariable != null) new Listing { + new WorkflowContainer { + name = "fetch-credentials" + image = "ghcr.io/atlanhq/rest-master:af621a5" + command = new Listing { + "/bin/sh" + "-c" + """ + if [ -z "$\(credVariable.toUpperCase())" ]; then exit 0; fi + python3 main.py GET http://heracles-service.heracles.svc.cluster.local/credentials/$\(credVariable.toUpperCase())/use --raw-input '{}' --raw-input-file-pattern '' --raw-input-file-sort '' --raw-input-multiline f --execution-script "if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}):\\n LOGGER.debug('Heracles is unavailable. Performing retry with back-off')\\n failure_handler = FailureHandler.RETRY\\nif state == ExecutionState.OUTPUT_PROCESS:\\n output = json.loads(output)\\nif state == ExecutionState.API_POST:\\n stop = True" --raw-input-paginate '0' --auth-type oauth2 --auth-oauth2-type client_credentials --auth-oauth2-impersonate-user "$IMPERSONATE_USER" --auth-oauth2-client-credentials-client-id CLIENT_ID --auth-oauth2-client-credentials-secret CLIENT_SECRET --auth-oauth2-client-credentials-token-url TOKEN_URL --output-chunk-size '0' --output-file-prefix /tmp/credentials --pagination-wait-time '0' --max-retries '10' + """ + }.toList() + env = new Listing { + new NameValuePair { + name = "IMPERSONATE_USER" + value = "{{=sprig.ternary(sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow), '', 'true' == inputs.parameters['impersonate'])}}" + } + new NameValuePair { + name = credVariable.toUpperCase() + value = "{{inputs.parameters.\(credVariable)}}" + } + new NameValuePair { + name = "OAUTHLIB_INSECURE_TRANSPORT" + value = "1" + } + new NamedSecret { + name = "CLIENT_ID" + secretName = "argo-client-creds" + secretKey = "login" + } + new NamedSecret { + name = "CLIENT_SECRET" + secretName = "argo-client-creds" + secretKey = "password" + } + new NamedSecret { + name = "TOKEN_URL" + secretName = "argo-client-creds" + secretKey = "host" + } + }.toList() + mirrorVolumeMounts = true + } + } else null +} + +/// Render the workflow template YAML file. +const function getWorkflowTemplate(m: Framework): FileOutput = new FileOutput { + value = new WorkflowTemplate { + name = m.name + template = getProcessTemplate( + m.name, + m.uiConfig, + m.publishConfig, + m.name, + m.outputs, + m.containerImage, + m.containerImagePullPolicy, + m.command, + m.args, + null, // Only include UI-based parameters at the top-most level + true, // Always include file details at the top-most level (?) + List() // Do not passthrough any non-UI-provided artifacts as inputs at the top-most level + ) + } + renderer = new YamlRenderer {} +} + +/// Render the index.js file. +const function getIndexJs(): FileOutput = new FileOutput { + text = """ + function dummy() { + console.log("don't call this.") + } + module.exports = dummy; + """ +} + +/// Render the package.json file. +const function getPackageJson(m: Framework): FileOutput = new FileOutput { + value = new PackageDefinition { + packageId = m.packageId + packageName = m.packageName + version = m.version.toString() + description = m.description + iconUrl = m.iconUrl + docsUrl = m.docsUrl + keywords = m.keywords + allowSchedule = m.allowSchedule + certified = m.certified + preview = m.preview + connectorType = m.connectorType + category = m.category + } + renderer = new JsonRenderer {} +} + +/// Render the Kotlin class file that strongly-types the configuration handover. +const function getConfigClassKt(m: Framework, className: String): FileOutput = new FileOutput { + text = new Listing { + """ + /* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ + import com.atlan.model.assets.Connection + import com.atlan.pkg.CustomConfig + import com.atlan.pkg.model.ConnectorAndConnections + import com.atlan.pkg.serde.WidgetSerde + import com.fasterxml.jackson.annotation.JsonAutoDetect + import com.fasterxml.jackson.annotation.JsonProperty + import com.fasterxml.jackson.databind.annotation.JsonDeserialize + import com.fasterxml.jackson.databind.annotation.JsonSerialize + import javax.annotation.processing.Generated; + + /** + * Expected configuration for the \(m.packageName) custom package. + */ + @Generated("com.atlan.pkg.CustomPackage") + @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) + data class \(className)( + """ + for (k, u in m.uiConfig.properties) {( + if (u is Framework.DropDown || u is Framework.MultipleGroups || u is Framework.MultipleUsers || u is Framework.ConnectionSelector) + """ + @JsonDeserialize(using = WidgetSerde.MultiSelectDeserializer::class) + @JsonSerialize(using = WidgetSerde.MultiSelectSerializer::class) + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): List\(getFallback(u)), + """ + else if (u is Framework.ConnectorTypeSelector) + """ + @JsonDeserialize(using = WidgetSerde.ConnectorAndConnectionsDeserializer::class) + @JsonSerialize(using = WidgetSerde.ConnectorAndConnectionsSerializer::class) + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): ConnectorAndConnections? = null, + """ + else if (u is Framework.ConnectionCreator) + """ + @JsonDeserialize(using = WidgetSerde.ConnectionDeserializer::class) + @JsonSerialize(using = WidgetSerde.ConnectionSerializer::class) + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): Connection? = null, + """ + else if (u is Framework.BooleanInput) + """ + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): Boolean\(getFallback(u)), + """ + else if (u is Framework.NumericInput) + """ + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): Number\(getFallback(u)), + """ + else if (u is Framework.DateInput) + """ + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): Long\(getFallback(u)), + """ + else + """ + @JsonProperty(\"\(k)\") val \(getLowerCamelCase(k)): String\(getFallback(u)), + """ + )} + """ + ) : CustomConfig() + """ + }.join("\n") +} + +const function getLowerCamelCase(s: String): String = + s.split(Regex("[\\W_]+")).foldIndexed("", (idx, acc, word) -> + if (idx == 0) + "\(acc)\(word.decapitalize())" + else + "\(acc)\(word.capitalize())" + ) + +const function getFallback(u: Framework.UIElement): String = + if (u.fallback == null) + "? = null" + else if (u.fallback is List) + " = listOf(\((u.fallback as List).fold(List(), (acc: List, value) -> + acc.add("\"\(value)\"") + ).join(", ")))" + else if (u.fallback is String) + " = \"\(u.fallback)\"" + else + " = \(u.fallback)" + +const function getFallbackParam(u: Framework.UIElement): Any = + if (u.fallback is List) + "[\((u.fallback as List).fold(List(), (acc: List, value) -> + acc.add("\"\(value)\"") + ).join(", "))]" + else + u.fallback + +/// Render an empty file. +const function getBlankFile(): FileOutput = new FileOutput { + text = "" +} + +/// Render a baseline version file for a Python custom package. +const function getVersionPy(m: Framework): FileOutput = new FileOutput { + text = """ + \(m.version) + """ +} + +/// Render a baseline version file for a Python custom package. +const function getMainPy(pkgName: String): FileOutput = new FileOutput { + text = """ + from \(pkgName).\(pkgName)_cfg import RuntimeConfig + import logging + + LOGGER = logging.getLogger(__name__) + + def main(): + runtime_config = RuntimeConfig() + custom_config = runtime_config.custom_config + # Retrieve inputs from custom_config + + LOGGER.info("Starting execution of \(pkgName)...") + + + if __name__ == "__main__": + main() + """ +} + +/// Render the logging configuration for a Python custom package. +const function getLoggingConfPy(): FileOutput = new FileOutput { + text = """ + [loggers] + keys=root,pyatlan,urllib3 + + [handlers] + keys=consoleHandler,fileHandler,jsonHandler + + [formatters] + keys=simpleFormatter,jsonFormatter + + [logger_root] + level=DEBUG + handlers=consoleHandler,fileHandler + + [logger_pyatlan] + level=DEBUG + handlers= + qualname=pyatlan + propagate=1 + + [logger_urllib3] + level=DEBUG + handlers= + qualname=urllib3 + propagate=0 + + [handler_consoleHandler] + class=StreamHandler + level=INFO + formatter=simpleFormatter + args=(sys.stdout,) + + [handler_fileHandler] + class=FileHandler + level=DEBUG + formatter=simpleFormatter + args=('/tmp/debug.log',) + + [handler_jsonHandler] + class=FileHandler + level=DEBUG + formatter=jsonFormatter + args=('/tmp/pyatlan.json',) + + [formatter_simpleFormatter] + format=%(asctime)s - %(name)s - %(levelname)s - %(message)s + + [formatter_jsonFormatter] + format=%(asctime)s - %(name)s - %(levelname)s - %(message)s + class=pyatlan.utils.JsonFormatter + """ +} + +/// Render the baseline requirements.txt for a Python custom package. +const function getRequirementsPy(): FileOutput = new FileOutput { + text = """ + pyatlan + opentelemetry-api==1.29.0 + opentelemetry-sdk==1.29.0 + opentelemetry-instrumentation-logging==0.50b0 + opentelemetry-exporter-otlp==1.29.0 + """ +} + +/// Render the baseline requirements-dev.txt for a Python custom package. +const function getRequirementsDevPy(): FileOutput = new FileOutput { + text = """ + pytest + pytest-order + nanoid + """ +} + +/// Render the baseline Dockerfile for a Python custom package. +const function getDockerfilePy(pkgName: String): FileOutput = new FileOutput { + text = """ + FROM python:3.9-bookworm + + LABEL org.opencontainers.image.vendor="Atlan Pte. Ltd." \\ + org.opencontainers.image.source="https://github.com/atlanhq/atlan-python" \\ + org.opencontainers.image.description="Atlan image for \(pkgName) custom package." \\ + org.opencontainers.image.licenses=Apache-2 + + COPY requirements.txt requirements.txt + COPY \(pkgName) /app/\(pkgName) + COPY package.pkl /app/package.pkl + COPY version.txt /app/version.txt + + RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \\ + && chmod +x /usr/local/bin/dumb-init \\ + && pip3 install -r requirements.txt + + WORKDIR /app + ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] + """ +} + +/// Render the Python class file that strongly-types the configuration handover. +const function getConfigClassPy(m: Framework): FileOutput = new FileOutput { + text = new Listing { + """ + from datetime import datetime + from pathlib import Path + from pydantic.v1 import BaseModel, BaseSettings, Field, validator + from pyatlan.model.assets import Connection + from pyatlan.pkg.models import ConnectorAndConnection + from pyatlan.pkg.utils import ( + add_otel_handler, + validate_connection, + validate_multiselect, + validate_connector_and_connection, + ) + from typing import Any, Optional, Union, Dict + import logging.config + + PARENT = Path(__file__).parent + LOGGING_CONF = PARENT / "logging.conf" + if LOGGING_CONF.exists(): + logging.config.fileConfig(LOGGING_CONF) + ROOT_LOGGER = logging.getLogger() + add_otel_handler(ROOT_LOGGER, ROOT_LOGGER.level, {}) + LOGGER = logging.getLogger(__name__) + + ENV = 'env' + + class CustomConfig(BaseModel): + \"\"\"\"\"\"\"\" + """ + for (k, u in m.uiConfig.properties) { + if (u is Framework.DropDown || u is Framework.MultipleGroups || u is Framework.MultipleUsers || u is Framework.ConnectionSelector) + """ + \(k): Optional[List[str]] = Field(default_factory=list) + _validate_\(k) = validator("\(k)", pre=True, allow_reuse=True)(validate_multiselect) + """ + + else if (u is Framework.ConnectorTypeSelector) + """ + \(k): Optional[ConnectorAndConnection] = None + _validate_\(k) = validator("\(k)", pre=True, allow_reuse=True)(validate_connector_and_connection) + """ + else if (u is Framework.ConnectionCreator) + """ + \(k): Optional[Connection] = None + _validate_\(k) = validator("\(k)", pre=True, allow_reuse=True)(validate_connection) + """ + else if (u is Framework.BooleanInput) + """ + \(k): bool = None + """ + else if (u is Framework.NumericInput) + """ + \(k): Optional[Union[int,float]] = None + """ + else if (u is Framework.DateInput) + """ + \(k): Optional[datetime] = None + """ + else + """ + \(k): str + """ + } + """ + + class RuntimeConfig(BaseSettings): + user_id:Optional[str] = Field(default="") + agent:Optional[str] = Field(default="") + agent_id:Optional[str] = Field(default="") + agent_pkg:Optional[str] = Field(default="") + agent_wfl:Optional[str] = Field(default="") + custom_config:Optional[CustomConfig] = None + + class Config: + fields = { + 'user_id': { + ENV: 'ATLAN_USER_ID', + }, + 'agent': { + ENV: 'X_ATLAN_AGENT' + }, + 'agent_id': { + ENV: 'X_ATLAN_AGENT_ID' + }, + 'agent_pkg': { + ENV: 'X_ATLAN_AGENT_PACKAGE_NAME' + }, + 'agent_wfl': { + ENV: 'X_ATLAN_AGENT_WORKFLOW_ID' + }, + 'custom_config': { + ENV: 'NESTED_CONFIG' + } + } + @classmethod + def parse_env_var(cls, field_name:str, raw_value:str)->Any: + if field_name == 'custom_config': + return CustomConfig.parse_raw(raw_value) + return cls.json_loads(raw_value) + + @property + def envars_as_dict(self) -> Dict[str, Any]: + \"\"\" + :return dict: returns a dict of the environment variable names and values consumed by this RuntimeConfig. + the name of an environment variable will be the key and the value will be the value. This is provided + to facilitate testing + \"\"\" + ret_val: Dict[str, Any] = {} + for key, value in RuntimeConfig.Config.fields.items(): + if field_value := getattr(self, key): + ret_val[value["env"]] = field_value.json() + return ret_val + """ + }.join("\n") +} + +local class NamePathS3Tuple extends NamedPair { + hidden inputName: String + name: String = "\(inputName)_s3" + path: String = "/tmp/\(inputName)/{{inputs.parameters.\(inputName)}}" + s3: Mapping = new Mapping { + ["key"] = "{{inputs.parameters.\(inputName)}}" + } +} + +class ConfigMapEntry extends NamedPair { + name: String + fixed valueFrom = (value) { + when (default != null) { + ["default"] = default + } + } + hidden configMapName: String + hidden configMapKey: String + hidden default: String? = null + hidden optional: Boolean? = null + hidden fixed value = new Mapping { + ["configMapKeyRef"] = new Mapping { + ["name"] = configMapName + ["key"] = configMapKey + when (optional != null) { + ["optional"] = optional + } + } + } +} + +class NamedSecret extends NamedPair { + name: String + fixed valueFrom = new Mapping { + ["secretKeyRef"] = new Mapping { + ["name"] = secretName + ["key"] = secretKey + when (optional != null) { + ["optional"] = optional + } + } + } + hidden secretName: String + hidden secretKey: String + hidden optional: Boolean? = null +} + +local class NestedConfig extends NamedPair { + name: String = "NESTED_CONFIG" + fixed value: String = new Listing { + when (passthrough != null) { + if (passthrough.isEmpty) + "null" + else + "{" + new Listing { + for (k, v in passthrough) { + "\"\(k)\": \"\(v)\"" + } + }.join(",\n") + "}" + } + when (inputs != null) { + if (inputs.isEmpty) + "null" + else + "{" + new Listing { + for (k, ui in inputs) { + if (ui is Framework.FileUploader || ui is Framework.FileCopier) + "\"\(k)\": \"/tmp/\(k)/{{inputs.parameters.\(k)}}\"" + else if (ui is Framework.NumericInput || ui is Framework.DateInput || ui is Framework.BooleanInput) + "\"\(k)\": {{inputs.parameters.\(k)}}" + else + "\"\(k)\": {{=toJson(inputs.parameters.\(k))}}" + } + }.join(",\n") + "}" + } + }.join("\n") + hidden inputs: Map? + hidden passthrough: Map? +} + +class NameValueBoolPair extends NamedPair { + name: String + value: Boolean +} + +class NameValuePair extends NamedPair { + name: String + value: String +} + +class NamePathPair extends NamedPair { + name: String + path: String +} + +class VolumeMount extends NamedPair { + name: String + mountPath: String +} + +open class WorkflowVolume extends NamedPair { + name: String +} + +class EmptyDirVolume extends WorkflowVolume { + name: String + fixed emptyDir: Map = new Mapping {}.toMap() +} + +abstract class NamedPair { + name: String +} + +/// Used to render the appropriately-wrapped output file for the UI portion of a package. +local open class ConfigMap { + fixed apiVersion = "v1" + fixed kind = "ConfigMap" + fixed metadata: Mapping = new Mapping { + ["name"] = name + } + fixed data: Mapping = new Mapping { + ["config"] = new JsonRenderer {}.renderValue(config) + } + hidden name: String + hidden config: Framework.UIConfig +} + +local class PackageDefinition { + hidden packageId: String + hidden packageName: String + fixed name: String = packageId + version: String + description: String + keywords: Listing = new Listing {} + hidden iconUrl: String + hidden docsUrl: String + hidden allowSchedule: Boolean = true + hidden certified: Boolean = true + hidden preview: Boolean = false + hidden connectorType: Connectors.Type? + hidden category: String = "custom" + fixed homepage: String = "https://packages.atlan.com/-/web/detail/\(packageId)" + fixed main: String = "index.js" + fixed scripts: Map = Map() + fixed author: Map = new Mapping { + ["name"] = "Atlan CSA" + ["email"] = "csa@atlan.com" + ["url"] = "https://atlan.com" + }.toMap() + fixed repository: Map = new Mapping { + ["type"] = "git" + ["url"] = "https://github.com/atlanhq/marketplace-packages.git" + }.toMap() + fixed license: String = "MIT" + fixed bugs: Map = new Mapping { + ["url"] = "https://atlan.com" + ["email"] = "support@atlan.com" + }.toMap() + fixed config: PackageConfig = new PackageConfig { + labels = new Mapping { + ["orchestration.atlan.com/verified"] = "true" + ["orchestration.atlan.com/type"] = category + ["orchestration.atlan.com/source"] = connectorType?.value ?? "atlan" + ["orchestration.atlan.com/sourceCategory"] = connectorType?.category ?? "utility" + ["orchestration.atlan.com/certified"] = certified.toString() + ["orchestration.atlan.com/preview"] = preview.toString() + } + annotations = new Mapping { + ["orchestration.atlan.com/name"] = packageName + ["orchestration.atlan.com/allowSchedule"] = allowSchedule.toString() + ["orchestration.atlan.com/dependentPackage"] = "" + ["orchestration.atlan.com/emoji"] = "🚀" + ["orchestration.atlan.com/categories"] = keywords.join(",") + ["orchestration.atlan.com/icon"] = iconUrl + ["orchestration.atlan.com/logo"] = iconUrl + ["orchestration.atlan.com/docsUrl"] = docsUrl + } + } +} + +local class PackageConfig { + labels: Mapping + annotations: Mapping +} + +/// Used to render a standard spec in a workflow template, irrespective of input files and / or separate publish step. +local class WorkflowTemplateSpec { + hidden package: WorkflowTemplateDefinition + + hidden passthrough: List = new Listing { + for (k, u in package.inputs.config.properties) { + when (u is Framework.FileUploader) { + new NameValuePair { + name = k + value = "{{inputs.parameters.\(k)_key}}" + } + } else { + new NameValuePair { + name = k + value = "{{inputs.parameters.\(k)}}" + } + } + } + }.toList() + + hidden fixed fileMoves: Listing = new Listing { + for (k, u in package.inputs.config.properties) { + when (u is Framework.FileUploader) { + new TaskDefinition { + name = "move-\(getSafeTaskName(k))" + templateRef = new TemplateRef { + name = "atlan-workflow-helpers" + template = "move-artifact-to-s3" + } + condition = "'{{inputs.parameters.cloud_provider}}' == 'azure' && {{inputs.parameters.is_azure_artifacts}} == false && '{{inputs.parameters.\(k)_id}}' != ''" + arguments = new Arguments { + parameters = new Listing { + new NameValuePair { + name = "file-id" + value = "{{inputs.parameters.\(k)_id}}" + } + new NameValuePair { + name = "s3-file-key" + value = "{{inputs.parameters.\(k)_key}}" + } + }.toList() + } + } + } + } + } + + hidden fixed move: WorkflowTemplateDefinition? = + if (!fileMoves.isEmpty) + new WorkflowTemplateDefinition { + cfg = package.cfg + dag = new WorkflowDag { + tasks = fileMoves.toList() + } + name = "move" + pkg = package.pkg + f = true + } + else null + + fixed entrypoint = "main" + templates: Listing = new Listing { + new WorkflowTemplateDefinition { + name = "main" + dag = new WorkflowDag { + tasks = new Listing { + when (move != null) { + new TaskDefinition { + name = "move" + template = "move" + arguments = new Arguments { + parameters = passthrough + } + } + } + new TaskDefinition { + name = "process" + dependencies = new Listing { + when (move != null) { "move" } + }.toList() + template = "process" + arguments = new Arguments { + parameters = passthrough + } + } + when (package.pub != null) { + new TaskDefinition { + name = "load" + dependencies = List("process") + template = "load" + arguments = new Arguments { + parameters = passthrough + } + } + } + }.toList() + } + cfg = package.cfg + pkg = package.pkg + } + (package) { name = "process" } + when (move != null) { move } + when (package.pub != null) { + getProcessTemplate( + "load", + package.cfg, + package.pub, + package.pkg, + package.pub.outputs, + package.pub.containerImage, + package.pub.containerImagePullPolicy, + package.pub.command, + package.pub.args, + package.pub.parameters, // Always passthrough non-UI parameters + false, // Always passthrough direct input files (from previous step(s)) + package.pub.inputArtifacts.toList() + ) + } + } +} + +/// Used to render the appropriately-wrapped output file for the Argo hand-over portion of a package. +local class WorkflowTemplate { + fixed apiVersion = "argoproj.io/v1alpha1" + fixed kind = "WorkflowTemplate" + fixed metadata = new Mapping { + ["name"] = name + } + fixed spec = new WorkflowTemplateSpec { + package = template + } + hidden name: String + hidden template: WorkflowTemplateDefinition +} + +local class WorkflowTemplateDefinition { + name: String = "main" + fixed inputs: WorkflowInputs = new WorkflowInputs { + config = cfg + pkgName = pkg + fileInputSetup = f + credentialInjection = (credVariable != null) + passthroughFiles = directInputFiles + } + outputs: Framework.WorkflowOutputs? + volumes: Listing? + container: WorkflowContainer? + initContainers: Listing? + dag: WorkflowDag? + hidden cfg: Framework.UIConfig + hidden pub: Framework.PublishConfig? + hidden pkg: String = "" + hidden f: Boolean = false + hidden credVariable: String? = null + hidden directInputFiles: List? = null +} + +local class WorkflowInputs { + fixed parameters: List = params.toList() + fixed artifacts: List? = if (fileInputSetup) null else arts.toList() + hidden config: Framework.UIConfig + hidden pkgName: String = "" + hidden fileInputSetup: Boolean = false + hidden credentialInjection: Boolean = false + hidden passthroughFiles: List? = null + + hidden arts: Listing = new Listing { + for (k, u in config.properties) { + when (u is Framework.FileUploader || u is Framework.FileCopier) { + new NamePathS3Tuple { + inputName = k + } + } + } + when (passthroughFiles != null) { + ...passthroughFiles + } + } + hidden params: Listing = new Listing { + when (!pkgName.isEmpty) { + new NameValuePair { + name = S3_CONFIG_PREFIX + value = pkgName + } + when (fileInputSetup) { + new ConfigMapEntry { + name = "is_azure_artifacts" + configMapName = "atlan-defaults" + configMapKey = "azure_artifacts" + default = "false" + } + new ConfigMapEntry { + name = "cloud_provider" + configMapName = "atlan-defaults" + configMapKey = "cloud" + default = "aws" + } + } + } + for (k, u in config.properties) { + when (u is Framework.FileUploader || u is Framework.FileCopier) { + when (fileInputSetup) { + new NameValuePair { + name = k + value = "{}" + } + new NameValuePair { + name = "\(k)_key" + value = "{{= sprig.dig('fileKey', '\(DEFAULT_FILE)', sprig.mustFromJson(inputs.parameters.\(k))) }}" + } + new NameValuePair { + name = "\(k)_id" + value = "{{= sprig.dig('fileId', '', sprig.mustFromJson(inputs.parameters.\(k))) }}" + } + } else { + new NameValuePair { + name = k + value = DEFAULT_FILE + } + } + } else { + when (u is Framework.BooleanInput) { + new NameValueBoolPair { + name = k + value = if (u.fallback != null) getFallbackParam(u) as Boolean else u.ui.default as Boolean + } + } else { + when (u is Framework.NumericInput || u is Framework.DateInput) { + new NameValuePair { + name = k + value = if (u.fallback != null) getFallbackParam(u).toString() else "-1" + } + } else { + new NameValuePair { + name = k + value = if (u.fallback != null) getFallbackParam(u).toString() else "" + } + } + } + } + } + } +} + +local class WorkflowDag { + tasks: List +} + +local class TaskDefinition { + name: String + depends: String? + dependencies: List? + template: String? + templateRef: TemplateRef? + fixed `when` = condition + arguments: Arguments + hidden condition: String? +} + +local class TemplateRef { + name: String + template: String +} + +local class Arguments { + parameters: List +} + +local class WorkflowContainer { + name: String? + volumeMounts: Listing? + mirrorVolumeMounts: Boolean? + image: String + imagePullPolicy: Framework.ImagePullPolicy = "IfNotPresent" + command: List + args: List = List() + hidden passthroughParams: Map? + hidden config: Framework.UIConfig + hidden defaultVars: List = new Listing { + new NameValuePair { + name = "ATLAN_BASE_URL" + value = "INTERNAL" + } + new NameValuePair { + name = "ATLAN_USER_ID" + value = "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}" + } + new NameValuePair { + name = "X_ATLAN_AGENT" + value = "workflow" + } + new NameValuePair { + name = "X_ATLAN_AGENT_ID" + value = "{{workflow.name}}" + } + new NameValuePair { + name = "X_ATLAN_AGENT_PACKAGE_NAME" + value = "{{=sprig.dig('annotations', 'package', 'argoproj', 'io/name', '', workflow)}}" + } + new NameValuePair { + name = "X_ATLAN_AGENT_WORKFLOW_ID" + value = "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/workflow-template', '', workflow)}}" + } + new NameValuePair { + name = "AZURE_STORAGE_CONTAINER_NAME" + value = "objectstore" + } + new ConfigMapEntry { + name = "CLOUD_PROVIDER" + configMapName = "atlan-defaults" + configMapKey = "cloud" + default = "aws" + } + new ConfigMapEntry { + name = "AWS_S3_BUCKET_NAME" + configMapName = "atlan-defaults" + configMapKey = "bucket" + optional = true + } + new ConfigMapEntry { + name = "AWS_S3_REGION" + configMapName = "atlan-defaults" + configMapKey = "region" + optional = true + } + new ConfigMapEntry { + name = "GCP_STORAGE_BUCKET" + configMapName = "atlan-defaults" + configMapKey = "bucket" + optional = true + } + new NamedSecret { + name = "GCP_PROJECT_ID" + secretName = "bucket-details-secret" + secretKey = "GCP_PROJECT_ID" + optional = true + } + new NamedSecret { + name = "AZURE_STORAGE_ACCESS_KEY" + secretName = "azurestorage" + secretKey = "AZURE_STORAGE_ACCESS_KEY" + optional = true + } + new NamedSecret { + name = "AZURE_STORAGE_ACCOUNT" + secretName = "azurestorage" + secretKey = "AZURE_STORAGE_ACCOUNT" + optional = true + } + new NamedSecret { + name = "CLIENT_ID" + secretName = "argo-client-creds" + secretKey = "login" + } + new NamedSecret { + name = "CLIENT_SECRET" + secretName = "argo-client-creds" + secretKey = "password" + } + new NamedSecret { + name = "SMTP_HOST" + secretName = "support-smtp-creds" + secretKey = "host" + } + new NamedSecret { + name = "SMTP_PORT" + secretName = "support-smtp-creds" + secretKey = "port" + } + new NamedSecret { + name = "SMTP_FROM" + secretName = "support-smtp-creds" + secretKey = "from" + } + new NamedSecret { + name = "SMTP_USER" + secretName = "support-smtp-creds" + secretKey = "login" + } + new NamedSecret { + name = "SMTP_PASS" + secretName = "workflow-parameter-store" + secretKey = "smtp_password" + } + new ConfigMapEntry { + name = "DOMAIN" + configMapName = "atlan-defaults" + configMapKey = "domain" + } + }.toList() + hidden configVars: List = config.properties.fold(List(), (acc: List, key, property) -> + acc.add(resolvePropertyToVar(key, property)) + ) + hidden nestedConfig: NestedConfig = + if (passthroughParams != null) + new NestedConfig { + passthrough = passthroughParams + } + else + new NestedConfig { + inputs = config.properties.fold(new Mapping {}, (acc: Mapping, key, property) -> + (acc) { + [key] = property + } + ).toMap() + } + env: List = defaultVars + configVars + List(nestedConfig) +} + +/// Used to render the configmap for a new connector type. +local class ConnectorConfigMap extends ConfigMap { + fixed metadata: Mapping = new Mapping { + ["name"] = name + ["labels"] = new Mapping { + ["workflows.argoproj.io/configmap-type"] = "Parameter" + ["orchestration.atlan.com/version"] = "1" + ["orchestration.atlan.com/source"] = config.source.replaceAll(" ", "") + } + } + fixed data: Mapping = new Mapping { + ["icon"] = config.icon + ["helpdeskLink"] = config.helpdesk + ["logo"] = config.logo + ["connector"] = config.source + ["defaultConnectorType"] = config.connectorType + ["jdbcCredentialTemplate"] = config.jdbcCredential + ["restCredentialTemplate"] = config.restCredential + ["odbcCredentialTemplate"] = config.odbcCredential + ["grpcCredentialTemplate"] = config.grpcCredential + ["restMetadataTemplate"] = config.restMetadata + ["restMetadataOutputTransformerTemplate"] = config.restTransformer + when (config.sage != null) { ["sageTemplate"] = config.sage } + when (config.soda != null) { ["sodaConnectionTemplate"] = config.soda } + ["config"] = new JsonRenderer {}.renderValue(config) + } + hidden name: String + hidden config: Credential +} + +const function resolvePropertyToVar(key: String, property: Framework.UIElement): NameValuePair = new NameValuePair { + name = key.toUpperCase() + value = if (property is Framework.FileUploader || property is Framework.FileCopier) + new NamePathS3Tuple { inputName = key }.path + else + "{{inputs.parameters.\(key)}}" +} + +const function getSafeTaskName(name: String): String = name.replaceAll("_", "-") +const DEFAULT_FILE = "argo-artifacts/atlan-update/@atlan-packages-last-safe-run.txt" +const S3_CONFIG_PREFIX = "output_prefix" diff --git a/package-toolkit/testing/src/main/kotlin/com/atlan/pkg/PackageTest.kt b/package-toolkit/testing/src/main/kotlin/com/atlan/pkg/PackageTest.kt index 31b72a0a6f..ad587207a9 100644 --- a/package-toolkit/testing/src/main/kotlin/com/atlan/pkg/PackageTest.kt +++ b/package-toolkit/testing/src/main/kotlin/com/atlan/pkg/PackageTest.kt @@ -140,6 +140,54 @@ abstract class PackageTest( return false } + /** + * Check whether the provided line appears in the specified file. + * + * @param filename for the file + * @param line the line to check the presence of + * @param relativeTo (optional) path under which the log file should be present + */ + fun fileHasLine( + filename: String, + line: String, + relativeTo: String = testDirectory, + ) { + val file = validateFile(filename, relativeTo) + file.useLines { lines -> + lines.forEach { candidate -> + if (candidate == line) { + // short-circuit + return + } + } + } + assertEquals("Transformed file does not contain expected details.", line) + } + + /** + * Check whether the provided line appears as the start of any line in the specified file. + * + * @param filename for the file + * @param line the line to check any line in the file starts with + * @param relativeTo (optional) path under which the log file should be present + */ + fun fileHasLineStartingWith( + filename: String, + line: String, + relativeTo: String = testDirectory, + ) { + val file = validateFile(filename, relativeTo) + file.useLines { lines -> + lines.forEach { candidate -> + if (candidate.startsWith(line)) { + // short-circuit + return + } + } + } + assertEquals("Transformed file does not contain any line starting with expected details.", line) + } + /** * Validate (through assertions) that these files exist and are non-empty files. * @@ -160,7 +208,7 @@ abstract class PackageTest( * Validate (through assertions) that these files exist, but are empty. * * @param files list of filenames - * @param relativeTo (optional) path under whic hthe files should be present + * @param relativeTo (optional) path under which the files should be present */ fun validateFileExistsButEmpty( files: List, diff --git a/samples/packages/enrichment-migrator/src/main/kotlin/EnrichmentMigrator.kt b/samples/packages/enrichment-migrator/src/main/kotlin/EnrichmentMigrator.kt index 67b5e93918..88565edb97 100644 --- a/samples/packages/enrichment-migrator/src/main/kotlin/EnrichmentMigrator.kt +++ b/samples/packages/enrichment-migrator/src/main/kotlin/EnrichmentMigrator.kt @@ -10,9 +10,13 @@ import com.atlan.model.assets.Connection import com.atlan.model.assets.Database import com.atlan.model.fields.CustomMetadataField import com.atlan.pkg.Utils -import com.atlan.pkg.aim.Importer import com.atlan.pkg.serde.RowSerde +import de.siegmar.fastcsv.writer.CsvWriter +import de.siegmar.fastcsv.writer.LineDelimiter +import de.siegmar.fastcsv.writer.QuoteStrategies import java.io.File +import java.nio.file.Paths +import java.nio.file.StandardOpenOption import kotlin.jvm.optionals.getOrElse /** @@ -94,52 +98,46 @@ object EnrichmentMigrator { } start.toList() } - ctx.config.targetConnection.forEach { targetConnectionQN -> - val targetDatabaseNames = getTargetDatabaseName(ctx.client, targetConnectionQN, ctx.config.targetDatabasePattern) - targetDatabaseNames.forEach { targetDatabaseName -> - val mCtx = - MigratorContext( - sourceConnectionQN = sourceConnectionQN, - targetConnectionQN = targetConnectionQN, - targetConnectionName = getConnectionName(ctx.client, targetConnectionQN), - includeArchived = ctx.config.includeArchived, - sourceDatabaseName = sourceDatabaseName, - targetDatabaseName = targetDatabaseName, - ) - val targetConnectionFilename = - if (targetDatabaseName.isNotBlank()) { - "${targetConnectionQN}_$targetDatabaseName".replace("/", "_") - } else { - targetConnectionQN.replace("/", "_") - } - val transformedFile = - "$outputDirectory${File.separator}CSA_EM_transformed_$targetConnectionFilename.csv" - val transformer = - Transformer( - mCtx, - extractFile, - header.toList(), - logger, - ctx.config.fieldSeparator[0], - ) - transformer.transform(transformedFile) - // 3. Import the transformed file - val importConfig = - AssetImportCfg( - assetsFile = transformedFile, - assetsUpsertSemantic = "update", - assetsFailOnErrors = ctx.config.failOnErrors, - assetsBatchSize = ctx.config.batchSize, - assetsFieldSeparator = ctx.config.fieldSeparator, - assetsCaseSensitive = ctx.config.caseSensitive, - assetsTableViewAgnostic = ctx.config.tableViewAgnostic, - ) - Utils.initializeContext(importConfig, ctx).use { iCtx -> - Importer.import(iCtx, outputDirectory)?.close() + val transformedFile = "$outputDirectory${File.separator}transformed-file.csv" + CsvWriter + .builder() + .fieldSeparator(ctx.config.fieldSeparator[0]) + .quoteCharacter('"') + .quoteStrategy(QuoteStrategies.NON_EMPTY) + .lineDelimiter(LineDelimiter.PLATFORM) + .build( + Paths.get(transformedFile), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE, + ).use { writer -> + writer.writeRecord(header.toList()) + + ctx.config.targetConnection.forEach { targetConnectionQN -> + val targetDatabaseNames = getTargetDatabaseName(ctx.client, targetConnectionQN, ctx.config.targetDatabasePattern) + targetDatabaseNames.forEach { targetDatabaseName -> + val mCtx = + MigratorContext( + sourceConnectionQN = sourceConnectionQN, + targetConnectionQN = targetConnectionQN, + targetConnectionName = getConnectionName(ctx.client, targetConnectionQN), + includeArchived = ctx.config.includeArchived, + sourceDatabaseName = sourceDatabaseName, + targetDatabaseName = targetDatabaseName, + ) + val transformer = + Transformer( + mCtx, + extractFile, + header.toList(), + logger, + ctx.config.fieldSeparator[0], + ) + transformer.transform(writer) + } } } - } } } diff --git a/samples/packages/enrichment-migrator/src/main/resources/package.pkl b/samples/packages/enrichment-migrator/src/main/resources/package.pkl index 26e769321d..083775bbbc 100644 --- a/samples/packages/enrichment-migrator/src/main/resources/package.pkl +++ b/samples/packages/enrichment-migrator/src/main/resources/package.pkl @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: Apache-2.0 Copyright 2024 Atlan Pte. Ltd. */ -amends "modulepath:/Config.pkl" +amends "modulepath:/Framework.pkl" import "pkl:semver" packageId = "@csa/enrichment-migrator" @@ -21,6 +21,7 @@ containerCommand { outputs { files { ["debug-logs"] = "/tmp/debug.log" + ["transformed_file"] = "/tmp/transformed-file.csv" } } keywords { @@ -29,6 +30,19 @@ keywords { } preview = true +publishConfig = new AssetImport { + versionTag = version.toString() + assetsFile = transferFile(outputs, "transformed_file", "assets.csv") + assetsUpsertSemantic = "update" + assetsConfig = transferConfigInput(uiConfig, "config_type") + assetsFailOnErrors = transferConfigInput(uiConfig, "fail_on_errors") + assetsCaseSensitive = transferConfigInput(uiConfig, "case_sensitive") + assetsTableViewAgnostic = transferConfigInput(uiConfig, "table_view_agnostic") + assetsFieldSeparator = transferConfigInput(uiConfig, "field_separator") + assetsBatchSize = transferConfigInput(uiConfig, "batch_size") + trackBatches = true +} + uiConfig { tasks { ["Assets"] { diff --git a/samples/packages/enrichment-migrator/src/main/resources/tmp/src/main/kotlin/EnrichmentMigratorCfg.kt b/samples/packages/enrichment-migrator/src/main/resources/tmp/src/main/kotlin/EnrichmentMigratorCfg.kt deleted file mode 100644 index 234a6b3bea..0000000000 --- a/samples/packages/enrichment-migrator/src/main/resources/tmp/src/main/kotlin/EnrichmentMigratorCfg.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2024 Atlan Pte. Ltd. */ -import com.atlan.pkg.CustomConfig -import com.atlan.pkg.serde.WidgetSerde -import com.fasterxml.jackson.annotation.JsonAutoDetect -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.annotation.JsonDeserialize -import com.fasterxml.jackson.databind.annotation.JsonSerialize -import javax.annotation.processing.Generated - -/** - * Expected configuration for the Enrichment Migrator custom package. - */ -@Generated("com.atlan.pkg.CustomPackage") -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) -data class EnrichmentMigratorCfg( - @JsonDeserialize(using = WidgetSerde.MultiSelectDeserializer::class) - @JsonSerialize(using = WidgetSerde.MultiSelectSerializer::class) - @JsonProperty("source_connection") val sourceConnection: List? = null, - @JsonProperty("source_qn_prefix") val sourceQnPrefix: String? = null, - @JsonProperty("target_database_pattern") val targetDatabasePattern: String? = null, - @JsonDeserialize(using = WidgetSerde.MultiSelectDeserializer::class) - @JsonSerialize(using = WidgetSerde.MultiSelectSerializer::class) - @JsonProperty("target_connection") val targetConnection: List? = null, - @JsonProperty("include_archived") val includeArchived: Boolean? = null, - @JsonProperty("config_type") val configType: String? = null, - @JsonProperty("fail_on_errors") val failOnErrors: Boolean? = null, - @JsonProperty("field_separator") val fieldSeparator: String? = null, - @JsonProperty("batch_size") val batchSize: Number? = null, - @JsonProperty("limit_type") val limitType: String? = null, - @JsonDeserialize(using = WidgetSerde.MultiSelectDeserializer::class) - @JsonSerialize(using = WidgetSerde.MultiSelectSerializer::class) - @JsonProperty("attributes_list") val attributesList: List? = null, - @JsonProperty("cm_limit_type") val cmLimitType: String? = null, - @JsonProperty("custom_metadata") val customMetadata: String? = null, -) : CustomConfig() diff --git a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorArchivedTest.kt b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorArchivedTest.kt index 4f74f6bffc..d5bf2840f4 100644 --- a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorArchivedTest.kt +++ b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorArchivedTest.kt @@ -10,7 +10,9 @@ import com.atlan.model.enums.AtlanStatus import com.atlan.model.search.IndexSearchResponse import com.atlan.pkg.PackageTest import com.atlan.pkg.Utils +import com.atlan.pkg.aim.Importer import com.atlan.util.AssetBatch +import java.io.File import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -27,6 +29,7 @@ class EnrichmentMigratorArchivedTest : PackageTest("a") { private val files = listOf( "asset-export.csv", + "transformed-file.csv", "debug.log", ) @@ -85,7 +88,13 @@ class EnrichmentMigratorArchivedTest : PackageTest("a") { ), EnrichmentMigrator::main, ) - Thread.sleep(15000) + runCustomPackage( + AssetImportCfg( + assetsFile = "$testDirectory${File.separator}transformed-file.csv", + assetsUpsertSemantic = "update", + ), + Importer::main, + ) } override fun teardown() { @@ -93,7 +102,19 @@ class EnrichmentMigratorArchivedTest : PackageTest("a") { } @Test - fun activeAssetMigrated() { + fun activeAssetInFile() { + val targetConnection = Connection.findByName(client, c1, connectorType)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db1/sch1/tbl1","Table","ACTIVE","tbl1",,,"Must have some enrichment to be included!" + """.trimIndent(), + ) + } + + @Test + fun activeAsset() { val targetConnection = Connection.findByName(client, c1, connectorType)[0]!! val request = Table @@ -122,9 +143,6 @@ class EnrichmentMigratorArchivedTest : PackageTest("a") { @Test fun filesCreated() { validateFilesExist(files) - val targetConnection = Connection.findByName(client, c1, connectorType)[0]!! - val filename = targetConnection.qualifiedName.replace("/", "_") - validateFilesExist(listOf("CSA_EM_transformed_$filename.csv")) } @Test diff --git a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorMultipleTargetTest.kt b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorMultipleTargetTest.kt index d21b6b93c7..8edba72d8e 100644 --- a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorMultipleTargetTest.kt +++ b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorMultipleTargetTest.kt @@ -7,7 +7,9 @@ import com.atlan.model.assets.Table import com.atlan.model.enums.AtlanConnectorType import com.atlan.pkg.PackageTest import com.atlan.pkg.Utils +import com.atlan.pkg.aim.Importer import com.atlan.util.AssetBatch +import java.io.File import kotlin.test.Test import kotlin.test.assertEquals @@ -27,6 +29,7 @@ class EnrichmentMigratorMultipleTargetTest : PackageTest("mt") { private val files = listOf( "asset-export.csv", + "transformed-file.csv", "debug.log", ) @@ -73,7 +76,6 @@ class EnrichmentMigratorMultipleTargetTest : PackageTest("mt") { override fun setup() { createConnections() createAssets() - Thread.sleep(15000) runCustomPackage( EnrichmentMigratorCfg( sourceConnection = listOf(Connection.findByName(client, c1, c1Type)?.get(0)?.qualifiedName!!), @@ -88,7 +90,13 @@ class EnrichmentMigratorMultipleTargetTest : PackageTest("mt") { ), EnrichmentMigrator::main, ) - Thread.sleep(15000) + runCustomPackage( + AssetImportCfg( + assetsFile = "$testDirectory${File.separator}transformed-file.csv", + assetsUpsertSemantic = "update", + ), + Importer::main, + ) } override fun teardown() { @@ -98,7 +106,19 @@ class EnrichmentMigratorMultipleTargetTest : PackageTest("mt") { } @Test - fun datesOnTarget() { + fun descriptionInFileOnTarget2() { + val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db1/sch1/tbl1","Table","tbl1","Some description." + """.trimIndent(), + ) + } + + @Test + fun descriptionOnTarget2() { val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! Table .select(client) @@ -110,14 +130,34 @@ class EnrichmentMigratorMultipleTargetTest : PackageTest("mt") { } } + @Test + fun descriptionInFileOnTarget3() { + val targetConnection = Connection.findByName(client, c3, c3Type)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db1/sch1/tbl1","Table","tbl1","Some description." + """.trimIndent(), + ) + } + + @Test + fun descriptionOnTarget3() { + val targetConnection = Connection.findByName(client, c3, c3Type)[0]!! + Table + .select(client) + .where(Table.QUALIFIED_NAME.startsWith(targetConnection.qualifiedName)) + .includeOnResults(Table.DESCRIPTION) + .stream() + .forEach { + assertEquals("Some description.", it.description) + } + } + @Test fun filesCreated() { validateFilesExist(files) - val t1 = Connection.findByName(client, c2, c2Type)[0]!! - val t2 = Connection.findByName(client, c3, c3Type)[0]!! - val f1 = t1.qualifiedName.replace("/", "_") - val f2 = t2.qualifiedName.replace("/", "_") - validateFilesExist(listOf("CSA_EM_transformed_$f1.csv", "CSA_EM_transformed_$f2.csv")) } @Test diff --git a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorPatternTest.kt b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorPatternTest.kt index cdac5324e3..8a4c362ffa 100644 --- a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorPatternTest.kt +++ b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorPatternTest.kt @@ -9,7 +9,9 @@ import com.atlan.model.assets.Table import com.atlan.model.enums.AtlanConnectorType import com.atlan.pkg.PackageTest import com.atlan.pkg.Utils +import com.atlan.pkg.aim.Importer import com.atlan.util.AssetBatch +import java.io.File import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith @@ -34,6 +36,7 @@ class EnrichmentMigratorPatternTest : PackageTest("p") { private val files = listOf( "asset-export.csv", + "transformed-file.csv", "debug.log", ) @@ -94,7 +97,13 @@ class EnrichmentMigratorPatternTest : PackageTest("p") { ), EnrichmentMigrator::main, ) - Thread.sleep(15000) + runCustomPackage( + AssetImportCfg( + assetsFile = "$testDirectory${File.separator}transformed-file.csv", + assetsUpsertSemantic = "update", + ), + Importer::main, + ) } override fun teardown() { @@ -225,12 +234,6 @@ class EnrichmentMigratorPatternTest : PackageTest("p") { @Test fun filesCreated() { validateFilesExist(files) - validateFilesExist( - listOf( - "CSA_EM_transformed_${this.targetConnectionQualifiedName}_$targetDbName1.csv".replace("/", "_"), - "CSA_EM_transformed_${this.targetConnectionQualifiedName}_$targetDbName2.csv".replace("/", "_"), - ), - ) } @Test @@ -251,6 +254,30 @@ class EnrichmentMigratorPatternTest : PackageTest("p") { } } + @Test + fun userDescriptionInFileForTarget2() { + val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db_test02/sch1/tbl1","Table","tbl1",,,"Some user description" + """.trimIndent(), + ) + } + + @Test + fun userDescriptionInFileForTarget3() { + val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db_test03/sch1/tbl1","Table","tbl1",,,"Some user description" + """.trimIndent(), + ) + } + @Test fun errorFreeLog() { validateErrorFreeLog() diff --git a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorSingleTargetTest.kt b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorSingleTargetTest.kt index 1e06355053..634e854d98 100644 --- a/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorSingleTargetTest.kt +++ b/samples/packages/enrichment-migrator/src/test/kotlin/EnrichmentMigratorSingleTargetTest.kt @@ -12,7 +12,10 @@ import com.atlan.model.typedefs.AttributeDef import com.atlan.model.typedefs.CustomMetadataDef import com.atlan.pkg.PackageTest import com.atlan.pkg.Utils +import com.atlan.pkg.aim.Importer +import com.atlan.pkg.serde.cell.TimestampXformer import com.atlan.util.AssetBatch +import java.io.File import java.time.Instant import kotlin.test.Test import kotlin.test.assertEquals @@ -34,6 +37,7 @@ class EnrichmentMigratorSingleTargetTest : PackageTest("st") { private val files = listOf( "asset-export.csv", + "transformed-file.csv", "debug.log", ) @@ -96,7 +100,13 @@ class EnrichmentMigratorSingleTargetTest : PackageTest("st") { ), EnrichmentMigrator::main, ) - Thread.sleep(15000) + runCustomPackage( + AssetImportCfg( + assetsFile = "$testDirectory${File.separator}transformed-file.csv", + assetsUpsertSemantic = "update", + ), + Importer::main, + ) } override fun teardown() { @@ -129,12 +139,21 @@ class EnrichmentMigratorSingleTargetTest : PackageTest("st") { } } + @Test + fun customMetadataInFile() { + val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! + fileHasLineStartingWith( + filename = "transformed-file.csv", + line = + """ + "${targetConnection.qualifiedName}/db1/sch1/tbl1","Table","tbl1",,,,,,,,,,,,,,,,"${TimestampXformer.encode(now)}" + """.trimIndent(), + ) + } + @Test fun filesCreated() { validateFilesExist(files) - val targetConnection = Connection.findByName(client, c2, c2Type)[0]!! - val filename = targetConnection.qualifiedName.replace("/", "_") - validateFilesExist(listOf("CSA_EM_transformed_$filename.csv")) } @Test