From df6b6894438d187b8dd9a5c0a084f036a5415787 Mon Sep 17 00:00:00 2001 From: Christopher Grote Date: Tue, 28 Jan 2025 20:53:12 +0000 Subject: [PATCH] Replace importers with xformers Signed-off-by: Christopher Grote --- .../kotlin/com/atlan/pkg/rab/AssetImporter.kt | 149 ------------------ .../com/atlan/pkg/rab/ColumnImporter.kt | 67 -------- .../com/atlan/pkg/rab/ConnectionImporter.kt | 30 +++- .../com/atlan/pkg/rab/DatabaseImporter.kt | 49 ------ .../atlan/pkg/rab/MaterializedViewImporter.kt | 50 ------ .../com/atlan/pkg/rab/SchemaImporter.kt | 51 ------ .../kotlin/com/atlan/pkg/rab/TableImporter.kt | 50 ------ .../kotlin/com/atlan/pkg/rab/ViewImporter.kt | 50 ------ .../kotlin/com/atlan/pkg/rab/DeltaTest.kt | 149 ------------------ 9 files changed, 25 insertions(+), 620 deletions(-) delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/AssetImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ColumnImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/DatabaseImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/MaterializedViewImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/SchemaImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/TableImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ViewImporter.kt delete mode 100644 samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/DeltaTest.kt diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/AssetImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/AssetImporter.kt deleted file mode 100644 index 48d33fd2ee..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/AssetImporter.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.Column -import com.atlan.model.assets.Connection -import com.atlan.model.assets.Database -import com.atlan.model.assets.ISQL -import com.atlan.model.assets.MaterializedView -import com.atlan.model.assets.Schema -import com.atlan.model.assets.Table -import com.atlan.model.assets.View -import com.atlan.model.enums.AssetCreationHandling -import com.atlan.pkg.PackageContext -import com.atlan.pkg.Utils -import com.atlan.pkg.serde.csv.CSVImporter -import com.atlan.pkg.serde.csv.CSVXformer -import com.atlan.pkg.serde.csv.ImportResults -import com.atlan.pkg.util.AssetResolver -import com.atlan.pkg.util.AssetResolver.ConnectionIdentity -import com.atlan.pkg.util.AssetResolver.QualifiedNameDetails -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import assets into Atlan from a provided CSV file. - * - * Only the assets and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param filename name of the file to import - * @param typeNameFilter asset types to which to restrict loading - * @param logger through which to record logging - * @param creationHandling what to do with assets that do not exist (create full, partial, or ignore) - * @param batchSize maximum number of records to save per API request - * @param trackBatches if true, minimal details about every asset created or updated is tracked (if false, only counts of each are tracked) - */ -abstract class AssetImporter( - ctx: PackageContext, - private val delta: DeltaProcessor?, - filename: String, - typeNameFilter: String, - logger: KLogger, - creationHandling: AssetCreationHandling = Utils.getCreationHandling(ctx.config.assetsUpsertSemantic, AssetCreationHandling.FULL), - batchSize: Int = ctx.config.assetsBatchSize.toInt(), - trackBatches: Boolean = ctx.config.trackBatches, -) : CSVImporter( - ctx = ctx, - filename = filename, - logger = logger, - typeNameFilter = typeNameFilter, - attrsToOverwrite = attributesToClear(ctx.config.assetsAttrToOverwrite.toMutableList(), "assets", logger), - creationHandling = creationHandling, - batchSize = batchSize, - trackBatches = trackBatches, - fieldSeparator = ctx.config.assetsFieldSeparator[0], - failOnErrors = ctx.config.assetsFailOnErrors, - ) { - /** {@inheritDoc} */ - override fun import(columnsToSkip: Set): ImportResults? { - // Can skip all of these columns when deserializing a row as they will be set by - // the creator methods anyway - return super.import( - setOf( - Asset.CONNECTION_NAME.atlanFieldName, - // ConnectionImporter.CONNECTOR_TYPE, // Let this be loaded, for mis-named connections - ISQL.DATABASE_NAME.atlanFieldName, - ISQL.SCHEMA_NAME.atlanFieldName, - ENTITY_NAME, - ColumnImporter.COLUMN_PARENT_QN, - Column.ORDER.atlanFieldName, - ), - ) - } - - companion object : AssetResolver { - const val ENTITY_NAME = "entityName" - - /** {@inheritDoc} */ - override fun getQualifiedNameDetails( - row: List, - header: List, - typeName: String, - ): QualifiedNameDetails { - val parent: QualifiedNameDetails? - val current: String - when (typeName) { - Connection.TYPE_NAME -> { - val connection = CSVXformer.trimWhitespace(row[header.indexOf(Asset.CONNECTION_NAME.atlanFieldName)]) - val connector = CSVXformer.trimWhitespace(row[header.indexOf(ConnectionImporter.CONNECTOR_TYPE)]).lowercase() - current = ConnectionIdentity(connection, connector).toString() - parent = null - } - Database.TYPE_NAME -> { - current = CSVXformer.trimWhitespace(row[header.indexOf(ISQL.DATABASE_NAME.atlanFieldName)]) - parent = getQualifiedNameDetails(row, header, Connection.TYPE_NAME) - } - Schema.TYPE_NAME -> { - current = CSVXformer.trimWhitespace(row[header.indexOf(ISQL.SCHEMA_NAME.atlanFieldName)]) - parent = getQualifiedNameDetails(row, header, Database.TYPE_NAME) - } - Table.TYPE_NAME, View.TYPE_NAME, MaterializedView.TYPE_NAME -> { - current = CSVXformer.trimWhitespace(row[header.indexOf(ENTITY_NAME)]) - parent = getQualifiedNameDetails(row, header, Schema.TYPE_NAME) - } - Column.TYPE_NAME -> { - current = CSVXformer.trimWhitespace(row[header.indexOf(ColumnImporter.COLUMN_NAME)]) - parent = getQualifiedNameDetails(row, header, Table.TYPE_NAME) - } - else -> throw IllegalStateException("Unknown SQL type: $typeName") - } - val unique = - parent?.let { - if (parent.uniqueQN.isBlank()) current else "${parent.uniqueQN}/$current" - } ?: current - val partial = - parent?.let { - if (parent.partialQN.isBlank()) current else "${parent.partialQN}/$current" - } ?: "" - return QualifiedNameDetails( - unique, - partial, - parent?.uniqueQN ?: "", - parent?.partialQN ?: "", - ) - } - } - - /** {@inheritDoc} */ - override fun includeRow( - row: List, - header: List, - typeIdx: Int, - qnIdx: Int, - ): Boolean { - if (super.includeRow(row, header, typeIdx, qnIdx)) { - delta?.resolveAsset(row, header)?.let { identity -> - return delta.reloadAsset(identity) - } ?: return true - } - return false - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ColumnImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ColumnImporter.kt deleted file mode 100644 index c5e19d4674..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ColumnImporter.kt +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.Column -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.serde.cell.DataTypeXformer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import columns into Atlan from a provided CSV file. - * - * Only the columns and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class ColumnImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = Column.TYPE_NAME, - logger = logger, - ) { - companion object { - const val COLUMN_PARENT_QN = "columnParentQualifiedName" - const val COLUMN_NAME = "columnName" - } - - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(COLUMN_NAME)?.let { it as String } ?: "" - val order = deserializer.getValue(Column.ORDER.atlanFieldName)?.let { it as Int } ?: 0 - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val parentQN = "$connectionQN/${qnDetails.parentPartialQN}" - val parentType = preprocessed.entityQualifiedNameToType[qnDetails.parentUniqueQN] ?: throw IllegalStateException("Could not find any table/view at: ${qnDetails.parentUniqueQN}") - val builder = Column.creator(name, parentType, parentQN, order) - val rawDataType = deserializer.getRawValue(Column.DATA_TYPE.atlanFieldName) - if (rawDataType.isNotBlank()) { - builder.rawDataTypeDefinition(rawDataType) - if (!rawDataType.contains("<") && !rawDataType.contains(">")) { - // Only attempt to parse things like precision, scale and max-length if this is not a complex type - DataTypeXformer.getPrecision(rawDataType)?.let { builder.precision(it) } - DataTypeXformer.getScale(rawDataType)?.let { builder.numericScale(it) } - DataTypeXformer.getMaxLength(rawDataType)?.let { builder.maxLength(it) } - } - } - return builder - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ConnectionImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ConnectionImporter.kt index 88b550258c..7d18e6ce69 100644 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ConnectionImporter.kt +++ b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ConnectionImporter.kt @@ -4,11 +4,15 @@ package com.atlan.pkg.rab import RelationalAssetsBuilderCfg import com.atlan.model.assets.Asset +import com.atlan.model.assets.Column import com.atlan.model.assets.Connection +import com.atlan.model.assets.ISQL import com.atlan.model.enums.AssetCreationHandling import com.atlan.model.enums.AtlanConnectorType import com.atlan.pkg.PackageContext import com.atlan.pkg.serde.RowDeserializer +import com.atlan.pkg.serde.csv.CSVImporter +import com.atlan.pkg.serde.csv.ImportResults import mu.KLogger import java.util.stream.Stream @@ -28,22 +32,38 @@ class ConnectionImporter( ctx: PackageContext, private val inputFile: String, logger: KLogger, -) : AssetImporter( +) : CSVImporter( ctx = ctx, - delta = null, filename = inputFile, - // Only allow full or updates to connections, as partial connections would be hidden - // and impossible to delete via utilities like the Connection Delete workflow - typeNameFilter = Connection.TYPE_NAME, logger = logger, + typeNameFilter = Connection.TYPE_NAME, + attrsToOverwrite = attributesToClear(ctx.config.assetsAttrToOverwrite.toMutableList(), "assets", logger), creationHandling = if (ctx.config.assetsUpsertSemantic == "update") AssetCreationHandling.NONE else AssetCreationHandling.FULL, batchSize = 1, trackBatches = true, + fieldSeparator = ctx.config.assetsFieldSeparator[0], + failOnErrors = ctx.config.assetsFailOnErrors, ) { companion object { const val CONNECTOR_TYPE = "connectorType" } + /** {@inheritDoc} */ + override fun import(columnsToSkip: Set): ImportResults? { + // Can skip all of these columns when deserializing a row as they will be set by + // the creator methods anyway + return super.import( + setOf( + Asset.CONNECTION_NAME.atlanFieldName, + ISQL.DATABASE_NAME.atlanFieldName, + ISQL.SCHEMA_NAME.atlanFieldName, + AssetXformer.ENTITY_NAME, + ColumnXformer.COLUMN_PARENT_QN, + Column.ORDER.atlanFieldName, + ), + ) + } + /** {@inheritDoc} */ @Suppress("UNCHECKED_CAST") override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/DatabaseImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/DatabaseImporter.kt deleted file mode 100644 index ba035ec790..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/DatabaseImporter.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.Database -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import databases into Atlan from a provided CSV file. - * - * Only the databases and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class DatabaseImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = Database.TYPE_NAME, - logger = logger, - ) { - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(Database.DATABASE_NAME.atlanFieldName)?.let { it as String } ?: "" - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - return Database - .creator(name, connectionQN) - .schemaCount(preprocessed.qualifiedNameToChildCount[qnDetails.uniqueQN]?.toInt()) - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/MaterializedViewImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/MaterializedViewImporter.kt deleted file mode 100644 index 12f963ceeb..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/MaterializedViewImporter.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.MaterializedView -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import materialized views into Atlan from a provided CSV file. - * - * Only the materialized views and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class MaterializedViewImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = MaterializedView.TYPE_NAME, - logger = logger, - ) { - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(ENTITY_NAME)?.let { it as String } ?: "" - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - val schemaQN = "$connectionQN/${qnDetails.parentPartialQN}" - return MaterializedView - .creator(name, schemaQN) - .columnCount(preprocessed.qualifiedNameToChildCount[qnDetails.uniqueQN]?.toLong()) - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/SchemaImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/SchemaImporter.kt deleted file mode 100644 index e8b3ec70d1..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/SchemaImporter.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.Schema -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import schemas into Atlan from a provided CSV file. - * - * Only the schemas and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class SchemaImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = Schema.TYPE_NAME, - logger = logger, - ) { - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(Schema.SCHEMA_NAME.atlanFieldName)?.let { it as String } ?: "" - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - val databaseQN = "$connectionQN/${qnDetails.parentPartialQN}" - return Schema - .creator(name, databaseQN) - .tableCount(preprocessed.qualifiedNameToTableCount[qnDetails.uniqueQN]?.toInt()) - .viewCount(preprocessed.qualifiedNameToViewCount[qnDetails.uniqueQN]?.toInt()) - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/TableImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/TableImporter.kt deleted file mode 100644 index 8fdfbfca51..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/TableImporter.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.Table -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import tables into Atlan from a provided CSV file. - * - * Only the tables and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class TableImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = Table.TYPE_NAME, - logger = logger, - ) { - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(ENTITY_NAME)?.let { it as String } ?: "" - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - val schemaQN = "$connectionQN/${qnDetails.parentPartialQN}" - return Table - .creator(name, schemaQN) - .columnCount(preprocessed.qualifiedNameToChildCount[qnDetails.uniqueQN]?.toLong()) - } -} diff --git a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ViewImporter.kt b/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ViewImporter.kt deleted file mode 100644 index da188d6e06..0000000000 --- a/samples/packages/relational-assets-builder/src/main/kotlin/com/atlan/pkg/rab/ViewImporter.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import RelationalAssetsBuilderCfg -import com.atlan.model.assets.Asset -import com.atlan.model.assets.View -import com.atlan.pkg.PackageContext -import com.atlan.pkg.serde.RowDeserializer -import com.atlan.pkg.util.DeltaProcessor -import mu.KLogger - -/** - * Import views into Atlan from a provided CSV file. - * - * Only the views and attributes in the provided CSV file will attempt to be loaded. - * By default, any blank values in a cell in the CSV file will be ignored. If you would like any - * particular column's blank values to actually overwrite (i.e. remove) existing values for that - * asset in Atlan, then add that column's field to getAttributesToOverwrite. - * - * @param ctx context through which this package is running - * @param delta the processor containing any details about file deltas - * @param preprocessed details of the preprocessed CSV file - * @param connectionImporter that was used to import connections - * @param logger through which to record logging - */ -class ViewImporter( - ctx: PackageContext, - private val delta: DeltaProcessor, - private val preprocessed: Importer.Results, - private val connectionImporter: ConnectionImporter, - logger: KLogger, -) : AssetImporter( - ctx = ctx, - delta = delta, - filename = preprocessed.preprocessedFile, - typeNameFilter = View.TYPE_NAME, - logger = logger, - ) { - /** {@inheritDoc} */ - override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> { - val name = deserializer.getValue(ENTITY_NAME)?.let { it as String } ?: "" - val connectionQN = connectionImporter.getBuilder(deserializer).build().qualifiedName - val qnDetails = getQualifiedNameDetails(deserializer.row, deserializer.heading, typeNameFilter) - val schemaQN = "$connectionQN/${qnDetails.parentPartialQN}" - return View - .creator(name, schemaQN) - .columnCount(preprocessed.qualifiedNameToChildCount[qnDetails.uniqueQN]?.toLong()) - } -} diff --git a/samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/DeltaTest.kt b/samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/DeltaTest.kt deleted file mode 100644 index 4a4c356e4a..0000000000 --- a/samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/DeltaTest.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - Copyright 2023 Atlan Pte. Ltd. */ -package com.atlan.pkg.rab - -import com.atlan.model.assets.Column -import com.atlan.model.assets.Connection -import com.atlan.model.assets.Table -import com.atlan.model.assets.View -import com.atlan.model.enums.AtlanConnectorType -import com.atlan.pkg.PackageTest -import com.atlan.pkg.Utils -import com.atlan.pkg.util.AssetResolver -import com.atlan.pkg.util.FileBasedDelta -import org.testng.Assert.assertTrue -import java.io.File -import java.nio.file.Paths -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Test pre-processing of full-load CSV files to detect which assets should be removed. - */ -class DeltaTest : PackageTest("rd") { - override val logger = Utils.getLogger(this.javaClass.name) - - private val conn1 = makeUnique("c1") - private val conn1Type = AtlanConnectorType.ICEBERG - private val conn1QN = "default/${conn1Type.value}/1234567890" - - private val previousFile = "assets.csv" - private val currentFile = "assets_latest.csv" - private var delta: FileBasedDelta? = null - - private val files = - listOf( - previousFile, - currentFile, - "debug.log", - ) - - private fun prepFile() { - // Prepare a copy of the file with unique names for connections - val previousIn = Paths.get("src", "test", "resources", previousFile).toFile() - val previousOut = Paths.get(testDirectory, previousFile).toFile() - replaceVars(previousIn, previousOut) - val latestIn = Paths.get("src", "test", "resources", currentFile).toFile() - val latestOut = Paths.get(testDirectory, currentFile).toFile() - replaceVars(latestIn, latestOut) - } - - private fun replaceVars( - input: File, - output: File, - ) { - input.useLines { lines -> - lines.forEach { line -> - val revised = - line - .replace("{{CONNECTION1}}", conn1) - output.appendText("$revised\n") - } - } - } - - override fun setup() { - prepFile() - val connectionsMap = - mapOf( - AssetResolver.ConnectionIdentity(conn1, conn1Type.value) to conn1QN, - ) - delta = FileBasedDelta(connectionsMap, AssetImporter, Utils.getLogger(this.javaClass.name), compareChecksums = true) - delta!!.calculateDelta( - Paths.get(testDirectory, currentFile).toString(), - Paths.get(testDirectory, previousFile).toString(), - ) - } - - @Test - fun hasSomethingToDelete() { - assertTrue(delta!!.hasAnythingToDelete()) - } - - @Test - fun totalAssetsToDelete() { - assertEquals(3, delta!!.assetsToDelete.size) - val types = - delta!! - .assetsToDelete - .entrySet() - .map { it.key.typeName } - .toList() - .toSet() - assertEquals(2, types.size) - assertTrue(types.contains(View.TYPE_NAME)) - assertTrue(types.contains(Column.TYPE_NAME)) - } - - @Test - fun specificAssetsToDelete() { - delta!!.assetsToDelete.entrySet().forEach { - when (it.key.typeName) { - View.TYPE_NAME -> assertTrue("$conn1QN/TEST_DB/TEST_SCHEMA/TEST_VIEW" == it.key.qualifiedName) - Column.TYPE_NAME -> { - assertTrue( - "$conn1QN/TEST_DB/TEST_SCHEMA/TEST_VIEW/COL3" == it.key.qualifiedName || - "$conn1QN/TEST_DB/TEST_SCHEMA/TEST_VIEW/COL4" == it.key.qualifiedName, - ) - } - } - } - } - - @Test - fun totalAssetsToReload() { - assertEquals(3, delta!!.assetsToReload.size) - val types = - delta!! - .assetsToReload - .entrySet() - .map { it.key.typeName } - .toList() - .toSet() - assertEquals(3, types.size) - assertTrue(types.contains(Connection.TYPE_NAME)) - assertTrue(types.contains(Table.TYPE_NAME)) - assertTrue(types.contains(Column.TYPE_NAME)) - } - - @Test - fun specificAssetsToReload() { - delta!!.assetsToReload.entrySet().forEach { - when (it.key.typeName) { - Connection.TYPE_NAME -> assertEquals(conn1QN, it.key.qualifiedName) - Table.TYPE_NAME -> assertEquals("$conn1QN/TEST_DB/TEST_SCHEMA/TEST_TBL", it.key.qualifiedName) - Column.TYPE_NAME -> assertEquals("$conn1QN/TEST_DB/TEST_SCHEMA/TEST_TBL/COL2", it.key.qualifiedName) - } - } - } - - @Test - fun filesCreated() { - validateFilesExist(files) - } - - @Test - fun errorFreeLog() { - validateErrorFreeLog() - } -}