diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 756c9024cdcdd1..87392d0ff385d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -196,7 +196,12 @@ public List getSplits(int numBackends) throws UserException { try { return preExecutionAuthenticator.execute(() -> doGetSplits(numBackends)); } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + Optional opt = checkNotSupportedException(e); + if (opt.isPresent()) { + throw opt.get(); + } else { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } } } @@ -232,7 +237,12 @@ public void doStartSplit() throws UserException { ); splitAssignment.finishSchedule(); } catch (Exception e) { - splitAssignment.setException(new UserException(e.getMessage(), e)); + Optional opt = checkNotSupportedException(e); + if (opt.isPresent()) { + splitAssignment.setException(new UserException(opt.get().getMessage(), opt.get())); + } else { + splitAssignment.setException(new UserException(e.getMessage(), e)); + } } }); } @@ -266,40 +276,7 @@ private TableScan createTableScan() throws UserException { private CloseableIterable planFileScanTask(TableScan scan) { long targetSplitSize = getRealFileSplitSize(0); - CloseableIterable splitFiles; - try { - splitFiles = TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize); - } catch (NullPointerException e) { - /* - Caused by: java.lang.NullPointerException: Type cannot be null - at org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull - (Preconditions.java:921) ~[iceberg-bundled-guava-1.4.3.jar:?] - at org.apache.iceberg.types.Types$NestedField.(Types.java:447) ~[iceberg-api-1.4.3.jar:?] - at org.apache.iceberg.types.Types$NestedField.optional(Types.java:416) ~[iceberg-api-1.4.3.jar:?] - at org.apache.iceberg.PartitionSpec.partitionType(PartitionSpec.java:132) ~[iceberg-api-1.4.3.jar:?] - at org.apache.iceberg.DeleteFileIndex.lambda$new$0(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.relocated.com.google.common.collect.RegularImmutableMap.forEach - (RegularImmutableMap.java:297) ~[iceberg-bundled-guava-1.4.3.jar:?] - at org.apache.iceberg.DeleteFileIndex.(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.DeleteFileIndex.(DeleteFileIndex.java:71) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.DeleteFileIndex$Builder.build(DeleteFileIndex.java:578) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.ManifestGroup.plan(ManifestGroup.java:183) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.ManifestGroup.planFiles(ManifestGroup.java:170) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:89) ~[iceberg-core-1.4.3.jar:?] - at org.apache.iceberg.SnapshotScan.planFiles(SnapshotScan.java:139) ~[iceberg-core-1.4.3.jar:?] - at org.apache.doris.datasource.iceberg.source.IcebergScanNode.doGetSplits - (IcebergScanNode.java:209) ~[doris-fe.jar:1.2-SNAPSHOT] - EXAMPLE: - CREATE TABLE iceberg_tb(col1 INT,col2 STRING) USING ICEBERG PARTITIONED BY (bucket(10,col2)); - INSERT INTO iceberg_tb VALUES( ... ); - ALTER TABLE iceberg_tb DROP PARTITION FIELD bucket(10,col2); - ALTER TABLE iceberg_tb DROP COLUMNS col2 STRING; - Link: https://github.com/apache/iceberg/pull/10755 - */ - LOG.warn("Iceberg TableScanUtil.splitFiles throw NullPointerException. Cause : ", e); - throw new NotSupportedException("Unable to read Iceberg table with dropped old partition column."); - } - return splitFiles; + return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize); } private Split createIcebergSplit(FileScanTask fileScanTask) { @@ -529,4 +506,39 @@ private void assignCountToSplits(List splits, long totalCount) { public int numApproximateSplits() { return NUM_SPLITS_PER_PARTITION * partitionPathSet.size() > 0 ? partitionPathSet.size() : 1; } + + private Optional checkNotSupportedException(Exception e) { + if (e instanceof NullPointerException) { + /* + Caused by: java.lang.NullPointerException: Type cannot be null + at org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull + (Preconditions.java:921) ~[iceberg-bundled-guava-1.4.3.jar:?] + at org.apache.iceberg.types.Types$NestedField.(Types.java:447) ~[iceberg-api-1.4.3.jar:?] + at org.apache.iceberg.types.Types$NestedField.optional(Types.java:416) ~[iceberg-api-1.4.3.jar:?] + at org.apache.iceberg.PartitionSpec.partitionType(PartitionSpec.java:132) ~[iceberg-api-1.4.3.jar:?] + at org.apache.iceberg.DeleteFileIndex.lambda$new$0(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.relocated.com.google.common.collect.RegularImmutableMap.forEach + (RegularImmutableMap.java:297) ~[iceberg-bundled-guava-1.4.3.jar:?] + at org.apache.iceberg.DeleteFileIndex.(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.DeleteFileIndex.(DeleteFileIndex.java:71) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.DeleteFileIndex$Builder.build(DeleteFileIndex.java:578) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.ManifestGroup.plan(ManifestGroup.java:183) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.ManifestGroup.planFiles(ManifestGroup.java:170) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:89) ~[iceberg-core-1.4.3.jar:?] + at org.apache.iceberg.SnapshotScan.planFiles(SnapshotScan.java:139) ~[iceberg-core-1.4.3.jar:?] + at org.apache.doris.datasource.iceberg.source.IcebergScanNode.doGetSplits + (IcebergScanNode.java:209) ~[doris-fe.jar:1.2-SNAPSHOT] + EXAMPLE: + CREATE TABLE iceberg_tb(col1 INT,col2 STRING) USING ICEBERG PARTITIONED BY (bucket(10,col2)); + INSERT INTO iceberg_tb VALUES( ... ); + ALTER TABLE iceberg_tb DROP PARTITION FIELD bucket(10,col2); + ALTER TABLE iceberg_tb DROP COLUMNS col2 STRING; + Link: https://github.com/apache/iceberg/pull/10755 + */ + LOG.warn("Iceberg TableScanUtil.splitFiles throw NullPointerException. Cause : ", e); + return Optional.of( + new NotSupportedException("Unable to read Iceberg table with dropped old partition column.")); + } + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index eb25336ab0b033..7d857524f2b6b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Catalog.TableNotExistException; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; @@ -94,7 +95,15 @@ protected List listDatabaseNames() { public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); try { - return hadoopAuthenticator.doAs(() -> catalog.tableExists(Identifier.create(dbName, tblName))); + return hadoopAuthenticator.doAs(() -> { + try { + catalog.getTable(Identifier.create(dbName, tblName)); + return true; + } catch (TableNotExistException e) { + return false; + } + }); + } catch (IOException e) { throw new RuntimeException("Failed to check table existence, catalog name: " + getName(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 347e8df5c8f75e..622d6dbdfaf6af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -273,7 +273,7 @@ private PaimonSchema loadPaimonSchemaBySchemaId(PaimonSchemaCacheKey key) throws PredicateBuilder builder = new PredicateBuilder(table.rowType()); Predicate predicate = builder.equal(0, key.getSchemaId()); // Adding predicates will also return excess data - List rows = PaimonUtil.read(table, new int[][] {{0}, {1}, {2}}, predicate); + List rows = PaimonUtil.read(table, new int[] {0, 1, 2}, predicate); for (InternalRow row : rows) { PaimonSchema schema = PaimonUtil.rowToSchema(row); if (schema.getSchemaId() == key.getSchemaId()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java index 5b711e070667b7..109394fabded6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java @@ -97,7 +97,7 @@ private PaimonSnapshot loadLatestSnapshot(PaimonSnapshotCacheKey key) throws IOE Table table = ((PaimonExternalCatalog) key.getCatalog()).getPaimonTable(key.getDbName(), key.getTableName() + Catalog.SYSTEM_TABLE_SPLITTER + SnapshotsTable.SNAPSHOTS); // snapshotId and schemaId - List rows = PaimonUtil.read(table, new int[][] {{0}, {1}}, null); + List rows = PaimonUtil.read(table, new int[] {0, 1}, null); long latestSnapshotId = 0L; long latestSchemaId = 0L; for (InternalRow row : rows) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index b3df41bc5cef17..bbb1eaf5096520 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -62,7 +62,7 @@ public class PaimonUtil { private static final Logger LOG = LogManager.getLogger(PaimonUtil.class); public static List read( - Table table, @Nullable int[][] projection, @Nullable Predicate predicate, + Table table, @Nullable int[] projection, @Nullable Predicate predicate, Pair, String>... dynamicOptions) throws IOException { Map options = new HashMap<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 28efbc58f51b84..0e9a8042a65f96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -208,7 +208,12 @@ public List getSplits(int numBackends) throws UserException { .valueOf(sessionVariable.getIgnoreSplitType()); List splits = new ArrayList<>(); int[] projected = desc.getSlots().stream().mapToInt( - slot -> (source.getPaimonTable().rowType().getFieldNames().indexOf(slot.getColumn().getName()))) + slot -> source.getPaimonTable().rowType() + .getFieldNames() + .stream() + .map(String::toLowerCase) + .collect(Collectors.toList()) + .indexOf(slot.getColumn().getName())) .toArray(); ReadBuilder readBuilder = source.getPaimonTable().newReadBuilder(); List paimonSplits = readBuilder.withFilter(predicates) diff --git a/fe/pom.xml b/fe/pom.xml index 2c61fe32299c38..8e59c80c89a31c 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -222,7 +222,7 @@ under the License. be-java-extensions - 2.1.1 + 2.1.3 1.11.4 1.13.1 3.4.3 @@ -318,7 +318,7 @@ under the License. - 1.4.3 + 1.6.1 0.49.0-public 17.0.0 2.7.4-11 @@ -365,7 +365,7 @@ under the License. 2.3.2 0.27 - 0.8.1 + 1.0.0 3.4.4 shade-format-flatbuffers