Skip to content

Commit

Permalink
[fix](bf index) add ngram bf index validation in nereids index defini…
Browse files Browse the repository at this point in the history
…tion check (#45780)

Related PR: #44973

Problem Summary:

Need to check ngram bf index properties in index definition for nereids.
  • Loading branch information
airborne12 authored Dec 31, 2024
1 parent 9bcb801 commit 8781666
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 23 deletions.
10 changes: 5 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ public class IndexDef {
private boolean isBuildDeferred = false;
private PartitionNames partitionNames;
private List<Integer> columnUniqueIds = Lists.newArrayList();
private static final int MIN_NGRAM_SIZE = 1;
private static final int MAX_NGRAM_SIZE = 255;
private static final int MIN_BF_SIZE = 64;
private static final int MAX_BF_SIZE = 65535;
public static final int MIN_NGRAM_SIZE = 1;
public static final int MAX_NGRAM_SIZE = 255;
public static final int MIN_BF_SIZE = 64;
public static final int MAX_BF_SIZE = 65535;

public static final String NGRAM_SIZE_KEY = "gram_size";
public static final String NGRAM_BF_SIZE_KEY = "bf_size";
Expand Down Expand Up @@ -273,7 +273,7 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
}
}

private void parseAndValidateProperty(Map<String, String> properties, String key, int minValue, int maxValue)
public static void parseAndValidateProperty(Map<String, String> properties, String key, int minValue, int maxValue)
throws AnalysisException {
String valueStr = properties.get(key);
if (valueStr == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,18 +150,12 @@ public void checkColumn(ColumnDefinition column, KeysType keysType,
"ngram_bf index should have gram_size and bf_size properties");
}
try {
int ngramSize = Integer.parseInt(properties.get(IndexDef.NGRAM_SIZE_KEY));
int bfSize = Integer.parseInt(properties.get(IndexDef.NGRAM_BF_SIZE_KEY));
if (ngramSize > 256 || ngramSize < 1) {
throw new AnalysisException(
"gram_size should be integer and less than 256");
}
if (bfSize > 65535 || bfSize < 64) {
throw new AnalysisException(
"bf_size should be integer and between 64 and 65535");
}
} catch (NumberFormatException e) {
throw new AnalysisException("invalid ngram properties:" + e.getMessage(), e);
IndexDef.parseAndValidateProperty(properties, IndexDef.NGRAM_SIZE_KEY, IndexDef.MIN_NGRAM_SIZE,
IndexDef.MAX_NGRAM_SIZE);
IndexDef.parseAndValidateProperty(properties, IndexDef.NGRAM_BF_SIZE_KEY, IndexDef.MIN_BF_SIZE,
IndexDef.MAX_BF_SIZE);
} catch (Exception ex) {
throw new AnalysisException("invalid ngram bf index params:" + ex.getMessage(), ex);
}
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,87 @@
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VariantType;

import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.Map;

public class IndexDefinitionTest {
@Test
void testVariantIndexFormatV1() throws AnalysisException {
IndexDefinition def = new IndexDefinition("variant_index", false, Lists.newArrayList("col1"), "INVERTED",
null, "comment");
null, "comment");
try {
boolean isIndexFormatV1 = true;
def.checkColumn(new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true,
null, "comment"), KeysType.UNIQUE_KEYS, true, null, isIndexFormatV1);
null, "comment"), KeysType.UNIQUE_KEYS, true, null, isIndexFormatV1);
Assertions.fail("No exception throws.");
} catch (AnalysisException e) {
Assertions.assertTrue(e instanceof AnalysisException);
org.junit.jupiter.api.Assertions.assertInstanceOf(
org.apache.doris.nereids.exceptions.AnalysisException.class, e);
Assertions.assertTrue(e.getMessage().contains("not supported in inverted index format V1"));
}
}

@Test
void testNgramBFIndex() throws AnalysisException {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
def.checkColumn(
new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null, "comment"),
KeysType.DUP_KEYS, false, null, false);
}

@Test
void testInvalidNgramBFIndexColumnType() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(
new ColumnDefinition("col1", IntegerType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, null, false));
}

@Test
void testNgramBFIndexInvalidSize() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "256");
properties.put("bf_size", "10000");

IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, null, false));
}

@Test
void testNgramBFIndexInvalidSize2() {
Map<String, String> properties = new HashMap<>();
properties.put("gram_size", "3");
properties.put("bf_size", "65536");

IndexDefinition def = new IndexDefinition("ngram_bf_index", false, Lists.newArrayList("col1"), "NGRAM_BF",
properties, "comment");
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
KeysType.DUP_KEYS, false, null, false));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ suite("test_ngram_bloomfilter_index") {
DISTRIBUTED BY HASH(`key_id`) BUCKETS 3
PROPERTIES("replication_num" = "1");
"""
exception "bf_size should be integer and between 64 and 65535"
exception "java.sql.SQLException: errCode = 2, detailMessage = invalid ngram bf index params:errCode = 2, detailMessage = 'bf_size' should be an integer between 64 and 65535."
}

def tableName3 = 'test_ngram_bloomfilter_index3'
Expand All @@ -104,10 +104,10 @@ suite("test_ngram_bloomfilter_index") {
"""
test {
sql """ALTER TABLE ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="65536") COMMENT 'http_url ngram_bf index'"""
exception "'bf_size' should be an integer between 64 and 65535"
exception "java.sql.SQLException: errCode = 2, detailMessage = 'bf_size' should be an integer between 64 and 65535."
}
test {
sql """ALTER TABLE ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="256", "bf_size"="65535") COMMENT 'http_url ngram_bf index'"""
exception "'gram_size' should be an integer between 1 and 255"
exception "java.sql.SQLException: errCode = 2, detailMessage = 'gram_size' should be an integer between 1 and 255."
}
}

0 comments on commit 8781666

Please sign in to comment.