Skip to content

Commit

Permalink
Use QueryService by default (#80)
Browse files Browse the repository at this point in the history
  • Loading branch information
alex268 authored Oct 17, 2024
2 parents 8c569f3 + 8a3608f commit d995bd3
Show file tree
Hide file tree
Showing 9 changed files with 183 additions and 113 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,6 @@ Specify the YDB JDBC driver in the dependencies:
</dependency>
</dependencies>
```
### Using QueryService mode
By default JDBC driver executes all queries via TableService, so it brings corresponding [limitations](https://ydb.tech/docs/en/concepts/limits-ydb#query).
To eliminate these limitations you can try a new experimentail [QueryService](https://ydb.tech/docs/en/conceptrs/query_service) mode by passing property `useQueryService=true` to the JDBC URL

### Authentication modes

YDB JDBC Driver supports the following [authentication modes](https://ydb.tech/en/docs/reference/ydb-sdk/auth):
Expand All @@ -62,6 +58,10 @@ Driver supports the following configuration properties, which can be specified i
* `secureConnection` - boolean value, true if TLS should be enforced (normally configured via `grpc://` or `grpcs://` scheme in the JDBC URL);
* `secureConnectionCertificate` - custom CA certificate for TLS connections, can be passed either as literal value or as a file reference.

### Using TableService mode
By default JDBC driver executes all queries via QueryService, which uses grpc streams for the results recieving.
If your database instance doesn't support this service, you can use old TableService mode by passing property `useQueryService=false` to the JDBC URL.

### Building
By default all tests are run using a local YDB instance in Docker (if host has Docker or Docker Machine installed)
To disable these tests run `mvn test -DYDB_DISABLE_INTEGRATION_TESTS=true`
Expand Down
2 changes: 1 addition & 1 deletion jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public class YdbConfig {
+ "{@code 0} disables the cache.", 256
);
static final YdbProperty<Boolean> USE_QUERY_SERVICE = YdbProperty.bool("useQueryService",
"Use QueryService instead of TableService", false
"Use QueryService instead of TableService", true
);

static final YdbProperty<String> USE_PREFIX_PATH = YdbProperty.string("usePrefixPath",
Expand Down
85 changes: 83 additions & 2 deletions jdbc/src/test/java/tech/ydb/jdbc/YdbDriverTablesTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public void defaultModeTest() throws SQLException {
Assertions.assertEquals(Date.valueOf(ld.plusDays(readed)), rs.getDate("date"));
}
}
Assertions.assertEquals(1000, readed);
Assertions.assertEquals(2002, readed);
}

// single update
Expand Down Expand Up @@ -319,7 +319,6 @@ public void forceScanAndBulkTest() throws SQLException {
@Test
public void streamResultsTest() throws SQLException {
try (Connection conn = DriverManager.getConnection(jdbcURL
.withArg("useQueryService", "true")
.withArg("useStreamResultSets", "true")
.build()
)) {
Expand Down Expand Up @@ -443,4 +442,86 @@ public void streamResultsTest() throws SQLException {
}
}
}

@Test
public void tableServiceModeTest() throws SQLException {
try (Connection connection = DriverManager.getConnection(jdbcURL.withArg("useQueryService", "false").build())) {
try {
connection.createStatement().execute(DROP_TABLE);
} catch (SQLException e) {
// ignore
}

connection.createStatement().execute(CREATE_TABLE);

LocalDate ld = LocalDate.of(2017, 12, 3);
String prefix = "text-value-";
int idx = 0;

// single upsert
try (PreparedStatement ps = connection.prepareStatement(UPSERT_ROW)) {
ps.setInt(1, ++idx);
ps.setString(2, prefix + idx);
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
ps.executeUpdate();
}

// single insert
try (PreparedStatement ps = connection.prepareStatement(INSERT_ROW)) {
ps.setInt(1, ++idx);
ps.setString(2, prefix + idx);
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
ps.executeUpdate();
}

// batch upsert
try (PreparedStatement ps = connection.prepareStatement(UPSERT_ROW)) {
for (int j = 0; j < 1000; j++) {
ps.setInt(1, ++idx);
ps.setString(2, prefix + idx);
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
ps.addBatch();
}
ps.executeBatch();
}

// batch insert
try (PreparedStatement ps = connection.prepareStatement(INSERT_ROW)) {
for (int j = 0; j < 1000; j++) {
ps.setInt(1, ++idx);
ps.setString(2, prefix + idx);
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
ps.addBatch();
}
ps.executeBatch();
}

// read all
try (Statement st = connection.createStatement()) {
int readed = 0;
try (ResultSet rs = st.executeQuery(SELECT_ALL)) {
while (rs.next()) {
readed++;
Assertions.assertEquals(readed, rs.getInt("id"));
Assertions.assertEquals(prefix + readed, rs.getString("value"));
Assertions.assertEquals(Date.valueOf(ld.plusDays(readed)), rs.getDate("date"));
}
}
Assertions.assertEquals(1000, readed);
}

// single update
try (PreparedStatement ps = connection.prepareStatement(UPDATE_ROW)) {
ps.setString(1, "updated-value");
ps.setInt(2, 1);
ps.executeUpdate();
}

// single delete
try (PreparedStatement ps = connection.prepareStatement(DELETE_ROW)) {
ps.setInt(1, 2);
ps.executeUpdate();
}
}
}
}
22 changes: 11 additions & 11 deletions jdbc/src/test/java/tech/ydb/jdbc/impl/YdbConnectionImplTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -876,15 +876,15 @@ private String createPayload(Random rnd, int length) {
@Timeout(value = 30, unit = TimeUnit.SECONDS, threadMode = Timeout.ThreadMode.SAME_THREAD)
public void testBigBulkAndScan() throws SQLException {
String bulkUpsert = QUERIES.upsertOne(SqlQueries.JdbcQuery.BULK, "c_Text", "Text?");
String scanSelectAll = QUERIES.scanSelectSQL();
String selectAll = QUERIES.selectSQL();
String selectOne = QUERIES.selectAllByKey("?");

Random rnd = new Random(0x234567);
int payloadLength = 1000;

try {
try (Connection conn = jdbc.createCustomConnection("useStreamResultSets", "true")) {
// BULK UPSERT
try (PreparedStatement ps = jdbc.connection().prepareStatement(bulkUpsert)) {
try (PreparedStatement ps = conn.prepareStatement(bulkUpsert)) {
for (int idx = 1; idx <= 10000; idx++) {
ps.setInt(1, idx);
String payload = createPayload(rnd, payloadLength);
Expand All @@ -898,7 +898,7 @@ public void testBigBulkAndScan() throws SQLException {
}

// SCAN all table
try (PreparedStatement ps = jdbc.connection().prepareStatement(scanSelectAll)) {
try (PreparedStatement ps = conn.prepareStatement(selectAll)) {
int readed = 0;
Assertions.assertTrue(ps.execute());
try (ResultSet rs = ps.getResultSet()) {
Expand All @@ -912,7 +912,7 @@ public void testBigBulkAndScan() throws SQLException {
}

// Canceled scan
try (PreparedStatement ps = jdbc.connection().prepareStatement(scanSelectAll)) {
try (PreparedStatement ps = conn.prepareStatement(selectAll)) {
Assertions.assertTrue(ps.execute());
ps.getResultSet().next();
ps.getResultSet().close();
Expand All @@ -930,14 +930,14 @@ public void testBigBulkAndScan() throws SQLException {
}

// Scan was cancelled, but connection still work
try (PreparedStatement ps = jdbc.connection().prepareStatement(selectOne)) {
try (PreparedStatement ps = conn.prepareStatement(selectOne)) {
ps.setInt(1, 1234);

Assertions.assertTrue(ps.execute());
try (ResultSet rs = ps.getResultSet()) {
Assertions.assertTrue(rs.next());
Assertions.assertEquals(1234, rs.getInt("key"));
Assertions.assertEquals(payloadLength, rs.getString("c_Text").length());
Assertions.assertEquals(payloadLength, rs.getString("c_Text").length());
Assertions.assertFalse(rs.next());
}
}
Expand Down Expand Up @@ -1176,8 +1176,8 @@ public void fullScanAnalyzerPreparedStatementTest() throws SQLException {
try (Connection connection = jdbc.createCustomConnection("jdbcFullScanDetector", "true")) {
try (PreparedStatement ps = connection.prepareStatement("print_JDBC_stats();")) {
sa.check(ps.executeQuery())
.assertMetaColumns()
.assertNoRows();
.assertMetaColumns()
.assertNoRows();
}

try (PreparedStatement ps = connection.prepareStatement(preparedSelectByKey)) {
Expand Down Expand Up @@ -1238,8 +1238,8 @@ public void fullScanAnalyzerPreparedStatementTest() throws SQLException {

try (PreparedStatement ps = connection.prepareStatement("print_JDBC_stats();")) {
sa.check(ps.executeQuery())
.assertMetaColumns()
.assertNoRows();
.assertMetaColumns()
.assertNoRows();
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ public class YdbPreparedStatementImplTest {
private static final String SELECT_BY_KEY_SQL = ""
+ "declare $key as Optional<Int32>;\n"
+ "select key, #column from #tableName where key=$key";
private static final String SCAN_SELECT_BY_KEY_SQL = ""
+ "declare $key as Optional<Int32>;\n"
+ "scan select key, #column from #tableName where key=$key";

@BeforeAll
public static void initTable() throws SQLException {
Expand Down Expand Up @@ -97,20 +94,6 @@ private YdbPreparedStatement prepareSelectByKey(String column) throws SQLExcepti
return jdbc.connection().prepareStatement(sql).unwrap(YdbPreparedStatement.class);
}

private PreparedStatement prepareScanSelect(String column) throws SQLException {
String sql = SIMPLE_SELECT_SQL
.replaceAll("#column", column)
.replaceAll("#tableName", TEST_TABLE_NAME);
return jdbc.connection().prepareStatement("SCAN " + sql);
}

private String scanSelectByKey(String column) throws SQLException {
String sql = SCAN_SELECT_BY_KEY_SQL
.replaceAll("#column", column)
.replaceAll("#tableName", TEST_TABLE_NAME);
return sql;
}

private YdbPreparedStatement prepareSelectAll() throws SQLException {
return jdbc.connection().prepareStatement(TEST_TABLE.selectSQL())
.unwrap(YdbPreparedStatement.class);
Expand Down Expand Up @@ -260,7 +243,7 @@ public void executeEmptyBatch(SqlQueries.YqlQuery mode) throws SQLException {
}

@Test
public void executeQueryBatchWithScanRead() throws SQLException {
public void executeQueryBatchWithBigRead() throws SQLException {
int valuesCount = 5000;
String[] values = new String[valuesCount];
for (int idx = 1; idx <= valuesCount; idx += 1) {
Expand All @@ -283,14 +266,7 @@ public void executeQueryBatchWithScanRead() throws SQLException {
}
}

ExceptionAssert.sqlException("Result #0 was truncated to 1000 rows", () -> {
// Result is truncated (and we catch that)
try (PreparedStatement select = prepareSimpleSelect("c_Text")) {
select.executeQuery();
}
});

try (PreparedStatement select = prepareScanSelect("c_Text")) {
try (PreparedStatement select = prepareSimpleSelect("c_Text")) {
TextSelectAssert check = TextSelectAssert.of(select.executeQuery(), "c_Text", "Text");

for (int idx = 1; idx <= valuesCount; idx += 1) {
Expand Down Expand Up @@ -365,42 +341,6 @@ public void executeQueryInTx(SqlQueries.YqlQuery mode) throws SQLException {
}
}

@ParameterizedTest(name = "with {0}")
@EnumSource(SqlQueries.YqlQuery.class)
public void executeScanQueryInTx(SqlQueries.YqlQuery mode) throws SQLException {
String upsertYql = TEST_TABLE.upsertOne(mode, "c_Text", "Text");
String scanSelectYql = scanSelectByKey("c_Text");

jdbc.connection().setAutoCommit(false);
YdbConnection conn = jdbc.connection().unwrap(YdbConnection.class);
try {
try (YdbPreparedStatement statement = conn.prepareStatement(upsertYql)) {
statement.setInt("key", 1);
statement.setString("c_Text", "value-1");
statement.execute();
}

try (YdbPreparedStatement select = conn.prepareStatement(scanSelectYql)) {
select.setInt("key", 1);

ExceptionAssert.sqlException(YdbConst.SCAN_QUERY_INSIDE_TRANSACTION, () -> select.executeQuery());

jdbc.connection().commit();

select.setInt("key", 1);
TextSelectAssert.of(select.executeQuery(), "c_Text", "Text")
.nextRow(1, "value-1")
.noNextRows();

select.setInt("key", 2);
TextSelectAssert.of(select.executeQuery(), "c_Text", "Text")
.noNextRows();
}
} finally {
jdbc.connection().setAutoCommit(true);
}
}

@Test
public void executeScanQueryAsUpdate() throws SQLException {
String sql = "SCAN " + TEST_TABLE.upsertOne(SqlQueries.JdbcQuery.STANDARD, "c_Text", "Optional<Text>");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,12 @@ public class YdbPreparedStatementWithDataQueryBatchedImplTest {
private static final YdbHelperExtension ydb = new YdbHelperExtension();

@RegisterExtension
private static final JdbcConnectionExtention jdbc = new JdbcConnectionExtention(ydb);
private static final JdbcConnectionExtention jdbc = new JdbcConnectionExtention(ydb)
.withArg("useQueryService", "false");

private static final String TEST_TABLE_NAME = "ydb_prepared_statement_with_batch_test";
private static final SqlQueries TEST_TABLE = new SqlQueries(TEST_TABLE_NAME);

private static final String UPSERT_SQL = ""
+ "declare $key as Optional<Int32>;\n"
+ "declare $#column as #type;\n"
+ "upsert into #tableName (key, #column) values ($key, $#column)";

private static final String BATCH_UPSERT_SQL = ""
+ "declare $values as List<Struct<key:Int32, #column:#type>>; \n"
+ "upsert into #tableName select * from as_table($values)";
Expand Down Expand Up @@ -77,13 +73,6 @@ public void afterEach() throws SQLException {
jdbc.connection().close();
}

private String upsertSql(String column, String type) {
return UPSERT_SQL
.replaceAll("#column", column)
.replaceAll("#type", type)
.replaceAll("#tableName", TEST_TABLE_NAME);
}

private String batchUpsertSql(String column, String type) {
return BATCH_UPSERT_SQL
.replaceAll("#column", column)
Expand Down
Loading

0 comments on commit d995bd3

Please sign in to comment.