From 0668ea251cfb90e13836fa38d3a49d934fd2e34b Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 12 Oct 2022 17:27:57 +0200 Subject: [PATCH 01/34] replace Connection with DataSource in OSHDBH2/JDBC --- .../heigit/ohsome/oshdb/api/db/OSHDBH2.java | 129 ++++++------------ .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 55 +++----- .../oshdb/api/mapreducer/MapReducer.java | 6 +- .../mapreducer/backend/MapReducerJdbc.java | 78 ++++++----- .../backend/MapReducerJdbcSinglethread.java | 21 +-- .../ohsome/oshdb/api/tests/TestCollect.java | 1 + 6 files changed, 121 insertions(+), 169 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java index 3f0dd6ac7..e2973284d 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.api.db; +import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -11,6 +12,10 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; +import org.h2.jdbcx.JdbcConnectionPool; +import org.h2.jdbcx.JdbcDataSource; +import org.heigit.ohsome.oshdb.OSHDB; import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; @@ -19,6 +24,8 @@ */ public class OSHDBH2 extends OSHDBJdbc { + private JdbcConnectionPool connectionPool; + /** * Opens a connection to oshdb data stored in a H2 database file. * @@ -27,111 +34,53 @@ public class OSHDBH2 extends OSHDBJdbc { * @throws SQLException if the database couldn't be opened * @throws ClassNotFoundException if the H2 database driver is not installed on the system */ - public OSHDBH2(String databaseFile) throws SQLException, ClassNotFoundException { - super( - "org.h2.Driver", - "jdbc:h2:" + databaseFile.replaceAll("\\.mv\\.db$", "") + ";ACCESS_MODE_DATA=r" - ); + public OSHDBH2(Path databaseFile) { + this(databaseFile, "sa", ""); } - public OSHDBH2(Connection conn) throws ClassNotFoundException, SQLException { - super(conn); + public OSHDBH2(String databaseFile) { + this(Path.of(databaseFile)); } - @Override - public OSHDBH2 prefix(String prefix) { - return (OSHDBH2) super.prefix(prefix); + public OSHDBH2(String url, String user, String password) { + this(JdbcConnectionPool.create(url, user, password)); } - @Override - public OSHDBH2 multithreading(boolean useMultithreading) { - return (OSHDBH2) super.multithreading(useMultithreading); + public OSHDBH2(Path path, String user, String password) { + this(pathToUrl(path), user, password); } - /** - * Creates an in-memory copy of the current oshdb data (using a volatile in-memory H2 database), - * for faster subsequent queries. - * - *

The original database connection will be closed during this process.

- * - *

Note that once the data has been cached in memory, this cannot be undone anymore by calling - * this method like `.inMemory(false)`.

- * - * @param cache wether in-memory caching should be activated or not - * @return an OSHDBDatabase using the cached in-memory copy of the oshdb data - * @throws SQLException if there's a problem while copying the data into memory - */ - public OSHDBH2 inMemory(boolean cache) throws SQLException { - if (!cache) { - return this; - } + public OSHDBH2(DataSource ds) { + super(ds); + } - try (Connection src = this.getConnection()) { - this.connection = DriverManager.getConnection("jdbc:h2:mem:"); - try ( - Statement srcStmt = src.createStatement(); - ResultSet srcRst = srcStmt.executeQuery("script nodata") - ) { - try (Statement destStmt = this.connection.createStatement()) { - while (srcRst.next()) { - destStmt.executeUpdate(srcRst.getString(1)); - } - } - } + private OSHDBH2(JdbcConnectionPool ds) { + super(ds); + this.connectionPool = ds; + } - Consumer copyData = tablename -> { - try (Statement srcStmt = src.createStatement()) { - List columnNames = columnNames(tablename, srcStmt); - String columns = columnNames.stream().collect(Collectors.joining(", ")); - String insertSql = insertSql(tablename, columnNames, columns); - String querySql = String.format("select %s from %s", columns, tablename); - try (var destStmt = this.connection.prepareStatement(insertSql); - var rs = srcStmt.executeQuery(querySql)) { - while (rs.next()) { - for (int i = 1; i <= columnNames.size(); i++) { - destStmt.setObject(i, rs.getObject(i)); - } - destStmt.execute(); - } - } - } catch (SQLException e) { - throw new OSHDBException(e); - } - }; + private static String pathToUrl(Path path) { + var absolutePath = path.toAbsolutePath().toString(); + absolutePath = absolutePath.replaceAll("\\.mv\\.db$", ""); + return String.format("jdbc:h2:%s;ACCESS_MODE_DATA=r", absolutePath); + } - try (Statement srcStmt = src.createStatement()) { - ResultSet rs = srcStmt.executeQuery("show tables"); - while (rs.next()) { - String tableName = rs.getString(1).toLowerCase(); - // we only need to cache tables that match the currently selected table prefix - Set tableNames = Stream.of(TableNames.values()) - .map(x -> x.toString(this.prefix())) - .map(String::toLowerCase) - .collect(Collectors.toSet()); - if (tableNames.contains(tableName)) { - copyData.accept(tableName); - } - } - } - } - return this; + @Override + public OSHDBH2 prefix(String prefix) { + return (OSHDBH2) super.prefix(prefix); } - private String insertSql(String tablename, List columnNames, String columns) { - String placeholders = columnNames.stream() - .map(ignored -> "?") - .collect(Collectors.joining(", ")); - return "insert into " + tablename + "(" + columns + ") values (" + placeholders + ")"; + @Override + public OSHDBH2 multithreading(boolean useMultithreading) { + return (OSHDBH2) super.multithreading(useMultithreading); } - private List columnNames(String tablename, Statement srcStmt) throws SQLException { - var columnNames = new ArrayList(); - var query = String.format("show columns from %s", tablename); - try (var rs = srcStmt.executeQuery(query)) { - while (rs.next()) { - columnNames.add(rs.getString(1)); - } + @Override + public void close() throws Exception { + try { + connectionPool.dispose(); + } finally { + super.close(); } - return columnNames; } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 84a323043..757a2f9a0 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -2,8 +2,6 @@ import com.google.common.base.Joiner; import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Collection; import java.util.LinkedList; @@ -11,6 +9,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerJdbcMultithread; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerJdbcSinglethread; @@ -23,24 +22,13 @@ /** * OSHDB database backend connector to a JDBC database file. */ -public class OSHDBJdbc extends OSHDBDatabase implements AutoCloseable { +public class OSHDBJdbc extends OSHDBDatabase { - protected Connection connection; + protected final DataSource dataSource; private boolean useMultithreading = true; - public OSHDBJdbc(String classToLoad, String jdbcString) - throws SQLException, ClassNotFoundException { - this(classToLoad, jdbcString, "sa", ""); - } - - public OSHDBJdbc(String classToLoad, String jdbcString, String user, String pw) - throws SQLException, ClassNotFoundException { - Class.forName(classToLoad); - this.connection = DriverManager.getConnection(jdbcString, user, pw); - } - - public OSHDBJdbc(Connection conn) { - this.connection = conn; + public OSHDBJdbc(DataSource source) { + this.dataSource = source; } @Override @@ -51,18 +39,19 @@ public OSHDBJdbc prefix(String prefix) { @Override public MapReducer createMapReducer(Class forClass) { try { - Collection expectedTables = Stream.of(OSMType.values()) - .map(TableNames::forOSMType).filter(Optional::isPresent).map(Optional::get) - .map(t -> t.toString(this.prefix()).toLowerCase()) - .collect(Collectors.toList()); + Collection expectedTables = Stream.of(OSMType.values()).map(TableNames::forOSMType) + .filter(Optional::isPresent).map(Optional::get) + .map(t -> t.toString(this.prefix()).toLowerCase()).collect(Collectors.toList()); List allTables = new LinkedList<>(); - var metaData = getConnection().getMetaData(); - try (var rs = metaData.getTables(null, null, "%", new String[]{"TABLE"})) { - while (rs.next()) { - allTables.add(rs.getString("TABLE_NAME").toLowerCase()); - } - if (!allTables.containsAll(expectedTables)) { - throw new OSHDBTableNotFoundException(Joiner.on(", ").join(expectedTables)); + try (var conn = getConnection()) { + var metaData = conn.getMetaData(); + try (var rs = metaData.getTables(null, null, "%", new String[] {"TABLE"})) { + while (rs.next()) { + allTables.add(rs.getString("TABLE_NAME").toLowerCase()); + } + if (!allTables.containsAll(expectedTables)) { + throw new OSHDBTableNotFoundException(Joiner.on(", ").join(expectedTables)); + } } } } catch (SQLException e) { @@ -82,21 +71,22 @@ public MapReducer createMapReducer(Class for public String metadata(String property) { var table = TableNames.T_METADATA.toString(this.prefix()); var selectSql = String.format("select value from %s where key=?", table); - try (PreparedStatement stmt = connection.prepareStatement(selectSql)) { + try (var conn = getConnection(); + var stmt = conn.prepareStatement(selectSql)) { stmt.setString(1, property); try (var result = stmt.executeQuery()) { if (result.next()) { return result.getString(1); } + return null; } } catch (SQLException e) { throw new OSHDBException(e); } - return null; } - public Connection getConnection() { - return this.connection; + public Connection getConnection() throws SQLException { + return dataSource.getConnection(); } public OSHDBJdbc multithreading(boolean useMultithreading) { @@ -110,6 +100,5 @@ public boolean multithreading() { @Override public void close() throws Exception { - this.connection.close(); } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index d50b594be..4cd425a3e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -214,10 +214,10 @@ protected MapReducer(MapReducer obj) { @Contract(pure = true) public MapReducer keytables(OSHDBJdbc keytables) { if (keytables != this.oshdb && this.oshdb instanceof OSHDBJdbc) { - Connection c = ((OSHDBJdbc) this.oshdb).getConnection(); boolean oshdbContainsKeytables = true; try { - new TagTranslator(c).close(); + Connection c = ((OSHDBJdbc) this.oshdb).getConnection(); + new TagTranslator(c).close(); } catch (OSHDBKeytablesNotFoundException e) { // this is the expected path -> the oshdb doesn't have the key tables oshdbContainsKeytables = false; @@ -1695,7 +1695,7 @@ protected TagTranslator getTagTranslator() { throw new OSHDBKeytablesNotFoundException(); } this.tagTranslator = new TagTranslator(this.keytables.getConnection()); - } catch (OSHDBKeytablesNotFoundException e) { + } catch (OSHDBKeytablesNotFoundException | SQLException e) { LOG.error(e.getMessage()); throw new RuntimeException(e); } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java index 872e33368..fdede0ce7 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.ObjectInputStream; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Iterator; @@ -20,6 +19,7 @@ import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.util.TableNames; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; @@ -48,22 +48,6 @@ public boolean isActive() { return true; } - protected ResultSet getOshCellsRawDataFromDb(CellIdRange cellIdRange) - throws SQLException { - String sqlQuery = this.typeFilter.stream() - .map(osmType -> - TableNames.forOSMType(osmType).map(tn -> tn.toString(this.oshdb.prefix())) - ) - .filter(Optional::isPresent).map(Optional::get) - .map(tn -> "(select data from " + tn + " where level = ?1 and id between ?2 and ?3)") - .collect(Collectors.joining(" union all ")); - PreparedStatement pstmt = ((OSHDBJdbc) this.oshdb).getConnection().prepareStatement(sqlQuery); - pstmt.setInt(1, cellIdRange.getStart().getZoomLevel()); - pstmt.setLong(2, cellIdRange.getStart().getId()); - pstmt.setLong(3, cellIdRange.getEnd().getId()); - return pstmt.executeQuery(); - } - /** * Returns data of one cell from the raw data stream. */ @@ -73,46 +57,72 @@ protected GridOSHEntity readOshCellRawData(ResultSet oshCellsRawData) (new ObjectInputStream(oshCellsRawData.getBinaryStream(1))).readObject(); } + protected String sqlQuery() { + var sqlQuery = this.typeFilter.stream() + .map( + osmType -> TableNames.forOSMType(osmType).map(tn -> tn.toString(this.oshdb.prefix()))) + .filter(Optional::isPresent).map(Optional::get) + .map(tn -> "(select data from " + tn + " where level = ?1 and id between ?2 and ?3)") + .collect(Collectors.joining(" union all ")); + return sqlQuery; + } + @Nonnull protected Stream getOshCellsStream(CellIdRange cellIdRange) { try { if (this.typeFilter.isEmpty()) { return Stream.empty(); } - ResultSet oshCellsRawData = getOshCellsRawDataFromDb(cellIdRange); - if (!oshCellsRawData.next()) { - return Stream.empty(); - } + + var conn = ((OSHDBJdbc) this.oshdb).getConnection(); + var pstmt = conn.prepareStatement(sqlQuery()); + pstmt.setInt(1, cellIdRange.getStart().getZoomLevel()); + pstmt.setLong(2, cellIdRange.getStart().getId()); + pstmt.setLong(3, cellIdRange.getEnd().getId()); + var oshCellsRawData = pstmt.executeQuery(); return StreamSupport.stream(Spliterators.spliteratorUnknownSize( new Iterator() { + GridOSHEntity next; @Override public boolean hasNext() { - try { - return !oshCellsRawData.isClosed(); - } catch (SQLException e) { - throw new RuntimeException(e); - } + return next != null || (next = getNext()) != null; } @Override public GridOSHEntity next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + var grid = next; + next = null; + return grid; + } + + private GridOSHEntity getNext() { try { - if (!hasNext()) { - throw new NoSuchElementException(); - } - GridOSHEntity data = readOshCellRawData(oshCellsRawData); if (!oshCellsRawData.next()) { - oshCellsRawData.close(); + try { + oshCellsRawData.close(); + } finally { + conn.close(); + } + return null; } - return data; + return readOshCellRawData(oshCellsRawData); } catch (IOException | ClassNotFoundException | SQLException e) { - throw new RuntimeException(e); + var exception = new OSHDBException(e); + try { + conn.close(); + } catch (Exception e2) { + exception.addSuppressed(e2); + } + throw exception; } } }, 0 ), false); } catch (SQLException e) { - throw new RuntimeException(e); + throw new OSHDBException(e); } } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java index c8dcc1fd1..756c3f0e6 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java @@ -2,11 +2,11 @@ import com.google.common.collect.Streams; import java.io.IOException; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; +import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; @@ -66,14 +66,17 @@ private S reduce( return result; } for (CellIdRange cellIdRange : this.getCellIdRanges()) { - ResultSet oshCellsRawData = getOshCellsRawDataFromDb(cellIdRange); - - while (oshCellsRawData.next()) { - GridOSHEntity oshCellRawData = readOshCellRawData(oshCellsRawData); - result = combiner.apply( - result, - cellProcessor.apply(oshCellRawData, cellIterator) - ); + try (var conn = ((OSHDBJdbc) this.oshdb).getConnection(); + var pstmt = conn.prepareStatement(sqlQuery());) { + pstmt.setInt(1, cellIdRange.getStart().getZoomLevel()); + pstmt.setLong(2, cellIdRange.getStart().getId()); + pstmt.setLong(3, cellIdRange.getEnd().getId()); + try (var oshCellsRawData = pstmt.executeQuery()) { + while (oshCellsRawData.next()) { + GridOSHEntity oshCellRawData = readOshCellRawData(oshCellsRawData); + result = combiner.apply(result, cellProcessor.apply(oshCellRawData, cellIterator)); + } + } } } return result; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java index 7d7c75662..9ebe2a9d9 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.Iterables; +import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.SortedMap; From 7e06523c61519536a4ef54801a352df878f6cb9d Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 13:47:13 +0200 Subject: [PATCH 02/34] fix complaining javadoc, add hasTag method to OSHDBTags --- .../heigit/ohsome/oshdb/api/db/OSHDBH2.java | 4 +-- .../oshdb/api/mapreducer/MapReducer.java | 2 +- .../oshdb/filter/GeometryTypeFilter.java | 4 +-- .../ohsome/oshdb/filter/TagFilterEquals.java | 2 +- .../oshdb/filter/TagFilterNotEquals.java | 2 +- .../org/heigit/ohsome/oshdb/OSHDBTags.java | 27 +++++++++++++------ 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java index e2973284d..c0006386e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java @@ -31,8 +31,6 @@ public class OSHDBH2 extends OSHDBJdbc { * * @param databaseFile the file name and path to the H2 database file. (the ".mv.db" file ending * of H2 should be omitted here) - * @throws SQLException if the database couldn't be opened - * @throws ClassNotFoundException if the H2 database driver is not installed on the system */ public OSHDBH2(Path databaseFile) { this(databaseFile, "sa", ""); @@ -78,7 +76,7 @@ public OSHDBH2 multithreading(boolean useMultithreading) { @Override public void close() throws Exception { try { - connectionPool.dispose(); + connectionPool.dispose(); } finally { super.close(); } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index 4cd425a3e..239219440 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -428,7 +428,7 @@ private MapReducer osmTypeInternal(Set typeFilter) { private MapReducer osmTag(OSHDBTag tag) { MapReducer ret = this.copy(); ret.preFilters.add(oshEntity -> oshEntity.hasTagKey(tag.getKey())); - ret.filters.add(osmEntity -> osmEntity.getTags().hasTagValue(tag.getKey(), tag.getValue())); + ret.filters.add(osmEntity -> osmEntity.getTags().hasTag(tag)); return ret; } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java index 69fc25085..f6d118b90 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java @@ -150,8 +150,8 @@ public boolean applyOSM(OSMEntity entity) { OSMMember[] wayNodes = ((OSMWay) entity).getMembers(); return wayNodes.length >= 4 && wayNodes[0].getId() == wayNodes[wayNodes.length - 1].getId(); } else if (osmType == OSMType.RELATION) { - return entity.getTags().hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue()) - || entity.getTags().hasTagValue(typeBoundary.getKey(), typeBoundary.getValue()); + return entity.getTags().hasTag(typeMultipolygon) + || entity.getTags().hasTag(typeBoundary); } } return true; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java index 1f3065dcb..a41bc1f25 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java @@ -21,7 +21,7 @@ public OSHDBTag getTag() { @Override public boolean applyOSM(OSMEntity entity) { - return entity.getTags().hasTagValue(tag.getKey(), tag.getValue()); + return entity.getTags().hasTag(tag); } @Override diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java index 84b444a0a..9b5e212c0 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java @@ -27,7 +27,7 @@ public boolean applyOSH(OSHEntity entity) { @Override public boolean applyOSM(OSMEntity entity) { - return !entity.getTags().hasTagValue(tag.getKey(), tag.getValue()); + return !entity.getTags().hasTag(tag); } @Override diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java index fdb909a0a..62eec5908 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java @@ -41,9 +41,21 @@ public boolean hasTagKey(OSHDBTagKey key) { /** * Test for a certain key/value combination. + * + * @param key the key to search for. + * @param value the value to search for. + * @return true, if key/value combination is present. */ public abstract boolean hasTagValue(int key, int value); + /** + * Test for a certain key/value combination. + * + * @param tag the tag to search for. + * @return true, if tag is present. + */ + public abstract boolean hasTag(OSHDBTag tag); + /** * KV based OSHDBTags. * @@ -125,15 +137,14 @@ public boolean hasTagKeyExcluding(int key, int[] uninterestingValues) { @Override public boolean hasTagValue(int key, int value) { + return hasTag(new OSHDBTag(key, value)); + } + + @Override + public boolean hasTag(OSHDBTag tag) { for (int i = 0; i < tags.length; i += 2) { - if (tags[i] < key) { - continue; - } - if (tags[i] == key) { - return tags[i + 1] == value; - } - if (tags[i] > key) { - return false; + if (tags[i] == tag.getKey() && tags[i + 1] == tag.getValue()) { + return true; } } return false; From ff52c34753b98e34a75965c83881be9702843c1d Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 14:06:51 +0200 Subject: [PATCH 03/34] extract TagTranslator to interface --- .../oshdb/api/mapreducer/MapReducer.java | 5 +- .../oshdb/api/tests/TestOSHDBFilter.java | 6 +- .../ohsome/oshdb/filter/FilterTest.java | 15 +- .../taginterpreter/DefaultTagInterpreter.java | 7 +- .../tagtranslator/DefaultTagTranslator.java | 328 ++++++++++++ .../util/tagtranslator/TagTranslator.java | 496 ++++-------------- .../IterateByContributionTest.java | 6 +- .../util/tagtranslator/TagTranslatorTest.java | 14 +- 8 files changed, 461 insertions(+), 416 deletions(-) create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index 239219440..658660078 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -68,6 +68,7 @@ import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; import org.heigit.ohsome.oshdb.util.taginterpreter.TagInterpreter; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestampList; @@ -217,7 +218,7 @@ public MapReducer keytables(OSHDBJdbc keytables) { boolean oshdbContainsKeytables = true; try { Connection c = ((OSHDBJdbc) this.oshdb).getConnection(); - new TagTranslator(c).close(); + new DefaultTagTranslator(c).close(); } catch (OSHDBKeytablesNotFoundException e) { // this is the expected path -> the oshdb doesn't have the key tables oshdbContainsKeytables = false; @@ -1694,7 +1695,7 @@ protected TagTranslator getTagTranslator() { if (this.keytables == null) { throw new OSHDBKeytablesNotFoundException(); } - this.tagTranslator = new TagTranslator(this.keytables.getConnection()); + this.tagTranslator = new DefaultTagTranslator(this.keytables.getConnection()); } catch (OSHDBKeytablesNotFoundException | SQLException e) { LOG.error(e.getMessage()); throw new RuntimeException(e); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java index c738333a0..ba7364e36 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java @@ -16,7 +16,7 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.junit.jupiter.api.Test; /** @@ -41,7 +41,7 @@ class TestOSHDBFilter { */ TestOSHDBFilter() throws Exception { OSHDBH2 oshdb = new OSHDBH2("../data/test-data"); - filterParser = new FilterParser(new TagTranslator(oshdb.getConnection())); + filterParser = new FilterParser(new DefaultTagTranslator(oshdb.getConnection())); this.oshdb = oshdb; } @@ -125,7 +125,7 @@ void testFilterGroupByEntity() throws Exception { @Test @SuppressWarnings("ResultOfMethodCallIgnored") void testFilterNonExistentTag() throws Exception { - FilterParser parser = new FilterParser(new TagTranslator(oshdb.getConnection())); + FilterParser parser = new FilterParser(new DefaultTagTranslator(oshdb.getConnection())); try { createMapReducerOSMEntitySnapshot() .filter(parser.parse("type:way and nonexistentkey=*")) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java index e56abefbb..4ab5ae458 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java @@ -1,6 +1,7 @@ package org.heigit.ohsome.oshdb.filter; import java.io.IOException; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; @@ -19,6 +20,7 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -27,22 +29,23 @@ * Tests the parsing of filters and the application to OSM entities. */ abstract class FilterTest { + protected Connection conn; protected FilterParser parser; protected TagTranslator tagTranslator; @BeforeEach void setup() throws SQLException, ClassNotFoundException, OSHDBKeytablesNotFoundException { - Class.forName("org.h2.Driver"); - this.tagTranslator = new TagTranslator(DriverManager.getConnection( - "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", - "sa", "" - )); + this.conn = DriverManager.getConnection( + "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", + "sa", "" + ); + this.tagTranslator = new DefaultTagTranslator(conn); this.parser = new FilterParser(this.tagTranslator); } @AfterEach void teardown() throws SQLException { - this.tagTranslator.getConnection().close(); + conn.close(); } protected int[] createTestTags(String... keyValues) { diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java index 037757b1c..96257818e 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java @@ -10,6 +10,7 @@ import java.util.TreeSet; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.json.simple.JSONArray; @@ -35,7 +36,7 @@ public class DefaultTagInterpreter extends BaseTagInterpreter { = "json/uninterestingTags.json"; /** - * Constructor using given {@link TagTranslator} and default values as areaTagsDefinitonFile and + * Constructor using given {@link DefaultTagTranslator} and default values as areaTagsDefinitonFile and * uninterestingTagsDefinitionFile. * *

@@ -52,10 +53,10 @@ public DefaultTagInterpreter(TagTranslator tagTranslator) throws IOException, Pa } /** - * Constructor using given {@link TagTranslator}, areaTagsDefinitonFile, and + * Constructor using given {@link DefaultTagTranslator}, areaTagsDefinitonFile, and * uninterestingTagsDefinitionFile. * - * @param tagTranslator {@link TagTranslator} used by {@link TagInterpreter} + * @param tagTranslator {@link DefaultTagTranslator} used by {@link TagInterpreter} * @param areaTagsDefinitionFile filename of a JSON file containing tags that are supposed to be * areas * @param uninterestingTagsDefinitionFile filename of a JSON file containing tags to be ignored diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java new file mode 100644 index 000000000..72f918bbe --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java @@ -0,0 +1,328 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +import static java.lang.String.format; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; +import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.EnumSet; +import java.util.concurrent.ConcurrentHashMap; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; +import org.heigit.ohsome.oshdb.util.TableNames; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Easily translate your textual tags and roles to OSHDB's internal + * representation (encoded as integers) and vice versa. + * + *

This class handles missing/not-found data in the following ways:

+ *
    + *
  • + * for (tag/role) strings that cannot be found in a keytable, the tagtranslator will generate a + * (temporary, internal and negative) id which can afterwards be resolved back to the input + * string when using the same tagtranslator object. + *
    • + *
  • + * for ids that are not found neither in the keytable nor the tagtranslator's cache, a runtime + * exception is thrown + *
    • + *
+ */ +public class DefaultTagTranslator implements TagTranslator { + private static final Logger LOG = LoggerFactory.getLogger(DefaultTagTranslator.class); + private static final String UNABLE_TO_ACCESS_KEYTABLES = "Unable to access keytables"; + + private final PreparedStatement keyIdQuery; + private final PreparedStatement keyTxtQuery; + private final PreparedStatement valueIdQuery; + private final PreparedStatement valueTxtQuery; + private final PreparedStatement roleIdQuery; + private final PreparedStatement roleTxtQuery; + + private final ConcurrentHashMap keyToInt; + private final ConcurrentHashMap keyToString; + private final ConcurrentHashMap tagToInt; + private final ConcurrentHashMap tagToString; + private final ConcurrentHashMap roleToInt; + private final ConcurrentHashMap roleToString; + + private final Connection conn; + + /** + * A TagTranslator for a specific DB-Connection. It has its own internal cache + * to speed up searching. + * + * @param conn a connection to a database (containing oshdb keytables). + * @throws OSHDBKeytablesNotFoundException if the supplied database doesn't contain the required + * "keyTables" tables + */ + public DefaultTagTranslator(Connection conn) throws OSHDBKeytablesNotFoundException { + this.conn = conn; + this.keyToInt = new ConcurrentHashMap<>(0); + this.keyToString = new ConcurrentHashMap<>(0); + this.tagToInt = new ConcurrentHashMap<>(0); + this.tagToString = new ConcurrentHashMap<>(0); + this.roleToInt = new ConcurrentHashMap<>(0); + this.roleToString = new ConcurrentHashMap<>(0); + + // test connection for presence of actual "keytables" tables + EnumSet keyTables = + EnumSet.of(E_KEY, E_KEYVALUE, E_ROLE); + for (TableNames table : keyTables) { + try (Statement testTablePresentQuery = this.conn.createStatement()) { + var selectSql = format("select 1 from %s limit 1", table); + testTablePresentQuery.execute(selectSql); + } catch (SQLException e) { + throw new OSHDBKeytablesNotFoundException(); + } + } + + // create prepared statements for querying tags from keytables + try { + keyIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_KEY)); + keyTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_KEY)); + valueIdQuery = conn.prepareStatement(format("select k.ID as KEYID,kv.VALUEID as VALUEID" + + " from %s kv" + + " inner join %s k on k.ID = kv.KEYID" + + " where k.TXT = ? and kv.TXT = ?;", E_KEYVALUE, E_KEY)); + valueTxtQuery = conn.prepareStatement(format("select k.TXT as KEYTXT,kv.TXT as VALUETXT" + + " from %s kv" + + " inner join %s k on k.ID = kv.KEYID" + + " where k.ID = ? and kv.VALUEID = ?;", E_KEYVALUE, E_KEY)); + roleIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_ROLE)); + roleTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_ROLE)); + } catch (SQLException e) { + throw new OSHDBKeytablesNotFoundException(); + } + } + + @Override + public void close() throws SQLException { + keyIdQuery.close(); + keyTxtQuery.close(); + valueIdQuery.close(); + valueTxtQuery.close(); + roleIdQuery.close(); + roleTxtQuery.close(); + } + + @Override + public OSHDBTagKey getOSHDBTagKeyOf(String key) { + return getOSHDBTagKeyOf(new OSMTagKey(key)); + } + + @Override + public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { + if (this.keyToInt.containsKey(key)) { + return this.keyToInt.get(key); + } + OSHDBTagKey keyInt; + try { + synchronized (keyIdQuery) { + keyIdQuery.setString(1, key.toString()); + try (ResultSet keys = keyIdQuery.executeQuery()) { + if (!keys.next()) { + LOG.debug("Unable to find tag key {} in keytables.", key); + keyInt = new OSHDBTagKey(getFakeId(key.toString())); + } else { + keyInt = new OSHDBTagKey(keys.getInt("ID")); + } + } + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + this.keyToString.put(keyInt, key); + this.keyToInt.put(key, keyInt); + return keyInt; + } + + @Override + public OSMTagKey getOSMTagKeyOf(int key) { + return getOSMTagKeyOf(new OSHDBTagKey(key)); + } + + @Override + public OSMTagKey getOSMTagKeyOf(OSHDBTagKey key) { + if (this.keyToString.containsKey(key)) { + return this.keyToString.get(key); + } + OSMTagKey keyString; + try { + synchronized (keyTxtQuery) { + keyTxtQuery.setInt(1, key.toInt()); + try (ResultSet keys = keyTxtQuery.executeQuery()) { + if (!keys.next()) { + throw new OSHDBTagOrRoleNotFoundException(format( + "Unable to find tag key id %d in keytables.", key.toInt() + )); + } else { + keyString = new OSMTagKey(keys.getString("TXT")); + } + } + this.keyToInt.put(keyString, key); + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + this.keyToString.put(key, keyString); + return keyString; + } + + @Override + public OSHDBTag getOSHDBTagOf(String key, String value) { + return this.getOSHDBTagOf(new OSMTag(key, value)); + } + + @Override + public OSHDBTag getOSHDBTagOf(OSMTag tag) { + // check if Key and Value are in cache + if (this.tagToInt.containsKey(tag)) { + return this.tagToInt.get(tag); + } + OSHDBTag tagInt; + // key or value is not in cache so let's go toInt them + try { + synchronized (valueIdQuery) { + valueIdQuery.setString(1, tag.getKey()); + valueIdQuery.setString(2, tag.getValue()); + try (ResultSet values = valueIdQuery.executeQuery()) { + if (!values.next()) { + LOG.info("Unable to find tag {}={} in keytables.", tag.getKey(), tag.getValue()); + tagInt = new OSHDBTag( + this.getOSHDBTagKeyOf(tag.getKey()).toInt(), getFakeId(tag.getValue())); + } else { + tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); + } + } + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + this.tagToString.put(tagInt, tag); + this.tagToInt.put(tag, tagInt); + return tagInt; + } + + @Override + public OSMTag getOSMTagOf(int key, int value) { + return this.getOSMTagOf(new OSHDBTag(key, value)); + } + + @Override + public OSMTag getOSMTagOf(OSHDBTag tag) { + // check if Key and Value are in cache + if (this.tagToString.containsKey(tag)) { + return this.tagToString.get(tag); + } + OSMTag tagString; + + // key or value is not in cache so let's go toInt them + try { + synchronized (valueTxtQuery) { + valueTxtQuery.setInt(1, tag.getKey()); + valueTxtQuery.setInt(2, tag.getValue()); + try (ResultSet values = valueTxtQuery.executeQuery()) { + if (!values.next()) { + throw new OSHDBTagOrRoleNotFoundException(format( + "Unable to find tag id %d=%d in keytables.", + tag.getKey(), tag.getValue() + )); + } else { + tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); + } + } + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + // put it in caches + this.tagToInt.put(tagString, tag); + this.tagToString.put(tag, tagString); + return tagString; + } + + @Override + public OSHDBRole getOSHDBRoleOf(String role) { + return this.getOSHDBRoleOf(new OSMRole(role)); + } + + @Override + public OSHDBRole getOSHDBRoleOf(OSMRole role) { + if (this.roleToInt.containsKey(role)) { + return this.roleToInt.get(role); + } + OSHDBRole roleInt; + try { + synchronized (roleIdQuery) { + roleIdQuery.setString(1, role.toString()); + try (ResultSet roles = roleIdQuery.executeQuery()) { + if (!roles.next()) { + LOG.info("Unable to find role {} in keytables.", role); + roleInt = OSHDBRole.of(getFakeId(role.toString())); + } else { + roleInt = OSHDBRole.of(roles.getInt("ID")); + } + } + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + this.roleToString.put(roleInt, role); + this.roleToInt.put(role, roleInt); + return roleInt; + } + + @Override + public OSMRole getOSMRoleOf(int role) { + return this.getOSMRoleOf(OSHDBRole.of(role)); + } + + @Override + public OSMRole getOSMRoleOf(OSHDBRole role) { + if (this.roleToString.containsKey(role)) { + return this.roleToString.get(role); + } + OSMRole roleString; + try { + synchronized (roleTxtQuery) { + roleTxtQuery.setInt(1, role.getId()); + try (ResultSet roles = roleTxtQuery.executeQuery()) { + if (!roles.next()) { + throw new OSHDBTagOrRoleNotFoundException(format( + "Unable to find role id %d in keytables.", role.getId() + )); + } else { + roleString = new OSMRole(roles.getString("TXT")); + } + } + } + } catch (SQLException ex) { + LOG.error(UNABLE_TO_ACCESS_KEYTABLES); + throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); + } + this.roleToInt.put(roleString, role); + this.roleToString.put(role, roleString); + return roleString; + } + + private int getFakeId(String s) { + return -(s.hashCode() & 0x7fffffff); + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index 3495e6238..153d847eb 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -1,400 +1,112 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; -import static java.lang.String.format; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; -import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.EnumSet; -import java.util.concurrent.ConcurrentHashMap; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; -import org.heigit.ohsome.oshdb.util.TableNames; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Easily translate your textual tags and roles to OSHDB's internal - * representation (encoded as integers) and vice versa. - * - *

This class handles missing/not-found data in the following ways:

- *
    - *
  • - * for (tag/role) strings that cannot be found in a keytable, the tagtranslator will generate a - * (temporary, internal and negative) id which can afterwards be resolved back to the input - * string when using the same tagtranslator object. - *
    • - *
  • - * for ids that are not found neither in the keytable nor the tagtranslator's cache, a runtime - * exception is thrown - *
    • - *
- */ -public class TagTranslator implements AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(TagTranslator.class); - private static final String UNABLE_TO_ACCESS_KEYTABLES = "Unable to access keytables"; - - private final PreparedStatement keyIdQuery; - private final PreparedStatement keyTxtQuery; - private final PreparedStatement valueIdQuery; - private final PreparedStatement valueTxtQuery; - private final PreparedStatement roleIdQuery; - private final PreparedStatement roleTxtQuery; - - private final ConcurrentHashMap keyToInt; - private final ConcurrentHashMap keyToString; - private final ConcurrentHashMap tagToInt; - private final ConcurrentHashMap tagToString; - private final ConcurrentHashMap roleToInt; - private final ConcurrentHashMap roleToString; - - private final Connection conn; - - /** - * A TagTranslator for a specific DB-Connection. It has its own internal cache - * to speed up searching. - * - * @param conn a connection to a database (containing oshdb keytables). - * @throws OSHDBKeytablesNotFoundException if the supplied database doesn't contain the required - * "keyTables" tables - */ - public TagTranslator(Connection conn) throws OSHDBKeytablesNotFoundException { - this.conn = conn; - this.keyToInt = new ConcurrentHashMap<>(0); - this.keyToString = new ConcurrentHashMap<>(0); - this.tagToInt = new ConcurrentHashMap<>(0); - this.tagToString = new ConcurrentHashMap<>(0); - this.roleToInt = new ConcurrentHashMap<>(0); - this.roleToString = new ConcurrentHashMap<>(0); - - // test connection for presence of actual "keytables" tables - EnumSet keyTables = - EnumSet.of(E_KEY, E_KEYVALUE, E_ROLE); - for (TableNames table : keyTables) { - try (Statement testTablePresentQuery = this.conn.createStatement()) { - var selectSql = format("select 1 from %s limit 1", table); - testTablePresentQuery.execute(selectSql); - } catch (SQLException e) { - throw new OSHDBKeytablesNotFoundException(); - } - } - - // create prepared statements for querying tags from keytables - try { - keyIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_KEY)); - keyTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_KEY)); - valueIdQuery = conn.prepareStatement(format("select k.ID as KEYID,kv.VALUEID as VALUEID" - + " from %s kv" - + " inner join %s k on k.ID = kv.KEYID" - + " where k.TXT = ? and kv.TXT = ?;", E_KEYVALUE, E_KEY)); - valueTxtQuery = conn.prepareStatement(format("select k.TXT as KEYTXT,kv.TXT as VALUETXT" - + " from %s kv" - + " inner join %s k on k.ID = kv.KEYID" - + " where k.ID = ? and kv.VALUEID = ?;", E_KEYVALUE, E_KEY)); - roleIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_ROLE)); - roleTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_ROLE)); - } catch (SQLException e) { - throw new OSHDBKeytablesNotFoundException(); - } - } - - @Override - public void close() throws SQLException { - keyIdQuery.close(); - keyTxtQuery.close(); - valueIdQuery.close(); - valueTxtQuery.close(); - roleIdQuery.close(); - roleTxtQuery.close(); - } - - /** - * Get oshdb's internal representation of a tag key (string). - * - * @param key the tag key as a string - * @return the corresponding oshdb representation of this key - */ - public OSHDBTagKey getOSHDBTagKeyOf(String key) { - return getOSHDBTagKeyOf(new OSMTagKey(key)); - } - - /** - * Get oshdb's internal representation of a tag key. - * - * @param key the tag key as an OSMTagKey object - * @return the corresponding oshdb representation of this key - */ - public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { - if (this.keyToInt.containsKey(key)) { - return this.keyToInt.get(key); - } - OSHDBTagKey keyInt; - try { - synchronized (keyIdQuery) { - keyIdQuery.setString(1, key.toString()); - try (ResultSet keys = keyIdQuery.executeQuery()) { - if (!keys.next()) { - LOG.debug("Unable to find tag key {} in keytables.", key); - keyInt = new OSHDBTagKey(getFakeId(key.toString())); - } else { - keyInt = new OSHDBTagKey(keys.getInt("ID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.keyToString.put(keyInt, key); - this.keyToInt.put(key, keyInt); - return keyInt; - } - - /** - * Get a tag key's string representation from oshdb's internal data format. - * - * @param key the tag key (represented as an integer) - * @return the textual representation of this tag key - * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found - */ - public OSMTagKey getOSMTagKeyOf(int key) { - return getOSMTagKeyOf(new OSHDBTagKey(key)); - } - - /** - * Get a tag key's string representation from oshdb's internal data format. - * - * @param key the tag key (as an OSHDBTagKey object) - * @return the textual representation of this tag key - * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found - */ - public OSMTagKey getOSMTagKeyOf(OSHDBTagKey key) { - if (this.keyToString.containsKey(key)) { - return this.keyToString.get(key); - } - OSMTagKey keyString; - try { - synchronized (keyTxtQuery) { - keyTxtQuery.setInt(1, key.toInt()); - try (ResultSet keys = keyTxtQuery.executeQuery()) { - if (!keys.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find tag key id %d in keytables.", key.toInt() - )); - } else { - keyString = new OSMTagKey(keys.getString("TXT")); - } - } - this.keyToInt.put(keyString, key); - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.keyToString.put(key, keyString); - return keyString; - } - - /** - * Get oshdb's internal representation of a tag (key-value string pair). - * - * @param key the (string) key of the tag - * @param value the (string) value of the tag - * @return the corresponding oshdb representation of this tag - */ - public OSHDBTag getOSHDBTagOf(String key, String value) { - return this.getOSHDBTagOf(new OSMTag(key, value)); - } - - /** - * Get oshdb's internal representation of a tag (key-value pair). - * - * @param tag a key-value pair as an OSMTag object - * @return the corresponding oshdb representation of this tag - */ - public OSHDBTag getOSHDBTagOf(OSMTag tag) { - // check if Key and Value are in cache - if (this.tagToInt.containsKey(tag)) { - return this.tagToInt.get(tag); - } - OSHDBTag tagInt; - // key or value is not in cache so let's go toInt them - try { - synchronized (valueIdQuery) { - valueIdQuery.setString(1, tag.getKey()); - valueIdQuery.setString(2, tag.getValue()); - try (ResultSet values = valueIdQuery.executeQuery()) { - if (!values.next()) { - LOG.info("Unable to find tag {}={} in keytables.", tag.getKey(), tag.getValue()); - tagInt = new OSHDBTag( - this.getOSHDBTagKeyOf(tag.getKey()).toInt(), getFakeId(tag.getValue())); - } else { - tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.tagToString.put(tagInt, tag); - this.tagToInt.put(tag, tagInt); - return tagInt; - } - - /** - * Get a tag's string representation from oshdb's internal data format. - * - * @param key the key of the tag (represented as an integer) - * @param value the value of the tag (represented as an integer) - * @return the textual representation of this tag - * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found - */ - public OSMTag getOSMTagOf(int key, int value) { - return this.getOSMTagOf(new OSHDBTag(key, value)); - } - - /** - * Get a tag's string representation from oshdb's internal data format. - * - * @param tag the tag (as an OSHDBTag object) - * @return the textual representation of this tag - * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found - */ - public OSMTag getOSMTagOf(OSHDBTag tag) { - // check if Key and Value are in cache - if (this.tagToString.containsKey(tag)) { - return this.tagToString.get(tag); - } - OSMTag tagString; - - // key or value is not in cache so let's go toInt them - try { - synchronized (valueTxtQuery) { - valueTxtQuery.setInt(1, tag.getKey()); - valueTxtQuery.setInt(2, tag.getValue()); - try (ResultSet values = valueTxtQuery.executeQuery()) { - if (!values.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find tag id %d=%d in keytables.", - tag.getKey(), tag.getValue() - )); - } else { - tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - // put it in caches - this.tagToInt.put(tagString, tag); - this.tagToString.put(tag, tagString); - return tagString; - } - - /** - * Get oshdb's internal representation of a role (string). - * - * @param role the role string to fetch - * @return the corresponding oshdb representation of this role - */ - public OSHDBRole getOSHDBRoleOf(String role) { - return this.getOSHDBRoleOf(new OSMRole(role)); - } - - /** - * Get oshdb's internal representation of a role. - * - * @param role the role to fetch as an OSMRole object - * @return the corresponding oshdb representation of this role - */ - public OSHDBRole getOSHDBRoleOf(OSMRole role) { - if (this.roleToInt.containsKey(role)) { - return this.roleToInt.get(role); - } - OSHDBRole roleInt; - try { - synchronized (roleIdQuery) { - roleIdQuery.setString(1, role.toString()); - try (ResultSet roles = roleIdQuery.executeQuery()) { - if (!roles.next()) { - LOG.info("Unable to find role {} in keytables.", role); - roleInt = OSHDBRole.of(getFakeId(role.toString())); - } else { - roleInt = OSHDBRole.of(roles.getInt("ID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.roleToString.put(roleInt, role); - this.roleToInt.put(role, roleInt); - return roleInt; - } - - /** - * Get a role's string representation from oshdb's internal data format. - * - * @param role the role ID (represented as an integer) - * @return the textual representation of this role - * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found - */ - public OSMRole getOSMRoleOf(int role) { - return this.getOSMRoleOf(OSHDBRole.of(role)); - } - - /** - * Get a role's string representation from oshdb's internal data format. - * - * @param role the role ID (as an OSHDBRole object) - * @return the textual representation of this role - * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found - */ - public OSMRole getOSMRoleOf(OSHDBRole role) { - if (this.roleToString.containsKey(role)) { - return this.roleToString.get(role); - } - OSMRole roleString; - try { - synchronized (roleTxtQuery) { - roleTxtQuery.setInt(1, role.getId()); - try (ResultSet roles = roleTxtQuery.executeQuery()) { - if (!roles.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find role id %d in keytables.", role.getId() - )); - } else { - roleString = new OSMRole(roles.getString("TXT")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.roleToInt.put(roleString, role); - this.roleToString.put(role, roleString); - return roleString; - } - - private int getFakeId(String s) { - return -(s.hashCode() & 0x7fffffff); - } - public Connection getConnection() { - return conn; - } +public interface TagTranslator extends AutoCloseable { + /** + * Get oshdb's internal representation of a tag key (string). + * + * @param key the tag key as a string + * @return the corresponding oshdb representation of this key + */ + OSHDBTagKey getOSHDBTagKeyOf(String key); + + /** + * Get oshdb's internal representation of a tag key. + * + * @param key the tag key as an OSMTagKey object + * @return the corresponding oshdb representation of this key + */ + OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key); + + /** + * Get a tag key's string representation from oshdb's internal data format. + * + * @param key the tag key (represented as an integer) + * @return the textual representation of this tag key + * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found + */ + OSMTagKey getOSMTagKeyOf(int key); + + /** + * Get a tag key's string representation from oshdb's internal data format. + * + * @param key the tag key (as an OSHDBTagKey object) + * @return the textual representation of this tag key + * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found + */ + OSMTagKey getOSMTagKeyOf(OSHDBTagKey key); + + /** + * Get oshdb's internal representation of a tag (key-value string pair). + * + * @param key the (string) key of the tag + * @param value the (string) value of the tag + * @return the corresponding oshdb representation of this tag + */ + OSHDBTag getOSHDBTagOf(String key, String value); + + /** + * Get oshdb's internal representation of a tag (key-value pair). + * + * @param tag a key-value pair as an OSMTag object + * @return the corresponding oshdb representation of this tag + */ + OSHDBTag getOSHDBTagOf(OSMTag tag); + + /** + * Get a tag's string representation from oshdb's internal data format. + * + * @param key the key of the tag (represented as an integer) + * @param value the value of the tag (represented as an integer) + * @return the textual representation of this tag + * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found + */ + OSMTag getOSMTagOf(int key, int value); + + /** + * Get a tag's string representation from oshdb's internal data format. + * + * @param tag the tag (as an OSHDBTag object) + * @return the textual representation of this tag + * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found + */ + OSMTag getOSMTagOf(OSHDBTag tag); + + /** + * Get oshdb's internal representation of a role (string). + * + * @param role the role string to fetch + * @return the corresponding oshdb representation of this role + */ + OSHDBRole getOSHDBRoleOf(String role); + + /** + * Get oshdb's internal representation of a role. + * + * @param role the role to fetch as an OSMRole object + * @return the corresponding oshdb representation of this role + */ + OSHDBRole getOSHDBRoleOf(OSMRole role); + + /** + * Get a role's string representation from oshdb's internal data format. + * + * @param role the role ID (represented as an integer) + * @return the textual representation of this role + * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found + */ + OSMRole getOSMRoleOf(int role); + + /** + * Get a role's string representation from oshdb's internal data format. + * + * @param role the role ID (as an OSHDBRole object) + * @return the textual representation of this role + * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found + */ + OSMRole getOSMRoleOf(OSHDBRole role); } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java index 1b1a64d37..25ef48a14 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java @@ -18,6 +18,7 @@ import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.json.simple.parser.ParseException; import org.junit.jupiter.api.AfterAll; @@ -55,8 +56,7 @@ static void breakDownClass() throws SQLException { @SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection"}) @Test - void testIssue108() throws SQLException, IOException, ClassNotFoundException, - ParseException, OSHDBKeytablesNotFoundException { + void testIssue108() throws Exception { ResultSet oshCellsRawData = conn.prepareStatement( "select data from " + TableNames.T_NODES).executeQuery(); @@ -64,7 +64,7 @@ void testIssue108() throws SQLException, IOException, ClassNotFoundException, int countCreated = 0; int countOther = 0; - try (TagTranslator tt = new TagTranslator(conn)) { + try (TagTranslator tt = new DefaultTagTranslator(conn)) { while (oshCellsRawData.next()) { // get one cell from the raw data stream GridOSHEntity oshCellRawData = (GridOSHEntity) (new ObjectInputStream( diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java index f562b90e6..3ed2395bb 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java @@ -14,7 +14,7 @@ import org.junit.jupiter.api.Test; /** - * Tests the {@link TagTranslator} class. + * Tests the {@link DefaultTagTranslator} class. */ class TagTranslatorTest { private static Connection conn; @@ -46,7 +46,7 @@ static void breakDownClass() throws SQLException { @Test void testTag2Int() throws OSHDBKeytablesNotFoundException { OSMTag tag = new OSMTag("building", "yes"); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSHDBTag expResult = new OSHDBTag(1, 0); OSHDBTag result = instance.getOSHDBTagOf(tag); assertEquals(expResult, result); @@ -55,7 +55,7 @@ void testTag2Int() throws OSHDBKeytablesNotFoundException { @Test void testTag2String() throws OSHDBKeytablesNotFoundException { OSHDBTag tag = new OSHDBTag(1, 2); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSMTag expResult = new OSMTag("building", "residential"); OSMTag result = instance.getOSMTagOf(tag); assertEquals(expResult, result); @@ -64,7 +64,7 @@ void testTag2String() throws OSHDBKeytablesNotFoundException { @Test void testKey2Int() throws OSHDBKeytablesNotFoundException { OSMTagKey key = new OSMTagKey("highway"); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSHDBTagKey expResult = new OSHDBTagKey(2); OSHDBTagKey result = instance.getOSHDBTagKeyOf(key); assertEquals(expResult, result); @@ -73,7 +73,7 @@ void testKey2Int() throws OSHDBKeytablesNotFoundException { @Test void testKey2String() throws OSHDBKeytablesNotFoundException { OSHDBTagKey key = new OSHDBTagKey(1); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSMTagKey expResult = new OSMTagKey("building"); OSMTagKey result = instance.getOSMTagKeyOf(key); assertEquals(expResult, result); @@ -82,7 +82,7 @@ void testKey2String() throws OSHDBKeytablesNotFoundException { @Test void testRole2Int() throws OSHDBKeytablesNotFoundException { OSMRole role = new OSMRole("from"); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSHDBRole expResult = OSHDBRole.of(4); OSHDBRole result = instance.getOSHDBRoleOf(role); assertEquals(expResult, result); @@ -91,7 +91,7 @@ void testRole2Int() throws OSHDBKeytablesNotFoundException { @Test void testRole2String() throws OSHDBKeytablesNotFoundException { OSHDBRole role = OSHDBRole.of(1); - TagTranslator instance = new TagTranslator(TagTranslatorTest.conn); + TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSMRole expResult = new OSMRole("inner"); OSMRole result = instance.getOSMRoleOf(role); assertEquals(expResult, result); From 551905f2559fe39df4816abebe7b6d9923e286d0 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 14:50:36 +0200 Subject: [PATCH 04/34] rename oshdb->osm method to lookup --- .../tagtranslator/DefaultTagTranslator.java | 46 +---- .../util/tagtranslator/TagTranslator.java | 169 ++++++++---------- .../util/tagtranslator/TagTranslatorTest.java | 13 +- 3 files changed, 81 insertions(+), 147 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java index 72f918bbe..5dabce3cc 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java @@ -149,39 +149,6 @@ public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { return keyInt; } - @Override - public OSMTagKey getOSMTagKeyOf(int key) { - return getOSMTagKeyOf(new OSHDBTagKey(key)); - } - - @Override - public OSMTagKey getOSMTagKeyOf(OSHDBTagKey key) { - if (this.keyToString.containsKey(key)) { - return this.keyToString.get(key); - } - OSMTagKey keyString; - try { - synchronized (keyTxtQuery) { - keyTxtQuery.setInt(1, key.toInt()); - try (ResultSet keys = keyTxtQuery.executeQuery()) { - if (!keys.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find tag key id %d in keytables.", key.toInt() - )); - } else { - keyString = new OSMTagKey(keys.getString("TXT")); - } - } - this.keyToInt.put(keyString, key); - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.keyToString.put(key, keyString); - return keyString; - } - @Override public OSHDBTag getOSHDBTagOf(String key, String value) { return this.getOSHDBTagOf(new OSMTag(key, value)); @@ -219,12 +186,7 @@ public OSHDBTag getOSHDBTagOf(OSMTag tag) { } @Override - public OSMTag getOSMTagOf(int key, int value) { - return this.getOSMTagOf(new OSHDBTag(key, value)); - } - - @Override - public OSMTag getOSMTagOf(OSHDBTag tag) { + public OSMTag lookupTag(OSHDBTag tag) { // check if Key and Value are in cache if (this.tagToString.containsKey(tag)) { return this.tagToString.get(tag); @@ -290,12 +252,12 @@ public OSHDBRole getOSHDBRoleOf(OSMRole role) { } @Override - public OSMRole getOSMRoleOf(int role) { - return this.getOSMRoleOf(OSHDBRole.of(role)); + public OSMRole lookupRole(int role) { + return this.lookupRole(OSHDBRole.of(role)); } @Override - public OSMRole getOSMRoleOf(OSHDBRole role) { + public OSMRole lookupRole(OSHDBRole role) { if (this.roleToString.containsKey(role)) { return this.roleToString.get(role); } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index 153d847eb..b27d43bc5 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -6,107 +6,88 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; public interface TagTranslator extends AutoCloseable { - /** - * Get oshdb's internal representation of a tag key (string). - * - * @param key the tag key as a string - * @return the corresponding oshdb representation of this key - */ - OSHDBTagKey getOSHDBTagKeyOf(String key); + /** + * Get oshdb's internal representation of a tag key (string). + * + * @param key the tag key as a string + * @return the corresponding oshdb representation of this key + */ + OSHDBTagKey getOSHDBTagKeyOf(String key); - /** - * Get oshdb's internal representation of a tag key. - * - * @param key the tag key as an OSMTagKey object - * @return the corresponding oshdb representation of this key - */ - OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key); + /** + * Get oshdb's internal representation of a tag key. + * + * @param key the tag key as an OSMTagKey object + * @return the corresponding oshdb representation of this key + */ + OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key); - /** - * Get a tag key's string representation from oshdb's internal data format. - * - * @param key the tag key (represented as an integer) - * @return the textual representation of this tag key - * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found - */ - OSMTagKey getOSMTagKeyOf(int key); + /** + * Get oshdb's internal representation of a tag (key-value string pair). + * + * @param key the (string) key of the tag + * @param value the (string) value of the tag + * @return the corresponding oshdb representation of this tag + */ + OSHDBTag getOSHDBTagOf(String key, String value); - /** - * Get a tag key's string representation from oshdb's internal data format. - * - * @param key the tag key (as an OSHDBTagKey object) - * @return the textual representation of this tag key - * @throws OSHDBTagOrRoleNotFoundException if the given tag key cannot be found - */ - OSMTagKey getOSMTagKeyOf(OSHDBTagKey key); + /** + * Get oshdb's internal representation of a tag (key-value pair). + * + * @param tag a key-value pair as an OSMTag object + * @return the corresponding oshdb representation of this tag + */ + OSHDBTag getOSHDBTagOf(OSMTag tag); - /** - * Get oshdb's internal representation of a tag (key-value string pair). - * - * @param key the (string) key of the tag - * @param value the (string) value of the tag - * @return the corresponding oshdb representation of this tag - */ - OSHDBTag getOSHDBTagOf(String key, String value); + /** + * Get oshdb's internal representation of a role (string). + * + * @param role the role string to fetch + * @return the corresponding oshdb representation of this role + */ + OSHDBRole getOSHDBRoleOf(String role); - /** - * Get oshdb's internal representation of a tag (key-value pair). - * - * @param tag a key-value pair as an OSMTag object - * @return the corresponding oshdb representation of this tag - */ - OSHDBTag getOSHDBTagOf(OSMTag tag); + /** + * Get oshdb's internal representation of a role. + * + * @param role the role to fetch as an OSMRole object + * @return the corresponding oshdb representation of this role + */ + OSHDBRole getOSHDBRoleOf(OSMRole role); - /** - * Get a tag's string representation from oshdb's internal data format. - * - * @param key the key of the tag (represented as an integer) - * @param value the value of the tag (represented as an integer) - * @return the textual representation of this tag - * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found - */ - OSMTag getOSMTagOf(int key, int value); + /** + * Get a tag's string representation from oshdb's internal data format. + * + * @param tag the tag (as an OSHDBTag object) + * @return the textual representation of this tag + * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found + */ + OSMTag lookupTag(OSHDBTag tag); - /** - * Get a tag's string representation from oshdb's internal data format. - * - * @param tag the tag (as an OSHDBTag object) - * @return the textual representation of this tag - * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found - */ - OSMTag getOSMTagOf(OSHDBTag tag); + /** + * Get a tag's string representation from oshdb's internal data format. + * + * @param key the key of the tag (represented as an integer) + * @param value the value of the tag (represented as an integer) + * @return the textual representation of this tag + */ + default OSMTag lookupTag(int key, int value) { + return lookupTag(new OSHDBTag(key, value)); + } - /** - * Get oshdb's internal representation of a role (string). - * - * @param role the role string to fetch - * @return the corresponding oshdb representation of this role - */ - OSHDBRole getOSHDBRoleOf(String role); + /** + * Get a role's string representation from oshdb's internal data format. + * + * @param role the role ID (as an OSHDBRole object) + * @return the textual representation of this role + */ + OSMRole lookupRole(OSHDBRole role); - /** - * Get oshdb's internal representation of a role. - * - * @param role the role to fetch as an OSMRole object - * @return the corresponding oshdb representation of this role - */ - OSHDBRole getOSHDBRoleOf(OSMRole role); - - /** - * Get a role's string representation from oshdb's internal data format. - * - * @param role the role ID (represented as an integer) - * @return the textual representation of this role - * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found - */ - OSMRole getOSMRoleOf(int role); - - /** - * Get a role's string representation from oshdb's internal data format. - * - * @param role the role ID (as an OSHDBRole object) - * @return the textual representation of this role - * @throws OSHDBTagOrRoleNotFoundException if the given role cannot be found - */ - OSMRole getOSMRoleOf(OSHDBRole role); + /** + * Get a role's string representation from oshdb's internal data format. + * + * @param role the role ID (represented as an integer) + * @return the textual representation of this role + */ + OSMRole lookupRole(int role); } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java index 3ed2395bb..c1ad30257 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java @@ -57,7 +57,7 @@ void testTag2String() throws OSHDBKeytablesNotFoundException { OSHDBTag tag = new OSHDBTag(1, 2); TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSMTag expResult = new OSMTag("building", "residential"); - OSMTag result = instance.getOSMTagOf(tag); + OSMTag result = instance.lookupTag(tag); assertEquals(expResult, result); } @@ -70,15 +70,6 @@ void testKey2Int() throws OSHDBKeytablesNotFoundException { assertEquals(expResult, result); } - @Test - void testKey2String() throws OSHDBKeytablesNotFoundException { - OSHDBTagKey key = new OSHDBTagKey(1); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSMTagKey expResult = new OSMTagKey("building"); - OSMTagKey result = instance.getOSMTagKeyOf(key); - assertEquals(expResult, result); - } - @Test void testRole2Int() throws OSHDBKeytablesNotFoundException { OSMRole role = new OSMRole("from"); @@ -93,7 +84,7 @@ void testRole2String() throws OSHDBKeytablesNotFoundException { OSHDBRole role = OSHDBRole.of(1); TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); OSMRole expResult = new OSMRole("inner"); - OSMRole result = instance.getOSMRoleOf(role); + OSMRole result = instance.lookupRole(role); assertEquals(expResult, result); } } From adf44c51eaa412a418bfea489dec4a12af368f42 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 16:10:50 +0200 Subject: [PATCH 05/34] change owner ship of tagtranslator to oshdbdatabase, use datasource instead of connection --- .../ohsome/oshdb/api/db/OSHDBIgnite.java | 29 +++++--- .../api/tests/TestMapReduceOSHDBIgnite.java | 22 +++--- .../TestMapReduceOSHDBIgniteAffinityCall.java | 2 +- .../TestMapReduceOSHDBIgniteLocalPeek.java | 2 +- .../TestMapReduceOSHDBIgniteMissingCache.java | 3 +- .../TestMapReduceOSHDBIgniteScanQuery.java | 2 +- .../ohsome/oshdb/api/db/OSHDBDatabase.java | 3 + .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 23 +++++- .../oshdb/api/mapreducer/MapReducer.java | 61 +--------------- .../ohsome/oshdb/api/tests/TestMapReduce.java | 13 +--- .../oshdb/api/tests/TestOSHDBFilter.java | 5 +- .../ohsome/oshdb/filter/FilterTest.java | 18 ++--- .../OSHDBKeytablesNotFoundException.java | 2 +- .../tagtranslator/DefaultTagTranslator.java | 24 ++++--- .../util/tagtranslator/TagTranslator.java | 2 +- .../IterateByContributionTest.java | 27 +++---- .../util/tagtranslator/TagTranslatorTest.java | 71 +++++++++---------- 17 files changed, 138 insertions(+), 171 deletions(-) diff --git a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java index 845aa59c1..caa2c88ba 100644 --- a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java +++ b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java @@ -2,10 +2,12 @@ import com.google.common.base.Joiner; import java.io.File; +import java.sql.SQLException; import java.util.Collection; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.lang.IgniteRunnable; @@ -18,6 +20,8 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** * OSHDB database backend connector to a Ignite system. @@ -40,6 +44,7 @@ public enum ComputeMode { } private final Ignite ignite; + private final OSHDBDatabase keytables; private final boolean owner; private ComputeMode computeMode = ComputeMode.LOCAL_PEEK; @@ -50,8 +55,8 @@ public enum ComputeMode { * * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite() { - this(new File("ignite-config.xml")); + public OSHDBIgnite(OSHDBDatabase keytables) { + this(new File("ignite-config.xml"), keytables); } /** @@ -60,8 +65,8 @@ public OSHDBIgnite() { * @param ignite Ignite instance to use. * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(Ignite ignite) { - this(ignite, false); + public OSHDBIgnite(Ignite ignite, OSHDBDatabase keytables) { + this(ignite, false, keytables); } /** @@ -70,8 +75,8 @@ public OSHDBIgnite(Ignite ignite) { * @param igniteConfigFilePath ignite configuration file * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(String igniteConfigFilePath) { - this(new File(igniteConfigFilePath)); + public OSHDBIgnite(String igniteConfigFilePath, OSHDBDatabase keytables) { + this(new File(igniteConfigFilePath), keytables); } /** @@ -80,13 +85,14 @@ public OSHDBIgnite(String igniteConfigFilePath) { * @param igniteConfig ignite configuration file * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(File igniteConfig) { - this(startClient(igniteConfig), true); + public OSHDBIgnite(File igniteConfig, OSHDBDatabase keytables) { + this(startClient(igniteConfig), true, keytables); } - private OSHDBIgnite(Ignite ignite, boolean owner) { + public OSHDBIgnite(Ignite ignite, boolean owner, OSHDBDatabase keytables) { this.ignite = ignite; this.owner = owner; + this.keytables = keytables; checkStateActive(); } @@ -102,6 +108,11 @@ private void checkStateActive() { } } + @Override + public TagTranslator getTagTranslator() { + return keytables.getTagTranslator(); + } + @Override public OSHDBIgnite prefix(String prefix) { return (OSHDBIgnite) super.prefix(prefix); diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java index 2a00747de..11bbe5b4a 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java @@ -9,6 +9,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; +import java.util.function.Consumer; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; @@ -21,6 +22,7 @@ import org.apache.ignite.logger.slf4j.Slf4jLogger; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.grid.GridOSHNodes; @@ -46,16 +48,8 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce { ignite = Ignition.start(cfg); } - TestMapReduceOSHDBIgnite(OSHDBIgnite oshdb) throws Exception { - super(oshdb); - + private static OSHDBDatabase initOshdb(Consumer computeMode) { final String prefix = "tests"; - oshdb.prefix(prefix); - - OSHDBH2 oshdbH2 = new OSHDBH2("../data/test-data"); - this.keytables = oshdbH2; - - Ignite ignite = ((OSHDBIgnite) this.oshdb).getIgnite(); ignite.cluster().state(ClusterState.ACTIVE); CacheConfiguration cacheCfg = @@ -69,6 +63,7 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce { ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_WAYS.toString(prefix))); ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_RELATIONS.toString(prefix))); + OSHDBH2 oshdbH2 = new OSHDBH2("../data/test-data"); // load test data into ignite cache try (IgniteDataStreamer streamer = ignite.dataStreamer(cache.getName())) { Connection h2Conn = oshdbH2.getConnection(); @@ -96,5 +91,14 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce { } ignite.cluster().state(ClusterState.ACTIVE_READ_ONLY); + + var oshdb = new OSHDBIgnite(ignite, oshdbH2); + oshdb.prefix(prefix); + computeMode.accept(oshdb); + return oshdb; + } + + TestMapReduceOSHDBIgnite(Consumer computeMode) throws Exception { + super(initOshdb(computeMode)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java index 1d487df63..8e5d5040c 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java @@ -20,7 +20,7 @@ class TestMapReduceOSHDBIgniteAffinityCall extends TestMapReduceOSHDBIgnite { * @throws Exception if something goes wrong */ TestMapReduceOSHDBIgniteAffinityCall() throws Exception { - super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL)); + super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL)); } @Test diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java index a455c8c60..3eb69e6d4 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java @@ -14,6 +14,6 @@ class TestMapReduceOSHDBIgniteLocalPeek extends TestMapReduceOSHDBIgnite { * @throws Exception if something goes wrong */ TestMapReduceOSHDBIgniteLocalPeek() throws Exception { - super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK)); + super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java index 029afa96e..f3f79dadf 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java @@ -2,7 +2,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.junit.jupiter.api.Test; @@ -16,7 +15,7 @@ class TestMapReduceOSHDBIgniteMissingCache extends TestMapReduceOSHDBIgnite { * @throws Exception if something goes wrong */ TestMapReduceOSHDBIgniteMissingCache() throws Exception { - super(new OSHDBIgnite(ignite)); + super(oshdb -> {}); this.oshdb.prefix(""); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java index f9c2159b6..75fa3006a 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java @@ -17,7 +17,7 @@ class TestMapReduceOSHDBIgniteScanQuery extends TestMapReduceOSHDBIgnite { * @throws Exception if something goes wrong */ TestMapReduceOSHDBIgniteScanQuery() throws Exception { - super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY)); + super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY)); } @Override diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java index 600247699..166d3aa4e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java @@ -4,6 +4,7 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** * OSHDB database backend connector. @@ -12,6 +13,8 @@ public abstract class OSHDBDatabase implements AutoCloseable { private String prefix = ""; private Long timeout = null; + public abstract TagTranslator getTagTranslator(); + /** * Factory function that creates a mapReducer object of the appropriate data type class for this * oshdb backend implemenation. diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 757a2f9a0..1ce961c4a 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -18,6 +18,8 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** * OSHDB database backend connector to a JDBC database file. @@ -25,10 +27,29 @@ public class OSHDBJdbc extends OSHDBDatabase { protected final DataSource dataSource; + protected final DataSource keytablesSource; + protected DefaultTagTranslator tagTranslator; private boolean useMultithreading = true; public OSHDBJdbc(DataSource source) { + this(source, source); + } + + public OSHDBJdbc(DataSource source, DataSource keytables) { this.dataSource = source; + this.keytablesSource = keytables; + } + + @Override + public TagTranslator getTagTranslator() { + if (tagTranslator == null) { + try { + tagTranslator = new DefaultTagTranslator(keytablesSource); + } catch (SQLException e) { + throw new OSHDBException(e); + } + } + return tagTranslator; } @Override @@ -63,7 +84,6 @@ public MapReducer createMapReducer(Class for } else { mapReducer = new MapReducerJdbcSinglethread<>(this, forClass); } - mapReducer = mapReducer.keytables(this); return mapReducer; } @@ -100,5 +120,6 @@ public boolean multithreading() { @Override public void close() throws Exception { + tagTranslator.close(); } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index 658660078..1c921fe99 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -9,8 +9,6 @@ import com.tdunning.math.stats.TDigest; import java.io.IOException; import java.io.Serializable; -import java.sql.Connection; -import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; @@ -34,7 +32,6 @@ import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.OSHDBTimestamp; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; -import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.generic.NumberUtils; import org.heigit.ohsome.oshdb.api.generic.WeightedValue; import org.heigit.ohsome.oshdb.filter.AndOperator; @@ -52,7 +49,6 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBInvalidTimestampException; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.function.OSHEntityFilter; import org.heigit.ohsome.oshdb.util.function.OSMEntityFilter; import org.heigit.ohsome.oshdb.util.function.SerializableBiFunction; @@ -68,8 +64,6 @@ import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; import org.heigit.ohsome.oshdb.util.taginterpreter.TagInterpreter; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestampList; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; @@ -133,7 +127,6 @@ public abstract class MapReducer implements protected static final String UNSUPPORTED_GROUPING = "Unsupported grouping: %s"; protected transient OSHDBDatabase oshdb; - protected transient OSHDBJdbc keytables; protected Long timeout = null; @@ -155,7 +148,6 @@ public boolean isCancelable() { } // utility objects - private transient TagTranslator tagTranslator = null; private TagInterpreter tagInterpreter = null; // settings and filters @@ -180,12 +172,10 @@ protected MapReducer(OSHDBDatabase oshdb, Class vie // copy constructor protected MapReducer(MapReducer obj) { this.oshdb = obj.oshdb; - this.keytables = obj.keytables; this.viewClass = obj.viewClass; this.grouping = obj.grouping; - this.tagTranslator = obj.tagTranslator; this.tagInterpreter = obj.tagInterpreter; this.tstamps = obj.tstamps; @@ -204,38 +194,6 @@ protected MapReducer(MapReducer obj) { // "Setting" methods and associated internal helpers // ----------------------------------------------------------------------------------------------- - /** - * Sets the keytables database to use in the calculations to resolve strings (osm tags, roles) - * into internally used identifiers. If this function is never called, the main database - * (specified during the construction of this object) is used for this. - * - * @param keytables the database to use for resolving strings into internal identifiers - * @return a modified copy of this mapReducer (can be used to chain multiple commands together) - */ - @Contract(pure = true) - public MapReducer keytables(OSHDBJdbc keytables) { - if (keytables != this.oshdb && this.oshdb instanceof OSHDBJdbc) { - boolean oshdbContainsKeytables = true; - try { - Connection c = ((OSHDBJdbc) this.oshdb).getConnection(); - new DefaultTagTranslator(c).close(); - } catch (OSHDBKeytablesNotFoundException e) { - // this is the expected path -> the oshdb doesn't have the key tables - oshdbContainsKeytables = false; - } catch (SQLException e) { - throw new RuntimeException(e); - } - if (oshdbContainsKeytables) { - LOG.warn("It looks like as if the current OSHDB comes with keytables included. " - + "Usually this means that you should use this file's keytables " - + "and should not set the keytables manually."); - } - } - MapReducer ret = this.copy(); - ret.keytables = keytables; - return ret; - } - /** * Sets the tagInterpreter to use in the analysis. The tagInterpreter is used internally to * determine the geometry type of osm entities (e.g. an osm way can become either a LineString or @@ -580,7 +538,7 @@ public MapReducer filter(FilterExpression f) { @Override @Contract(pure = true) public MapReducer filter(String f) { - return this.filter(new FilterParser(this.getTagTranslator()).parse(f)); + return this.filter(new FilterParser(oshdb.getTagTranslator()).parse(f)); } // ----------------------------------------------------------------------------------------------- @@ -1684,26 +1642,11 @@ protected boolean isOSMEntitySnapshotViewQuery() { protected TagInterpreter getTagInterpreter() throws ParseException, IOException { if (this.tagInterpreter == null) { - this.tagInterpreter = new DefaultTagInterpreter(this.getTagTranslator()); + this.tagInterpreter = new DefaultTagInterpreter(oshdb.getTagTranslator()); } return this.tagInterpreter; } - protected TagTranslator getTagTranslator() { - if (this.tagTranslator == null) { - try { - if (this.keytables == null) { - throw new OSHDBKeytablesNotFoundException(); - } - this.tagTranslator = new DefaultTagTranslator(this.keytables.getConnection()); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - LOG.error(e.getMessage()); - throw new RuntimeException(e); - } - } - return this.tagTranslator; - } - // Helper that chains multiple oshEntity filters together protected OSHEntityFilter getPreFilter() { return this.preFilters.isEmpty() diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java index 66a9efe12..c8bf7d5fc 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java @@ -25,7 +25,6 @@ */ abstract class TestMapReduce { final OSHDBDatabase oshdb; - OSHDBJdbc keytables = null; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); private final OSHDBTimestamps timestamps6 = new OSHDBTimestamps("2010-01-01", "2015-01-01", @@ -38,21 +37,13 @@ abstract class TestMapReduce { } protected MapReducer createMapReducerOSMContribution() throws Exception { - MapReducer mapRed = OSMContributionView.on(oshdb); - if (this.keytables != null) { - mapRed = mapRed.keytables(this.keytables); - } - return mapRed + return OSMContributionView.on(oshdb) .areaOfInterest(bbox) .filter("type:node and highway=*"); } protected MapReducer createMapReducerOSMEntitySnapshot() throws Exception { - MapReducer mapRed = OSMEntitySnapshotView.on(oshdb); - if (this.keytables != null) { - mapRed = mapRed.keytables(this.keytables); - } - return mapRed + return OSMEntitySnapshotView.on(oshdb) .areaOfInterest(bbox) .filter("type:node and highway=*"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java index ba7364e36..0ecab66a4 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java @@ -16,7 +16,6 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.junit.jupiter.api.Test; /** @@ -41,7 +40,7 @@ class TestOSHDBFilter { */ TestOSHDBFilter() throws Exception { OSHDBH2 oshdb = new OSHDBH2("../data/test-data"); - filterParser = new FilterParser(new DefaultTagTranslator(oshdb.getConnection())); + filterParser = new FilterParser(oshdb.getTagTranslator()); this.oshdb = oshdb; } @@ -125,7 +124,7 @@ void testFilterGroupByEntity() throws Exception { @Test @SuppressWarnings("ResultOfMethodCallIgnored") void testFilterNonExistentTag() throws Exception { - FilterParser parser = new FilterParser(new DefaultTagTranslator(oshdb.getConnection())); + FilterParser parser = new FilterParser(oshdb.getTagTranslator()); try { createMapReducerOSMEntitySnapshot() .filter(parser.parse("type:way and nonexistentkey=*")) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java index 4ab5ae458..8e77483b5 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java @@ -1,11 +1,10 @@ package org.heigit.ohsome.oshdb.filter; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; @@ -21,7 +20,6 @@ import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -29,23 +27,27 @@ * Tests the parsing of filters and the application to OSM entities. */ abstract class FilterTest { - protected Connection conn; + protected JdbcConnectionPool source; protected FilterParser parser; - protected TagTranslator tagTranslator; + protected DefaultTagTranslator tagTranslator; @BeforeEach void setup() throws SQLException, ClassNotFoundException, OSHDBKeytablesNotFoundException { - this.conn = DriverManager.getConnection( + this.source = JdbcConnectionPool.create( "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", "sa", "" ); - this.tagTranslator = new DefaultTagTranslator(conn); + this.tagTranslator = new DefaultTagTranslator(source); this.parser = new FilterParser(this.tagTranslator); } @AfterEach void teardown() throws SQLException { - conn.close(); + try { + tagTranslator.close(); + } finally { + source.dispose(); + } } protected int[] createTestTags(String... keyValues) { diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/exceptions/OSHDBKeytablesNotFoundException.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/exceptions/OSHDBKeytablesNotFoundException.java index 28015aa53..2fc45ff78 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/exceptions/OSHDBKeytablesNotFoundException.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/exceptions/OSHDBKeytablesNotFoundException.java @@ -3,7 +3,7 @@ /** * Exception used if the OSHDB keytable cannot be found. */ -public class OSHDBKeytablesNotFoundException extends Exception { +public class OSHDBKeytablesNotFoundException extends OSHDBException { /** * Creates an exception with a message explaining that the OSHDB keytables db was not found. */ diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java index 5dabce3cc..45d95f3d8 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java @@ -12,6 +12,7 @@ import java.sql.Statement; import java.util.EnumSet; import java.util.concurrent.ConcurrentHashMap; +import javax.sql.DataSource; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; @@ -39,7 +40,7 @@ * * */ -public class DefaultTagTranslator implements TagTranslator { +public class DefaultTagTranslator implements TagTranslator, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(DefaultTagTranslator.class); private static final String UNABLE_TO_ACCESS_KEYTABLES = "Unable to access keytables"; @@ -66,9 +67,10 @@ public class DefaultTagTranslator implements TagTranslator { * @param conn a connection to a database (containing oshdb keytables). * @throws OSHDBKeytablesNotFoundException if the supplied database doesn't contain the required * "keyTables" tables + * @throws SQLException */ - public DefaultTagTranslator(Connection conn) throws OSHDBKeytablesNotFoundException { - this.conn = conn; + public DefaultTagTranslator(DataSource source) throws OSHDBKeytablesNotFoundException, SQLException { + this.conn = source.getConnection(); this.keyToInt = new ConcurrentHashMap<>(0); this.keyToString = new ConcurrentHashMap<>(0); this.tagToInt = new ConcurrentHashMap<>(0); @@ -109,12 +111,16 @@ public DefaultTagTranslator(Connection conn) throws OSHDBKeytablesNotFoundExcept @Override public void close() throws SQLException { - keyIdQuery.close(); - keyTxtQuery.close(); - valueIdQuery.close(); - valueTxtQuery.close(); - roleIdQuery.close(); - roleTxtQuery.close(); + try { + keyIdQuery.close(); + keyTxtQuery.close(); + valueIdQuery.close(); + valueTxtQuery.close(); + roleIdQuery.close(); + roleTxtQuery.close(); + } finally { + conn.close(); + } } @Override diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index b27d43bc5..5066ef499 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -5,7 +5,7 @@ import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; -public interface TagTranslator extends AutoCloseable { +public interface TagTranslator { /** * Get oshdb's internal representation of a tag key (string). * diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java index 25ef48a14..48b4a0a60 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java @@ -2,25 +2,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.IOException; import java.io.ObjectInputStream; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.TreeSet; import java.util.stream.Collectors; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.OSHDBBoundingBox; import org.heigit.ohsome.oshdb.OSHDBTimestamp; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; -import org.json.simple.parser.ParseException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -29,18 +24,15 @@ * Tests the {@link CellIterator#iterateByContribution(GridOSHEntity)} method. */ class IterateByContributionTest { - private static Connection conn; + private static JdbcConnectionPool source; /** * Set up of test framework, loading H2 driver and connection via jdbc. */ @BeforeAll static void setUpClass() throws ClassNotFoundException, SQLException { - // load H2-support - Class.forName("org.h2.Driver"); - // connect to the "Big"DB - IterateByContributionTest.conn = DriverManager.getConnection( + source = JdbcConnectionPool.create( "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", "sa", "" @@ -48,8 +40,8 @@ static void setUpClass() throws ClassNotFoundException, SQLException { } @AfterAll - static void breakDownClass() throws SQLException { - IterateByContributionTest.conn.close(); + static void breakDownClass() { + source.dispose(); } IterateByContributionTest() {} @@ -57,14 +49,14 @@ static void breakDownClass() throws SQLException { @SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection"}) @Test void testIssue108() throws Exception { - ResultSet oshCellsRawData = conn.prepareStatement( - "select data from " + TableNames.T_NODES).executeQuery(); - int countTotal = 0; int countCreated = 0; int countOther = 0; - try (TagTranslator tt = new DefaultTagTranslator(conn)) { + TagTranslator tt = new DefaultTagTranslator(source); + try (var conn = source.getConnection(); + var stmt = conn.createStatement(); + var oshCellsRawData = stmt.executeQuery("select data from " + TableNames.T_NODES)) { while (oshCellsRawData.next()) { // get one cell from the raw data stream GridOSHEntity oshCellRawData = (GridOSHEntity) (new ObjectInputStream( @@ -95,7 +87,6 @@ void testIssue108() throws Exception { } } } - assertEquals(4, countTotal); assertEquals(0, countCreated); assertEquals(4, countOther); diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java index c1ad30257..7e7e02028 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java @@ -1,10 +1,8 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; @@ -17,7 +15,7 @@ * Tests the {@link DefaultTagTranslator} class. */ class TagTranslatorTest { - private static Connection conn; + private static JdbcConnectionPool source; /** * Initialize tests by loading the H2 driver and open a connection via jdbc. @@ -27,64 +25,63 @@ class TagTranslatorTest { */ @BeforeAll static void setUpClass() throws ClassNotFoundException, SQLException { - // load H2-support - Class.forName("org.h2.Driver"); - - // connect to the test data DB - TagTranslatorTest.conn = - DriverManager.getConnection("jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", - "sa", ""); + source = JdbcConnectionPool.create("jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", "sa", ""); } @AfterAll static void breakDownClass() throws SQLException { - TagTranslatorTest.conn.close(); + source.dispose(); } TagTranslatorTest() {} @Test - void testTag2Int() throws OSHDBKeytablesNotFoundException { + void testTag2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMTag tag = new OSMTag("building", "yes"); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSHDBTag expResult = new OSHDBTag(1, 0); - OSHDBTag result = instance.getOSHDBTagOf(tag); - assertEquals(expResult, result); + try (var instance = new DefaultTagTranslator(source)){ + OSHDBTag expResult = new OSHDBTag(1, 0); + OSHDBTag result = instance.getOSHDBTagOf(tag); + assertEquals(expResult, result); + } } @Test - void testTag2String() throws OSHDBKeytablesNotFoundException { + void testTag2String() throws OSHDBKeytablesNotFoundException, SQLException { OSHDBTag tag = new OSHDBTag(1, 2); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSMTag expResult = new OSMTag("building", "residential"); - OSMTag result = instance.lookupTag(tag); - assertEquals(expResult, result); + try (var instance = new DefaultTagTranslator(source)){ + OSMTag expResult = new OSMTag("building", "residential"); + OSMTag result = instance.lookupTag(tag); + assertEquals(expResult, result); + } } @Test - void testKey2Int() throws OSHDBKeytablesNotFoundException { + void testKey2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMTagKey key = new OSMTagKey("highway"); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSHDBTagKey expResult = new OSHDBTagKey(2); - OSHDBTagKey result = instance.getOSHDBTagKeyOf(key); - assertEquals(expResult, result); + try (var instance = new DefaultTagTranslator(source)) { + OSHDBTagKey expResult = new OSHDBTagKey(2); + OSHDBTagKey result = instance.getOSHDBTagKeyOf(key); + assertEquals(expResult, result); + } } @Test - void testRole2Int() throws OSHDBKeytablesNotFoundException { + void testRole2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMRole role = new OSMRole("from"); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSHDBRole expResult = OSHDBRole.of(4); - OSHDBRole result = instance.getOSHDBRoleOf(role); - assertEquals(expResult, result); + try (var instance = new DefaultTagTranslator(source)){ + OSHDBRole expResult = OSHDBRole.of(4); + OSHDBRole result = instance.getOSHDBRoleOf(role); + assertEquals(expResult, result); + } } @Test - void testRole2String() throws OSHDBKeytablesNotFoundException { + void testRole2String() throws OSHDBKeytablesNotFoundException, SQLException { OSHDBRole role = OSHDBRole.of(1); - TagTranslator instance = new DefaultTagTranslator(TagTranslatorTest.conn); - OSMRole expResult = new OSMRole("inner"); - OSMRole result = instance.lookupRole(role); - assertEquals(expResult, result); + try (var instance = new DefaultTagTranslator(source)){ + OSMRole expResult = new OSMRole("inner"); + OSMRole result = instance.lookupRole(role); + assertEquals(expResult, result); + } } } From 55d471f7dccc2d08b9bcce32d81451fb10be1f91 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 16:18:53 +0200 Subject: [PATCH 06/34] fix fail build --- .../ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java index 45d95f3d8..d3f73382d 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java @@ -64,7 +64,7 @@ public class DefaultTagTranslator implements TagTranslator, AutoCloseable { * A TagTranslator for a specific DB-Connection. It has its own internal cache * to speed up searching. * - * @param conn a connection to a database (containing oshdb keytables). + * @param source a datasoure to a database (containing oshdb keytables). * @throws OSHDBKeytablesNotFoundException if the supplied database doesn't contain the required * "keyTables" tables * @throws SQLException From 6390ca8b4236abd96d3ff596678a268287e50e58 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 17 Oct 2022 21:00:41 +0200 Subject: [PATCH 07/34] rework of tagtranslater, jdbc and cached --- .../ohsome/oshdb/api/db/OSHDBIgnite.java | 3 - .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 11 +- .../ohsome/oshdb/filter/FilterParser.java | 2 +- .../oshdb/filter/GeometryTypeFilter.java | 4 +- .../heigit/ohsome/oshdb/filter/TagFilter.java | 12 +- .../ohsome/oshdb/filter/FilterTest.java | 15 +- .../ohsome/oshdb/filter/NegateTest.java | 8 +- .../heigit/ohsome/oshdb/filter/ParseTest.java | 25 +- oshdb-util/pom.xml | 6 + .../taginterpreter/BaseTagInterpreter.java | 15 +- .../taginterpreter/DefaultTagInterpreter.java | 91 +++--- .../tagtranslator/CachedTagTranslator.java | 81 +++++ .../util/tagtranslator/ClosableSqlArray.java | 29 ++ .../tagtranslator/DefaultTagTranslator.java | 296 ------------------ .../util/tagtranslator/JdbcTagTranslator.java | 203 ++++++++++++ .../util/tagtranslator/TagTranslator.java | 32 +- .../IterateByContributionTest.java | 10 +- .../util/tagtranslator/TagTranslatorTest.java | 98 ++++-- 18 files changed, 516 insertions(+), 425 deletions(-) create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java delete mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java diff --git a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java index caa2c88ba..735260cb2 100644 --- a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java +++ b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java @@ -2,12 +2,10 @@ import com.google.common.base.Joiner; import java.io.File; -import java.sql.SQLException; import java.util.Collection; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.sql.DataSource; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.lang.IgniteRunnable; @@ -20,7 +18,6 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 1ce961c4a..ba1a87e90 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -18,7 +18,7 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** @@ -28,7 +28,7 @@ public class OSHDBJdbc extends OSHDBDatabase { protected final DataSource dataSource; protected final DataSource keytablesSource; - protected DefaultTagTranslator tagTranslator; + protected JdbcTagTranslator tagTranslator; private boolean useMultithreading = true; public OSHDBJdbc(DataSource source) { @@ -43,11 +43,7 @@ public OSHDBJdbc(DataSource source, DataSource keytables) { @Override public TagTranslator getTagTranslator() { if (tagTranslator == null) { - try { - tagTranslator = new DefaultTagTranslator(keytablesSource); - } catch (SQLException e) { - throw new OSHDBException(e); - } + tagTranslator = new JdbcTagTranslator(keytablesSource); } return tagTranslator; } @@ -120,6 +116,5 @@ public boolean multithreading() { @Override public void close() throws Exception { - tagTranslator.close(); } } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java index df0875107..ddf651472 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java @@ -125,7 +125,7 @@ public FilterParser(TagTranslator tt, boolean allowContributorFilters) { stringSequence, (key, ignored, values) -> { List tags = new ArrayList<>(values.size()); - values.forEach(value -> tags.add(tt.getOSHDBTagOf(key, value))); + values.forEach(value -> tt.getOSHDBTagOf(key, value).ifPresent(tags::add)); return new TagFilterEqualsAnyOf(tags); }); final Parser idFilter = Parsers.sequence( diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java index f6d118b90..b91290d5e 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java @@ -57,8 +57,8 @@ public enum GeometryType { */ public GeometryTypeFilter(@Nonnull GeometryType geometryType, TagTranslator tt) { this.geometryType = geometryType; - this.typeMultipolygon = tt.getOSHDBTagOf(tagTypeMultipolygon); - this.typeBoundary = tt.getOSHDBTagOf(tagTypeBoundary); + this.typeMultipolygon = tt.getOSHDBTagOf(tagTypeMultipolygon).orElse(new OSHDBTag(-1, -1)); + this.typeBoundary = tt.getOSHDBTagOf(tagTypeBoundary).orElse(new OSHDBTag(-1, -2)); } private GeometryTypeFilter( diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilter.java index c2f72ea38..91c549fc6 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilter.java @@ -1,6 +1,8 @@ package org.heigit.ohsome.oshdb.filter; import javax.annotation.Nonnull; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTagInterface; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTagKey; @@ -42,18 +44,20 @@ enum Type { */ @Contract(pure = true) static TagFilter fromSelector(@Nonnull Type selector, OSMTagInterface tag, TagTranslator tt) { + //TODO we should check for key,key/value combination that not exist in the database and + // eagerly cancel the query!! switch (selector) { case EQUALS: if (tag instanceof OSMTag) { - return new TagFilterEquals(tt.getOSHDBTagOf((OSMTag) tag)); + return new TagFilterEquals(tt.getOSHDBTagOf((OSMTag) tag).orElse(new OSHDBTag(-1, -1))); } else { - return new TagFilterEqualsAny(tt.getOSHDBTagKeyOf((OSMTagKey) tag)); + return new TagFilterEqualsAny(tt.getOSHDBTagKeyOf((OSMTagKey) tag).orElse(new OSHDBTagKey(-1))); } case NOT_EQUALS: if (tag instanceof OSMTag) { - return new TagFilterNotEquals(tt.getOSHDBTagOf((OSMTag) tag)); + return new TagFilterNotEquals(tt.getOSHDBTagOf((OSMTag) tag).orElse(new OSHDBTag(-1, -1))); } else { - return new TagFilterNotEqualsAny(tt.getOSHDBTagKeyOf((OSMTagKey) tag)); + return new TagFilterNotEqualsAny(tt.getOSHDBTagKeyOf((OSMTagKey) tag).orElse(new OSHDBTagKey(-1))); } default: assert false : "invalid or null tag filter selector encountered"; diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java index 8e77483b5..799d889c4 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java @@ -19,7 +19,8 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -29,7 +30,7 @@ abstract class FilterTest { protected JdbcConnectionPool source; protected FilterParser parser; - protected DefaultTagTranslator tagTranslator; + protected TagTranslator tagTranslator; @BeforeEach void setup() throws SQLException, ClassNotFoundException, OSHDBKeytablesNotFoundException { @@ -37,23 +38,19 @@ void setup() throws SQLException, ClassNotFoundException, OSHDBKeytablesNotFound "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", "sa", "" ); - this.tagTranslator = new DefaultTagTranslator(source); + this.tagTranslator = new JdbcTagTranslator(source); this.parser = new FilterParser(this.tagTranslator); } @AfterEach void teardown() throws SQLException { - try { - tagTranslator.close(); - } finally { - source.dispose(); - } + source.dispose(); } protected int[] createTestTags(String... keyValues) { ArrayList tags = new ArrayList<>(keyValues.length); for (int i = 0; i < keyValues.length; i += 2) { - OSHDBTag t = tagTranslator.getOSHDBTagOf(keyValues[i], keyValues[i + 1]); + OSHDBTag t = tagTranslator.getOSHDBTagOf(keyValues[i], keyValues[i + 1]).get(); tags.add(t.getKey()); tags.add(t.getValue()); } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java index 56c9b3d48..6dde3e3a5 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java @@ -69,8 +69,8 @@ void testTagFilterNotEqualsAny() { @Test void testTagFilterEqualsAnyOf() { - OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential"); - OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track"); + OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential").get(); + OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track").get(); FilterExpression expression = new TagFilterEqualsAnyOf(Arrays.asList(tag1, tag2)); FilterExpression negation = expression.negate(); assertTrue(negation instanceof TagFilterNotEqualsAnyOf); @@ -78,8 +78,8 @@ void testTagFilterEqualsAnyOf() { @Test void testTagFilterNotEqualsAnyOf() { - OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential"); - OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track"); + OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential").get(); + OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track").get(); FilterExpression expression = new TagFilterNotEqualsAnyOf(Arrays.asList(tag1, tag2)); FilterExpression negation = expression.negate(); assertTrue(negation instanceof TagFilterEqualsAnyOf); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java index cf667cc66..8823ce100 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -23,7 +24,7 @@ class ParseTest extends FilterTest { void testTagFilterEquals() { FilterExpression expression = parser.parse("highway=residential"); assertTrue(expression instanceof TagFilterEquals); - OSHDBTag tag = tagTranslator.getOSHDBTagOf("highway", "residential"); + OSHDBTag tag = tagTranslator.getOSHDBTagOf("highway", "residential").get(); assertEquals(tag, ((TagFilterEquals) expression).getTag()); assertEquals("tag:" + tag.getKey() + "=" + tag.getValue(), expression.toString()); } @@ -43,7 +44,7 @@ void testTagFilterStrings() { void testTagFilterEqualsAny() { FilterExpression expression = parser.parse("highway=*"); assertTrue(expression instanceof TagFilterEqualsAny); - OSHDBTagKey tag = tagTranslator.getOSHDBTagKeyOf("highway"); + OSHDBTagKey tag = tagTranslator.getOSHDBTagKeyOf("highway").get(); assertEquals(tag, ((TagFilterEqualsAny) expression).getTag()); assertEquals("tag:" + tag.toInt() + "=*", expression.toString()); } @@ -52,7 +53,7 @@ void testTagFilterEqualsAny() { void testTagFilterNotEquals() { FilterExpression expression = parser.parse("highway!=residential"); assertTrue(expression instanceof TagFilterNotEquals); - OSHDBTag tag = tagTranslator.getOSHDBTagOf("highway", "residential"); + OSHDBTag tag = tagTranslator.getOSHDBTagOf("highway", "residential").get(); assertEquals(tag, ((TagFilterNotEquals) expression).getTag()); assertEquals("tag:" + tag.getKey() + "!=" + tag.getValue(), expression.toString()); } @@ -61,7 +62,7 @@ void testTagFilterNotEquals() { void testTagFilterNotEqualsAny() { FilterExpression expression = parser.parse("highway!=*"); assertTrue(expression instanceof TagFilterNotEqualsAny); - OSHDBTagKey tag = tagTranslator.getOSHDBTagKeyOf("highway"); + OSHDBTagKey tag = tagTranslator.getOSHDBTagKeyOf("highway").get(); assertEquals(tag, ((TagFilterNotEqualsAny) expression).getTag()); assertEquals("tag:" + tag.toInt() + "!=*", expression.toString()); } @@ -71,8 +72,8 @@ void testTagFilterNotEqualsAny() { void testTagFilterEqualsAnyOf() { FilterExpression expression = parser.parse("highway in (residential, track)"); assertTrue(expression instanceof TagFilterEqualsAnyOf); - OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential"); - OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track"); + OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential").get(); + OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track").get(); assertTrue(expression.toString().matches("tag:" + tag1.getKey() + "in(" + tag1.getValue() + "," + tag2.getValue() + "|" + tag2.getValue() + "," + tag1.getValue() + ")" @@ -84,8 +85,8 @@ void testTagFilterEqualsAnyOf() { void testTagFilterNotEqualsAnyOf() { FilterExpression expression = parser.parse("not highway in (residential, track)"); assertTrue(expression instanceof TagFilterNotEqualsAnyOf); - OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential"); - OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track"); + OSHDBTag tag1 = tagTranslator.getOSHDBTagOf("highway", "residential").get(); + OSHDBTag tag2 = tagTranslator.getOSHDBTagOf("highway", "track").get(); assertTrue(expression.toString().matches("tag:" + tag1.getKey() + "not-in(" + tag1.getValue() + "," + tag2.getValue() + "|" + tag2.getValue() + "," + tag1.getValue() + ")" @@ -110,8 +111,8 @@ void testTagFilterNotEqualsAnyOfCheckEmpty() { void testTagFilterEqualsAnyOfCheckMixed() { assertThrows(IllegalStateException.class, () -> { new TagFilterEqualsAnyOf(Arrays.asList( - tagTranslator.getOSHDBTagOf("highway", "residential"), - tagTranslator.getOSHDBTagOf("building", "yes") + tagTranslator.getOSHDBTagOf("highway", "residential").get(), + tagTranslator.getOSHDBTagOf("building", "yes").get() )); }); } @@ -120,8 +121,8 @@ void testTagFilterEqualsAnyOfCheckMixed() { void testTagFilterNotEqualsAnyOfCheckMixed() { assertThrows(IllegalStateException.class, () -> { new TagFilterNotEqualsAnyOf(Arrays.asList( - tagTranslator.getOSHDBTagOf("highway", "residential"), - tagTranslator.getOSHDBTagOf("building", "yes") + tagTranslator.getOSHDBTagOf("highway", "residential").get(), + tagTranslator.getOSHDBTagOf("building", "yes").get() )); }); } diff --git a/oshdb-util/pom.xml b/oshdb-util/pom.xml index b591603c1..fee65c6f1 100644 --- a/oshdb-util/pom.xml +++ b/oshdb-util/pom.xml @@ -28,6 +28,12 @@ jts-core ${jts.version} + + + com.github.ben-manes.caffeine + caffeine + 3.1.1 + com.googlecode.json-simple diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java index c08621ccc..bb2494eb9 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java @@ -4,6 +4,7 @@ import java.util.Map; import java.util.Set; +import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.osh.OSHWay; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMMember; @@ -17,8 +18,8 @@ * linestring geometry. */ class BaseTagInterpreter implements TagInterpreter { - int areaNoTagKeyId; - int areaNoTagValueId; + OSHDBTag areaNo; + Map> wayAreaTags; Map> relationAreaTags; Set uninterestingTagKeys; @@ -27,8 +28,7 @@ class BaseTagInterpreter implements TagInterpreter { int emptyRoleId; BaseTagInterpreter( - int areaNoTagKeyId, - int areaNoTagValueId, + OSHDBTag areaNo, Map> wayAreaTags, Map> relationAreaTags, Set uninterestingTagKeys, @@ -36,8 +36,7 @@ class BaseTagInterpreter implements TagInterpreter { int innerRoleId, int emptyRoleId ) { - this.areaNoTagKeyId = areaNoTagKeyId; - this.areaNoTagValueId = areaNoTagValueId; + this.areaNo = areaNo; this.wayAreaTags = wayAreaTags; this.relationAreaTags = relationAreaTags; this.uninterestingTagKeys = uninterestingTagKeys; @@ -48,10 +47,10 @@ class BaseTagInterpreter implements TagInterpreter { private boolean evaluateWayForArea(OSMWay entity) { var tags = entity.getTags(); - if (tags.hasTagValue(areaNoTagKeyId, areaNoTagValueId)) { + if (tags.hasTag(areaNo)) { return false; } - for (var tag : entity.getTags()) { + for (var tag : tags) { if (wayAreaTags.getOrDefault(tag.getKey(), emptySet()) .contains(tag.getValue())) { return true; diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java index 96257818e..64e8c007f 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java @@ -8,9 +8,11 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMRelation; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.json.simple.JSONArray; @@ -26,10 +28,10 @@ public class DefaultTagInterpreter extends BaseTagInterpreter { private static final Logger LOG = LoggerFactory.getLogger(DefaultTagInterpreter.class); - private int typeKey = -1; - private int typeMultipolygonValue = -1; - private int typeBoundaryValue = -1; - private int typeRouteValue = -1; + private int typeKey; + private OSHDBTag typeMultipolygon; + private OSHDBTag typeBoundary; + private OSHDBTag typeRoute; private static final String defaultAreaTagsDefinitionFile = "json/polygon-features.json"; private static final String defaultUninterestingTagsDefinitionFile @@ -67,7 +69,7 @@ public DefaultTagInterpreter( TagTranslator tagTranslator, String areaTagsDefinitionFile, String uninterestingTagsDefinitionFile ) throws IOException, ParseException { - super(-1, -1, null, null, null, -1, -1, -1); // initialize with dummy parameters for now + super(new OSHDBTag(-1, -1), null, null, null, -1, -1, -1); // initialize with dummy parameters for now // construct list of area tags for ways Map> wayAreaTags = new HashMap<>(); @@ -84,34 +86,37 @@ public DefaultTagInterpreter( switch ((String) tag.get("polygon")) { case "all": Set valueIds = new InvertedHashSet<>(); - int keyId = tagTranslator.getOSHDBTagKeyOf(key).toInt(); - valueIds.add(tagTranslator.getOSHDBTagOf(key, "no").getValue()); - wayAreaTags.put(keyId, valueIds); + tagTranslator.getOSHDBTagKeyOf(key).map(OSHDBTagKey::toInt).ifPresent(keyId -> { + tagTranslator.getOSHDBTagOf(key, "no").map(OSHDBTag::getValue).ifPresent(valueIds::add); + wayAreaTags.put(keyId, valueIds); + }); break; case "whitelist": valueIds = new HashSet<>(); - keyId = tagTranslator.getOSHDBTagKeyOf(key).toInt(); - JSONArray values = (JSONArray) tag.get("values"); - @SuppressWarnings("unchecked") // we expect only strings here in a valid definition file - Iterable iterableWhitelistValues = values; - for (String value : iterableWhitelistValues) { - OSMTag keyValue = new OSMTag(key, value); - valueIds.add(tagTranslator.getOSHDBTagOf(keyValue).getValue()); - } - valueIds.add(tagTranslator.getOSHDBTagOf(key, "no").getValue()); - wayAreaTags.put(keyId, valueIds); + tagTranslator.getOSHDBTagKeyOf(key).map(OSHDBTagKey::toInt).ifPresent(keyId -> { + JSONArray values = (JSONArray) tag.get("values"); + @SuppressWarnings("unchecked") // we expect only strings here in a valid definition file + Iterable iterableWhitelistValues = values; + for (String value : iterableWhitelistValues) { + OSMTag keyValue = new OSMTag(key, value); + tagTranslator.getOSHDBTagOf(keyValue).map(OSHDBTag::getValue) + .ifPresent(valueIds::add); + } + tagTranslator.getOSHDBTagOf(key, "no").map(OSHDBTag::getValue).ifPresent(valueIds::add); + wayAreaTags.put(keyId, valueIds); }); break; case "blacklist": valueIds = new InvertedHashSet<>(); - keyId = tagTranslator.getOSHDBTagKeyOf(key).toInt(); - values = (JSONArray) tag.get("values"); - @SuppressWarnings("unchecked") // we expect only strings here in a valid definition file - Iterable iterableBlacklistValues = values; - for (String value : iterableBlacklistValues) { - OSMTag keyValue = new OSMTag(key, value); - valueIds.add(tagTranslator.getOSHDBTagOf(keyValue).getValue()); - } - wayAreaTags.put(keyId, valueIds); + tagTranslator.getOSHDBTagKeyOf(key).map(OSHDBTagKey::toInt).ifPresent(keyId -> { + JSONArray values = (JSONArray) tag.get("values"); + @SuppressWarnings("unchecked") // we expect only strings here in a valid definition file + Iterable iterableBlacklistValues = values; + for (String value : iterableBlacklistValues) { + OSMTag keyValue = new OSMTag(key, value); + tagTranslator.getOSHDBTagOf(keyValue).map(OSHDBTag::getValue) + .ifPresent(valueIds::add); + } + wayAreaTags.put(keyId, valueIds); }); break; default: throw new ParseException(-13); @@ -119,16 +124,16 @@ public DefaultTagInterpreter( } // hardcoded type=multipolygon for relations - this.typeKey = tagTranslator.getOSHDBTagKeyOf("type").toInt(); - this.typeMultipolygonValue = tagTranslator.getOSHDBTagOf("type", "multipolygon").getValue(); - this.typeBoundaryValue = tagTranslator.getOSHDBTagOf("type", "boundary").getValue(); - this.typeRouteValue = tagTranslator.getOSHDBTagOf("type", "route").getValue(); + this.typeKey = tagTranslator.getOSHDBTagKeyOf("type").map(OSHDBTagKey::toInt).orElse(-1); + this.typeMultipolygon = tagTranslator.getOSHDBTagOf("type", "multipolygon").orElse(new OSHDBTag(typeKey, -1)); + this.typeBoundary = tagTranslator.getOSHDBTagOf("type", "boundary").orElse(new OSHDBTag(typeKey, -2)); + this.typeRoute = tagTranslator.getOSHDBTagOf("type", "route").orElse(new OSHDBTag(typeKey, -3)); // we still need to also store relation area tags for isOldStyleMultipolygon() functionality! Map> relAreaTags = new TreeMap<>(); Set relAreaTagValues = new TreeSet<>(); - relAreaTagValues.add(this.typeMultipolygonValue); - relAreaTagValues.add(this.typeBoundaryValue); + relAreaTagValues.add(this.typeMultipolygon.getValue()); + relAreaTagValues.add(this.typeBoundary.getValue()); relAreaTags.put(this.typeKey, relAreaTagValues); // list of uninteresting tags @@ -141,19 +146,19 @@ public DefaultTagInterpreter( @SuppressWarnings("unchecked") // we expect only strings here in a valid definition file Iterable iterableUninterestingTagsList = uninterestingTagsList; for (String tagKey : iterableUninterestingTagsList) { - uninterestingTagKeys.add(tagTranslator.getOSHDBTagKeyOf(tagKey).toInt()); + tagTranslator.getOSHDBTagKeyOf(tagKey).map(OSHDBTagKey::toInt).ifPresent(uninterestingTagKeys::add); } this.wayAreaTags = wayAreaTags; this.relationAreaTags = relAreaTags; this.uninterestingTagKeys = uninterestingTagKeys; - this.areaNoTagKeyId = tagTranslator.getOSHDBTagOf("area", "no").getKey(); - this.areaNoTagValueId = tagTranslator.getOSHDBTagOf("area", "no").getValue(); + var areaTagKey = tagTranslator.getOSHDBTagKeyOf("area").map(OSHDBTagKey::toInt).orElse(-2); + this.areaNo = tagTranslator.getOSHDBTagOf("area", "no").orElse(new OSHDBTag(areaTagKey, -1)); - this.outerRoleId = tagTranslator.getOSHDBRoleOf("outer").getId(); - this.innerRoleId = tagTranslator.getOSHDBRoleOf("inner").getId(); - this.emptyRoleId = tagTranslator.getOSHDBRoleOf("").getId(); + this.outerRoleId = tagTranslator.getOSHDBRoleOf("outer").map(OSHDBRole::getId).orElse(-1); + this.innerRoleId = tagTranslator.getOSHDBRoleOf("inner").map(OSHDBRole::getId).orElse(-2); + this.emptyRoleId = tagTranslator.getOSHDBRoleOf("").map(OSHDBRole::getId).orElse(-3); } @Override @@ -183,9 +188,7 @@ protected boolean evaluateRelationForArea(OSMRelation entity) { // `return entity.hasTagValue(k1,v1) || entity.hasTagValue(k2,v2);` for (var tag : tags) { if (tag.getKey() == typeKey) { - return tag.getValue() == typeMultipolygonValue || tag.getValue() == typeBoundaryValue; - } else if (tag.getKey() > typeKey) { - return false; + return tag.equals(typeMultipolygon) || tag.equals(typeBoundary); } } return false; @@ -193,6 +196,6 @@ protected boolean evaluateRelationForArea(OSMRelation entity) { // checks if the relation has the tag "type=route" private boolean evaluateRelationForLine(OSMRelation entity) { - return entity.getTags().hasTagValue(typeKey, typeRouteValue); + return entity.getTags().hasTag(typeRoute); } } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java new file mode 100644 index 000000000..956b4f910 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java @@ -0,0 +1,81 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; + +public class CachedTagTranslator implements TagTranslator { + + private TagTranslator source; + + private Cache lookupOSHDBTag; + private Cache lookupOSHDBRole; + + public CachedTagTranslator(TagTranslator source, long maxBytesValues, int maxNumRoles) { + this.source = source; + this.lookupOSHDBTag = Caffeine.newBuilder() + .weigher((oshdb, osm) -> osm.getValue().length() * 2) + .maximumWeight(maxBytesValues) + .build(); + this.lookupOSHDBRole = Caffeine.newBuilder() + .maximumSize(maxNumRoles) + .build(); + } + + public Cache getLookupOSHDBTag() { + return lookupOSHDBTag; + } + + public Cache getLookupOSHDBRole() { + return lookupOSHDBRole; + } + + @Override + public Optional getOSHDBTagKeyOf(OSMTagKey key) { + return source.getOSHDBTagKeyOf(key); + } + + @Override + public Optional getOSHDBTagOf(OSMTag osm) { + var oshdb = source.getOSHDBTagOf(osm); + oshdb.ifPresent(tag -> lookupOSHDBTag.put(tag, osm)); + return oshdb; + } + + @Override + public Map getOSHDBTagOf(Set tags) { + var oshdb = source.getOSHDBTagOf(tags); + oshdb.entrySet().forEach(entry -> lookupOSHDBTag.put(entry.getValue(), entry.getKey())); + return oshdb; + } + + @Override + public Optional getOSHDBRoleOf(OSMRole role) { + return source.getOSHDBRoleOf(role); + } + + @Override + public Map getOSHDBRoleOf(Set roles) { + return source.getOSHDBRoleOf(roles); + } + + @Override + public OSMTag lookupTag(OSHDBTag tag) { + return lookupOSHDBTag.get(tag, source::lookupTag); + } + + @Override + public Map lookupTag(Set tags) { + return lookupOSHDBTag.getAll(tags, source::lookupTag); + } + + @Override + public OSMRole lookupRole(OSHDBRole role) { + return lookupOSHDBRole.get(role, source::lookupRole); + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java new file mode 100644 index 000000000..492d67301 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java @@ -0,0 +1,29 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +import java.sql.Array; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collection; + +public class ClosableSqlArray implements AutoCloseable { + + public static ClosableSqlArray createArray(Connection conn, String typeName, Collection elements) throws SQLException { + var array = conn.createArrayOf(typeName, elements.toArray()); + return new ClosableSqlArray(array); + } + + private Array array; + + public ClosableSqlArray(Array array) { + this.array = array; + } + + public Array get() { + return array; + } + + @Override + public void close() throws Exception { + array.free(); + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java deleted file mode 100644 index d3f73382d..000000000 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/DefaultTagTranslator.java +++ /dev/null @@ -1,296 +0,0 @@ -package org.heigit.ohsome.oshdb.util.tagtranslator; - -import static java.lang.String.format; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; -import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.EnumSet; -import java.util.concurrent.ConcurrentHashMap; -import javax.sql.DataSource; -import org.heigit.ohsome.oshdb.OSHDBRole; -import org.heigit.ohsome.oshdb.OSHDBTag; -import org.heigit.ohsome.oshdb.util.OSHDBTagKey; -import org.heigit.ohsome.oshdb.util.TableNames; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Easily translate your textual tags and roles to OSHDB's internal - * representation (encoded as integers) and vice versa. - * - *

This class handles missing/not-found data in the following ways:

- *
    - *
  • - * for (tag/role) strings that cannot be found in a keytable, the tagtranslator will generate a - * (temporary, internal and negative) id which can afterwards be resolved back to the input - * string when using the same tagtranslator object. - *
    • - *
  • - * for ids that are not found neither in the keytable nor the tagtranslator's cache, a runtime - * exception is thrown - *
    • - *
- */ -public class DefaultTagTranslator implements TagTranslator, AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(DefaultTagTranslator.class); - private static final String UNABLE_TO_ACCESS_KEYTABLES = "Unable to access keytables"; - - private final PreparedStatement keyIdQuery; - private final PreparedStatement keyTxtQuery; - private final PreparedStatement valueIdQuery; - private final PreparedStatement valueTxtQuery; - private final PreparedStatement roleIdQuery; - private final PreparedStatement roleTxtQuery; - - private final ConcurrentHashMap keyToInt; - private final ConcurrentHashMap keyToString; - private final ConcurrentHashMap tagToInt; - private final ConcurrentHashMap tagToString; - private final ConcurrentHashMap roleToInt; - private final ConcurrentHashMap roleToString; - - private final Connection conn; - - /** - * A TagTranslator for a specific DB-Connection. It has its own internal cache - * to speed up searching. - * - * @param source a datasoure to a database (containing oshdb keytables). - * @throws OSHDBKeytablesNotFoundException if the supplied database doesn't contain the required - * "keyTables" tables - * @throws SQLException - */ - public DefaultTagTranslator(DataSource source) throws OSHDBKeytablesNotFoundException, SQLException { - this.conn = source.getConnection(); - this.keyToInt = new ConcurrentHashMap<>(0); - this.keyToString = new ConcurrentHashMap<>(0); - this.tagToInt = new ConcurrentHashMap<>(0); - this.tagToString = new ConcurrentHashMap<>(0); - this.roleToInt = new ConcurrentHashMap<>(0); - this.roleToString = new ConcurrentHashMap<>(0); - - // test connection for presence of actual "keytables" tables - EnumSet keyTables = - EnumSet.of(E_KEY, E_KEYVALUE, E_ROLE); - for (TableNames table : keyTables) { - try (Statement testTablePresentQuery = this.conn.createStatement()) { - var selectSql = format("select 1 from %s limit 1", table); - testTablePresentQuery.execute(selectSql); - } catch (SQLException e) { - throw new OSHDBKeytablesNotFoundException(); - } - } - - // create prepared statements for querying tags from keytables - try { - keyIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_KEY)); - keyTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_KEY)); - valueIdQuery = conn.prepareStatement(format("select k.ID as KEYID,kv.VALUEID as VALUEID" - + " from %s kv" - + " inner join %s k on k.ID = kv.KEYID" - + " where k.TXT = ? and kv.TXT = ?;", E_KEYVALUE, E_KEY)); - valueTxtQuery = conn.prepareStatement(format("select k.TXT as KEYTXT,kv.TXT as VALUETXT" - + " from %s kv" - + " inner join %s k on k.ID = kv.KEYID" - + " where k.ID = ? and kv.VALUEID = ?;", E_KEYVALUE, E_KEY)); - roleIdQuery = conn.prepareStatement(format("select ID from %s where TXT = ?;", E_ROLE)); - roleTxtQuery = conn.prepareStatement(format("select TXT from %s where ID = ?;", E_ROLE)); - } catch (SQLException e) { - throw new OSHDBKeytablesNotFoundException(); - } - } - - @Override - public void close() throws SQLException { - try { - keyIdQuery.close(); - keyTxtQuery.close(); - valueIdQuery.close(); - valueTxtQuery.close(); - roleIdQuery.close(); - roleTxtQuery.close(); - } finally { - conn.close(); - } - } - - @Override - public OSHDBTagKey getOSHDBTagKeyOf(String key) { - return getOSHDBTagKeyOf(new OSMTagKey(key)); - } - - @Override - public OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key) { - if (this.keyToInt.containsKey(key)) { - return this.keyToInt.get(key); - } - OSHDBTagKey keyInt; - try { - synchronized (keyIdQuery) { - keyIdQuery.setString(1, key.toString()); - try (ResultSet keys = keyIdQuery.executeQuery()) { - if (!keys.next()) { - LOG.debug("Unable to find tag key {} in keytables.", key); - keyInt = new OSHDBTagKey(getFakeId(key.toString())); - } else { - keyInt = new OSHDBTagKey(keys.getInt("ID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.keyToString.put(keyInt, key); - this.keyToInt.put(key, keyInt); - return keyInt; - } - - @Override - public OSHDBTag getOSHDBTagOf(String key, String value) { - return this.getOSHDBTagOf(new OSMTag(key, value)); - } - - @Override - public OSHDBTag getOSHDBTagOf(OSMTag tag) { - // check if Key and Value are in cache - if (this.tagToInt.containsKey(tag)) { - return this.tagToInt.get(tag); - } - OSHDBTag tagInt; - // key or value is not in cache so let's go toInt them - try { - synchronized (valueIdQuery) { - valueIdQuery.setString(1, tag.getKey()); - valueIdQuery.setString(2, tag.getValue()); - try (ResultSet values = valueIdQuery.executeQuery()) { - if (!values.next()) { - LOG.info("Unable to find tag {}={} in keytables.", tag.getKey(), tag.getValue()); - tagInt = new OSHDBTag( - this.getOSHDBTagKeyOf(tag.getKey()).toInt(), getFakeId(tag.getValue())); - } else { - tagInt = new OSHDBTag(values.getInt("KEYID"), values.getInt("VALUEID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.tagToString.put(tagInt, tag); - this.tagToInt.put(tag, tagInt); - return tagInt; - } - - @Override - public OSMTag lookupTag(OSHDBTag tag) { - // check if Key and Value are in cache - if (this.tagToString.containsKey(tag)) { - return this.tagToString.get(tag); - } - OSMTag tagString; - - // key or value is not in cache so let's go toInt them - try { - synchronized (valueTxtQuery) { - valueTxtQuery.setInt(1, tag.getKey()); - valueTxtQuery.setInt(2, tag.getValue()); - try (ResultSet values = valueTxtQuery.executeQuery()) { - if (!values.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find tag id %d=%d in keytables.", - tag.getKey(), tag.getValue() - )); - } else { - tagString = new OSMTag(values.getString("KEYTXT"), values.getString("VALUETXT")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - // put it in caches - this.tagToInt.put(tagString, tag); - this.tagToString.put(tag, tagString); - return tagString; - } - - @Override - public OSHDBRole getOSHDBRoleOf(String role) { - return this.getOSHDBRoleOf(new OSMRole(role)); - } - - @Override - public OSHDBRole getOSHDBRoleOf(OSMRole role) { - if (this.roleToInt.containsKey(role)) { - return this.roleToInt.get(role); - } - OSHDBRole roleInt; - try { - synchronized (roleIdQuery) { - roleIdQuery.setString(1, role.toString()); - try (ResultSet roles = roleIdQuery.executeQuery()) { - if (!roles.next()) { - LOG.info("Unable to find role {} in keytables.", role); - roleInt = OSHDBRole.of(getFakeId(role.toString())); - } else { - roleInt = OSHDBRole.of(roles.getInt("ID")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.roleToString.put(roleInt, role); - this.roleToInt.put(role, roleInt); - return roleInt; - } - - @Override - public OSMRole lookupRole(int role) { - return this.lookupRole(OSHDBRole.of(role)); - } - - @Override - public OSMRole lookupRole(OSHDBRole role) { - if (this.roleToString.containsKey(role)) { - return this.roleToString.get(role); - } - OSMRole roleString; - try { - synchronized (roleTxtQuery) { - roleTxtQuery.setInt(1, role.getId()); - try (ResultSet roles = roleTxtQuery.executeQuery()) { - if (!roles.next()) { - throw new OSHDBTagOrRoleNotFoundException(format( - "Unable to find role id %d in keytables.", role.getId() - )); - } else { - roleString = new OSMRole(roles.getString("TXT")); - } - } - } - } catch (SQLException ex) { - LOG.error(UNABLE_TO_ACCESS_KEYTABLES); - throw new OSHDBException(UNABLE_TO_ACCESS_KEYTABLES); - } - this.roleToInt.put(roleString, role); - this.roleToString.put(role, roleString); - return roleString; - } - - private int getFakeId(String s) { - return -(s.hashCode() & 0x7fffffff); - } -} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java new file mode 100644 index 000000000..5a8ffacf4 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -0,0 +1,203 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +import static java.util.stream.Collectors.toList; +import static org.heigit.ohsome.oshdb.util.tagtranslator.ClosableSqlArray.createArray; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import javax.sql.DataSource; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; +import org.heigit.ohsome.oshdb.util.TableNames; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; + +public class JdbcTagTranslator implements TagTranslator { + private static final String OSM_OSHDB_KEY = String.format("SELECT id, txt" + + " from %s k" + + " where k.txt = ?", TableNames.E_KEY); + + private static final String OSM_OSHDB_TAG = String.format("SELECT keyid, valueid, kv.txt" + + " from %s k" + + " left join %s kv on k.id = kv.keyid" + + " where k.txt = ? and kv.txt = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); + + private static final String OSHDB_OSM_TAG = String.format("SELECT k.txt, kv.txt, valueid" + + " from %s k" + + " left join %s kv on k.id = kv.keyid" + + " where k.id = ? and kv.valueid = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); + + private static final String OSM_OSHDB_ROLE = String.format("SELECT id, txt" + + " from %s " + + " where txt = any (?)", TableNames.E_ROLE); + + private static final String OSHDB_OSM_ROLE = String.format("SELECT txt, id" + + " from %s " + + " where id = any (?)", TableNames.E_ROLE); + + private final DataSource source; + private final Cache stringPool; + + public JdbcTagTranslator(DataSource source) { + this.source = source; + stringPool = Caffeine.newBuilder().weakValues().build(); + } + + private String intern(String s) { + return stringPool.get(s, i -> i); + } + + @Override + public Optional getOSHDBTagKeyOf(OSMTagKey key) { + try (var conn = source.getConnection(); + var pstmt = conn.prepareStatement(OSM_OSHDB_KEY)) { + pstmt.setString(1, key.toString()); + try (var rst = pstmt.executeQuery()) { + if (rst.next()) { + return Optional.of(new OSHDBTagKey(rst.getInt(1))); + } + return Optional.empty(); + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + @Override + public Optional getOSHDBTagOf(OSMTag tag) { + return Optional.ofNullable(loadTags(tag.getKey(), Map.of(tag.getValue(), tag)).get(tag)); + } + + @Override + public Map getOSHDBTagOf(Set tags) { + var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); + tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) + .put(tag.getValue(), tag)); + + var result = Maps.newConcurrentMap(); + keyTags.entrySet().parallelStream() + .map(entry -> loadTags(entry.getKey(), entry.getValue())) + .forEach(result::putAll); + return result; + } + + private Map loadTags(String key, Map values) { + try (var conn = source.getConnection(); + var sqlArray = createArray(conn, "text", values.keySet()); + var pstmt = conn.prepareStatement(OSM_OSHDB_TAG)) { + pstmt.setString(1, key); + pstmt.setArray(2, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + var map = Maps.newHashMapWithExpectedSize(values.size()); + while (rst.next()) { + var keyId = rst.getInt(1); + var valId = rst.getInt(2); + var value = rst.getString(3); + map.put(values.get(value), new OSHDBTag(keyId, valId)); + } + return map; + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + @Override + public Optional getOSHDBRoleOf(OSMRole role) { + return Optional.ofNullable(loadRoles(Set.of(role)).get(role)); + } + + @Override + public Map getOSHDBRoleOf(Set< ? extends OSMRole> roles) { + return loadRoles(roles); + } + + private Map loadRoles(Set roles) { + try (var conn = source.getConnection(); + var sqlArray = + createArray(conn, "text", roles.stream().map(OSMRole::toString).collect(toList())); + var pstmt = conn.prepareStatement(OSM_OSHDB_ROLE)) { + pstmt.setArray(1, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + var map = Maps.newHashMapWithExpectedSize(roles.size()); + while (rst.next()) { + var id = rst.getInt(1); + var txt = rst.getString(2); + map.put(new OSMRole(txt), OSHDBRole.of(id)); + } + return map; + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + @Override + public OSMTag lookupTag(OSHDBTag tag) { + return lookupTags(tag.getKey(), Map.of(tag.getValue(), tag)).get(tag); + } + + @Override + public Map lookupTag(Set tags) { + var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); + tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) + .put(tag.getValue(), tag)); + + var result = Maps.newConcurrentMap(); + keyTags.entrySet().parallelStream() + .map(entry -> lookupTags(entry.getKey(), entry.getValue())) + .forEach(result::putAll); + return result; + } + + private Map lookupTags(int key, Map values) { + try (var conn = source.getConnection(); + var sqlArray = createArray(conn, "int", values.keySet()); + var pstmt = conn.prepareStatement(OSHDB_OSM_TAG)) { + pstmt.setInt(1, key); + pstmt.setArray(2, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + var map = Maps.newHashMapWithExpectedSize(values.size()); + while (rst.next()) { + var keyTxt = intern(rst.getString(1)); + var valTxt = rst.getString(2); + var valId = rst.getInt(3); + map.put(values.get(valId), new OSMTag(keyTxt, valTxt)); + } + return map; + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + @Override + public OSMRole lookupRole(OSHDBRole role) { + return lookupRoles(Set.of(role)).get(role); + } + + private Map lookupRoles(Set roles) { + try (var conn = source.getConnection(); + var sqlArray = + createArray(conn, "int", roles.stream().map(OSHDBRole::getId).collect(toList())); + var pstmt = conn.prepareStatement(OSHDB_OSM_ROLE)) { + pstmt.setArray(1, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + var map = Maps.newHashMapWithExpectedSize(roles.size()); + while (rst.next()) { + var txt = rst.getString(1); + var id = rst.getInt(2); + map.put(OSHDBRole.of(id), new OSMRole(txt)); + } + return map; + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index 5066ef499..50553cadb 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -1,18 +1,24 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; public interface TagTranslator { + /** * Get oshdb's internal representation of a tag key (string). * * @param key the tag key as a string * @return the corresponding oshdb representation of this key */ - OSHDBTagKey getOSHDBTagKeyOf(String key); + default Optional getOSHDBTagKeyOf(String key) { + return getOSHDBTagKeyOf(new OSMTagKey(key)); + } /** * Get oshdb's internal representation of a tag key. @@ -20,7 +26,7 @@ public interface TagTranslator { * @param key the tag key as an OSMTagKey object * @return the corresponding oshdb representation of this key */ - OSHDBTagKey getOSHDBTagKeyOf(OSMTagKey key); + Optional getOSHDBTagKeyOf(OSMTagKey key); /** * Get oshdb's internal representation of a tag (key-value string pair). @@ -29,7 +35,9 @@ public interface TagTranslator { * @param value the (string) value of the tag * @return the corresponding oshdb representation of this tag */ - OSHDBTag getOSHDBTagOf(String key, String value); + default Optional getOSHDBTagOf(String key, String value) { + return getOSHDBTagOf(new OSMTag(key, value)); + } /** * Get oshdb's internal representation of a tag (key-value pair). @@ -37,7 +45,9 @@ public interface TagTranslator { * @param tag a key-value pair as an OSMTag object * @return the corresponding oshdb representation of this tag */ - OSHDBTag getOSHDBTagOf(OSMTag tag); + Optional getOSHDBTagOf(OSMTag tag); + + Map getOSHDBTagOf(Set tags); /** * Get oshdb's internal representation of a role (string). @@ -45,7 +55,9 @@ public interface TagTranslator { * @param role the role string to fetch * @return the corresponding oshdb representation of this role */ - OSHDBRole getOSHDBRoleOf(String role); + default Optional getOSHDBRoleOf(String role) { + return getOSHDBRoleOf(new OSMRole(role)); + } /** * Get oshdb's internal representation of a role. @@ -53,7 +65,9 @@ public interface TagTranslator { * @param role the role to fetch as an OSMRole object * @return the corresponding oshdb representation of this role */ - OSHDBRole getOSHDBRoleOf(OSMRole role); + Optional getOSHDBRoleOf(OSMRole role); + + Map getOSHDBRoleOf(Set roles); /** * Get a tag's string representation from oshdb's internal data format. @@ -64,6 +78,8 @@ public interface TagTranslator { */ OSMTag lookupTag(OSHDBTag tag); + Map lookupTag(Set tags); + /** * Get a tag's string representation from oshdb's internal data format. * @@ -89,5 +105,7 @@ default OSMTag lookupTag(int key, int value) { * @param role the role ID (represented as an integer) * @return the textual representation of this role */ - OSMRole lookupRole(int role); + default OSMRole lookupRole(int role) { + return lookupRole(OSHDBRole.of(role)); + } } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java index 48b4a0a60..09e2dd28d 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java @@ -14,7 +14,7 @@ import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; -import org.heigit.ohsome.oshdb.util.tagtranslator.DefaultTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -53,15 +53,15 @@ void testIssue108() throws Exception { int countCreated = 0; int countOther = 0; - TagTranslator tt = new DefaultTagTranslator(source); + var tt = new JdbcTagTranslator(source); try (var conn = source.getConnection(); var stmt = conn.createStatement(); var oshCellsRawData = stmt.executeQuery("select data from " + TableNames.T_NODES)) { while (oshCellsRawData.next()) { // get one cell from the raw data stream - GridOSHEntity oshCellRawData = (GridOSHEntity) (new ObjectInputStream( - oshCellsRawData.getBinaryStream(1)) - ).readObject(); + GridOSHEntity oshCellRawData = + (GridOSHEntity) (new ObjectInputStream(oshCellsRawData.getBinaryStream(1))) + .readObject(); TreeSet timestamps = new TreeSet<>(); timestamps.add(new OSHDBTimestamp(1325376000L)); diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java index 7e7e02028..1256b3b14 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java @@ -1,10 +1,15 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Map; import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osm.OSM; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.junit.jupiter.api.AfterAll; @@ -38,50 +43,99 @@ static void breakDownClass() throws SQLException { @Test void testTag2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMTag tag = new OSMTag("building", "yes"); - try (var instance = new DefaultTagTranslator(source)){ - OSHDBTag expResult = new OSHDBTag(1, 0); - OSHDBTag result = instance.getOSHDBTagOf(tag); - assertEquals(expResult, result); + var instance = new JdbcTagTranslator(source); + OSHDBTag expResult = new OSHDBTag(1, 0); + OSHDBTag result = instance.getOSHDBTagOf(tag).get(); + assertEquals(expResult, result); + } + + @Test + void testTags2Int() throws OSHDBKeytablesNotFoundException, SQLException { + var tags = Map.of( + new OSMTag("building", "yes"), new OSHDBTag(1, 0), + new OSMTag("building", "residential"), new OSHDBTag(1, 2), + new OSMTag("highway", "primary"), new OSHDBTag(2, 7)); + + var instance = new JdbcTagTranslator(source); + var result = instance.getOSHDBTagOf(tags.keySet()); + assertEquals(tags.size(), result.size()); + for (var entry : tags.entrySet()) { + var tag = result.get(entry.getKey()); + assertEquals(entry.getValue(), tag); } } + @Test void testTag2String() throws OSHDBKeytablesNotFoundException, SQLException { OSHDBTag tag = new OSHDBTag(1, 2); - try (var instance = new DefaultTagTranslator(source)){ - OSMTag expResult = new OSMTag("building", "residential"); - OSMTag result = instance.lookupTag(tag); - assertEquals(expResult, result); + OSMTag expResult = new OSMTag("building", "residential"); + var instance = new JdbcTagTranslator(source); + OSMTag result = instance.lookupTag(tag); + assertEquals(expResult, result); + } + + @Test + void testTags2String() { + var tags = Map.of( + new OSHDBTag(1, 0), new OSMTag("building", "yes"), + new OSHDBTag(1, 2), new OSMTag("building", "residential"), + new OSHDBTag(2, 7), new OSMTag("highway", "primary")); + var instance = new JdbcTagTranslator(source); + var result = instance.lookupTag(tags.keySet()); + assertEquals(tags.size(), result.size()); + for (var entry : tags.entrySet()) { + var tag = result.get(entry.getKey()); + assertEquals(entry.getValue(), tag); } } + @Test + void testOSMEntityTag2String() { + var osm = OSM.node(123, 1, 1000L, 100L, 1, new int[] {1, 0, 2, 7}, 0, 0); + var instance = new JdbcTagTranslator(source); + var tags = instance.lookupTag(osm.getTags()); + assertEquals(2, tags.size()); + assertEquals(new OSMTag("building", "yes"), tags.get(new OSHDBTag(1, 0))); + assertEquals(new OSMTag("highway", "primary"), tags.get(new OSHDBTag(2, 7))); + } + @Test void testKey2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMTagKey key = new OSMTagKey("highway"); - try (var instance = new DefaultTagTranslator(source)) { - OSHDBTagKey expResult = new OSHDBTagKey(2); - OSHDBTagKey result = instance.getOSHDBTagKeyOf(key); - assertEquals(expResult, result); - } + var instance = new JdbcTagTranslator(source); + OSHDBTagKey expResult = new OSHDBTagKey(2); + OSHDBTagKey result = instance.getOSHDBTagKeyOf(key).get(); + assertEquals(expResult, result); } @Test void testRole2Int() throws OSHDBKeytablesNotFoundException, SQLException { OSMRole role = new OSMRole("from"); - try (var instance = new DefaultTagTranslator(source)){ - OSHDBRole expResult = OSHDBRole.of(4); - OSHDBRole result = instance.getOSHDBRoleOf(role); - assertEquals(expResult, result); - } + var instance = new JdbcTagTranslator(source); + OSHDBRole expResult = OSHDBRole.of(4); + OSHDBRole result = instance.getOSHDBRoleOf(role).get(); + assertEquals(expResult, result); } @Test void testRole2String() throws OSHDBKeytablesNotFoundException, SQLException { OSHDBRole role = OSHDBRole.of(1); - try (var instance = new DefaultTagTranslator(source)){ - OSMRole expResult = new OSMRole("inner"); - OSMRole result = instance.lookupRole(role); - assertEquals(expResult, result); + var instance = new JdbcTagTranslator(source); + OSMRole expResult = new OSMRole("inner"); + OSMRole result = instance.lookupRole(role); + assertEquals(expResult, result); + } + + @Test + void testKeysIdentity() { + var instance = new JdbcTagTranslator(source); + var tags = new ArrayList(10); + for (var i = 0; i < tags.size(); i++) { + tags.add(instance.lookupTag(new OSHDBTag(1, i))); + } + for (var i = 1; i < tags.size(); i++) { + assertTrue(tags.get(i - 1).getKey() == tags.get(i).getKey()); } } } From 88c38a7190096910d3200d6030f2203920afbd4a Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 18 Oct 2022 12:55:50 +0200 Subject: [PATCH 08/34] fix broken javadoc --- .../main/java/org/heigit/ohsome/oshdb/util/TableNames.java | 4 ++++ .../oshdb/util/taginterpreter/DefaultTagInterpreter.java | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/TableNames.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/TableNames.java index 58a78cc29..e2520f352 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/TableNames.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/TableNames.java @@ -54,6 +54,8 @@ public String toString() { /** * Returns the table name with a given {@link String} prepended. + * + * @return table name with prefix */ public String toString(String prefix) { if (prefix != null && !prefix.trim().isEmpty()) { @@ -64,6 +66,8 @@ public String toString(String prefix) { /** * Returns the {@link TableNames} object for a given {@link OSMType}. + * + * @return TableName for type */ public static Optional forOSMType(OSMType type) { switch (type) { diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java index 64e8c007f..14f895a7f 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java @@ -38,7 +38,7 @@ public class DefaultTagInterpreter extends BaseTagInterpreter { = "json/uninterestingTags.json"; /** - * Constructor using given {@link DefaultTagTranslator} and default values as areaTagsDefinitonFile and + * Constructor using given {@link TagTranslator} and default values as areaTagsDefinitonFile and * uninterestingTagsDefinitionFile. * *

@@ -55,10 +55,10 @@ public DefaultTagInterpreter(TagTranslator tagTranslator) throws IOException, Pa } /** - * Constructor using given {@link DefaultTagTranslator}, areaTagsDefinitonFile, and + * Constructor using given {@link TagTranslator}, areaTagsDefinitonFile, and * uninterestingTagsDefinitionFile. * - * @param tagTranslator {@link DefaultTagTranslator} used by {@link TagInterpreter} + * @param tagTranslator {@link TagTranslator} used by {@link TagInterpreter} * @param areaTagsDefinitionFile filename of a JSON file containing tags that are supposed to be * areas * @param uninterestingTagsDefinitionFile filename of a JSON file containing tags to be ignored From b509437508fd1f33117832970f6991fce0f974e6 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 3 Nov 2022 10:21:35 +0100 Subject: [PATCH 09/34] cache keys in jdbctagtranslator --- .../tagtranslator/CachedTagTranslator.java | 8 +-- .../util/tagtranslator/JdbcTagTranslator.java | 57 +++++++++++++------ .../util/tagtranslator/TagTranslatorTest.java | 23 ++++---- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java index 956b4f910..4fea4e8b4 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java @@ -11,10 +11,10 @@ public class CachedTagTranslator implements TagTranslator { - private TagTranslator source; + private final TagTranslator source; - private Cache lookupOSHDBTag; - private Cache lookupOSHDBRole; + private final Cache lookupOSHDBTag; + private final Cache lookupOSHDBRole; public CachedTagTranslator(TagTranslator source, long maxBytesValues, int maxNumRoles) { this.source = source; @@ -50,7 +50,7 @@ public Optional getOSHDBTagOf(OSMTag osm) { @Override public Map getOSHDBTagOf(Set tags) { var oshdb = source.getOSHDBTagOf(tags); - oshdb.entrySet().forEach(entry -> lookupOSHDBTag.put(entry.getValue(), entry.getKey())); + oshdb.forEach((key, value) -> lookupOSHDBTag.put(value, key)); return oshdb; } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 5a8ffacf4..bbe6eeb1f 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -27,7 +27,12 @@ public class JdbcTagTranslator implements TagTranslator { + " left join %s kv on k.id = kv.keyid" + " where k.txt = ? and kv.txt = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); - private static final String OSHDB_OSM_TAG = String.format("SELECT k.txt, kv.txt, valueid" + + private static final String OSHDB_OSM_KEY = String.format("SELECT txt, id" + + " from %s " + + " where id = any(?)", TableNames.E_KEY); + + private static final String OSHDB_OSM_TAG = String.format("SELECT kv.txt, valueid" + " from %s k" + " left join %s kv on k.id = kv.keyid" + " where k.id = ? and kv.valueid = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); @@ -41,15 +46,12 @@ public class JdbcTagTranslator implements TagTranslator { + " where id = any (?)", TableNames.E_ROLE); private final DataSource source; - private final Cache stringPool; + private final Cache cacheKeys; public JdbcTagTranslator(DataSource source) { this.source = source; - stringPool = Caffeine.newBuilder().weakValues().build(); - } - - private String intern(String s) { - return stringPool.get(s, i -> i); + cacheKeys = Caffeine.newBuilder() + .build(); } @Override @@ -78,7 +80,6 @@ public Map getOSHDBTagOf(Set tags) { var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) .put(tag.getValue(), tag)); - var result = Maps.newConcurrentMap(); keyTags.entrySet().parallelStream() .map(entry -> loadTags(entry.getKey(), entry.getValue())) @@ -86,6 +87,8 @@ public Map getOSHDBTagOf(Set tags) { return result; } + + private Map loadTags(String key, Map values) { try (var conn = source.getConnection(); var sqlArray = createArray(conn, "text", values.keySet()); @@ -139,34 +142,56 @@ private Map loadRoles(Set roles) { @Override public OSMTag lookupTag(OSHDBTag tag) { - return lookupTags(tag.getKey(), Map.of(tag.getValue(), tag)).get(tag); + var keyTxt = cacheKeys.getAll(Set.of(tag.getKey()), this::lookupKeys).get(tag.getKey()); + return lookupTags(tag.getKey(), keyTxt, Map.of(tag.getValue(), tag)).get(tag); } + + @Override public Map lookupTag(Set tags) { var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) .put(tag.getValue(), tag)); - + var keys = cacheKeys.getAll(keyTags.keySet(), this::lookupKeys); var result = Maps.newConcurrentMap(); keyTags.entrySet().parallelStream() - .map(entry -> lookupTags(entry.getKey(), entry.getValue())) + .map(entry -> lookupTags(entry.getKey(), keys.get(entry.getKey()), entry.getValue())) .forEach(result::putAll); return result; } - private Map lookupTags(int key, Map values) { + private Map lookupKeys(Set osm) { + try (var conn = source.getConnection(); + var sqlArray = createArray(conn, "int", osm); + var pstmt = conn.prepareStatement(OSHDB_OSM_KEY)) { + pstmt.setArray(1, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + var map = Maps.newHashMapWithExpectedSize(osm.size()); + while (rst.next()) { + var keyTxt = rst.getString(1); + var keyId = rst.getInt(2); + map.put(keyId, keyTxt); + } + return map; + } + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + + private Map lookupTags(int keyId, String keyTxt, Map values) { try (var conn = source.getConnection(); var sqlArray = createArray(conn, "int", values.keySet()); var pstmt = conn.prepareStatement(OSHDB_OSM_TAG)) { - pstmt.setInt(1, key); + pstmt.setInt(1, keyId); pstmt.setArray(2, sqlArray.get()); try (var rst = pstmt.executeQuery()) { var map = Maps.newHashMapWithExpectedSize(values.size()); while (rst.next()) { - var keyTxt = intern(rst.getString(1)); - var valTxt = rst.getString(2); - var valId = rst.getInt(3); + var valTxt = rst.getString(1); + var valId = rst.getInt(2); map.put(values.get(valId), new OSMTag(keyTxt, valTxt)); } return map; diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java index 1256b3b14..ea207f83b 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Map; import org.h2.jdbcx.JdbcConnectionPool; @@ -17,7 +16,7 @@ import org.junit.jupiter.api.Test; /** - * Tests the {@link DefaultTagTranslator} class. + * Tests the {@link TagTranslator} class. */ class TagTranslatorTest { private static JdbcConnectionPool source; @@ -25,23 +24,21 @@ class TagTranslatorTest { /** * Initialize tests by loading the H2 driver and open a connection via jdbc. * - * @throws ClassNotFoundException gets thrown if H2 driver class cannot be found - * @throws SQLException is thrown if the connection fails */ @BeforeAll - static void setUpClass() throws ClassNotFoundException, SQLException { + static void setUpClass() { source = JdbcConnectionPool.create("jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", "sa", ""); } @AfterAll - static void breakDownClass() throws SQLException { + static void breakDownClass() { source.dispose(); } TagTranslatorTest() {} @Test - void testTag2Int() throws OSHDBKeytablesNotFoundException, SQLException { + void testTag2Int() throws OSHDBKeytablesNotFoundException { OSMTag tag = new OSMTag("building", "yes"); var instance = new JdbcTagTranslator(source); OSHDBTag expResult = new OSHDBTag(1, 0); @@ -50,7 +47,7 @@ void testTag2Int() throws OSHDBKeytablesNotFoundException, SQLException { } @Test - void testTags2Int() throws OSHDBKeytablesNotFoundException, SQLException { + void testTags2Int() throws OSHDBKeytablesNotFoundException { var tags = Map.of( new OSMTag("building", "yes"), new OSHDBTag(1, 0), new OSMTag("building", "residential"), new OSHDBTag(1, 2), @@ -67,7 +64,7 @@ void testTags2Int() throws OSHDBKeytablesNotFoundException, SQLException { @Test - void testTag2String() throws OSHDBKeytablesNotFoundException, SQLException { + void testTag2String() throws OSHDBKeytablesNotFoundException { OSHDBTag tag = new OSHDBTag(1, 2); OSMTag expResult = new OSMTag("building", "residential"); var instance = new JdbcTagTranslator(source); @@ -101,7 +98,7 @@ void testOSMEntityTag2String() { } @Test - void testKey2Int() throws OSHDBKeytablesNotFoundException, SQLException { + void testKey2Int() throws OSHDBKeytablesNotFoundException { OSMTagKey key = new OSMTagKey("highway"); var instance = new JdbcTagTranslator(source); OSHDBTagKey expResult = new OSHDBTagKey(2); @@ -110,7 +107,7 @@ void testKey2Int() throws OSHDBKeytablesNotFoundException, SQLException { } @Test - void testRole2Int() throws OSHDBKeytablesNotFoundException, SQLException { + void testRole2Int() throws OSHDBKeytablesNotFoundException { OSMRole role = new OSMRole("from"); var instance = new JdbcTagTranslator(source); OSHDBRole expResult = OSHDBRole.of(4); @@ -119,7 +116,7 @@ void testRole2Int() throws OSHDBKeytablesNotFoundException, SQLException { } @Test - void testRole2String() throws OSHDBKeytablesNotFoundException, SQLException { + void testRole2String() throws OSHDBKeytablesNotFoundException { OSHDBRole role = OSHDBRole.of(1); var instance = new JdbcTagTranslator(source); OSMRole expResult = new OSMRole("inner"); @@ -131,7 +128,7 @@ void testRole2String() throws OSHDBKeytablesNotFoundException, SQLException { void testKeysIdentity() { var instance = new JdbcTagTranslator(source); var tags = new ArrayList(10); - for (var i = 0; i < tags.size(); i++) { + for (var i = 0; i < 10; i++) { tags.add(instance.lookupTag(new OSHDBTag(1, i))); } for (var i = 1; i < tags.size(); i++) { From 9a4abd30de93134d284b0e22368aea292b5cb983 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 12:21:19 +0100 Subject: [PATCH 10/34] #470: make disposal of connection pool null-safe --- .../main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java index c0006386e..14f626e58 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java @@ -8,6 +8,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -19,12 +20,14 @@ import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import static java.util.Optional.ofNullable; + /** * OSHDB database backend connector to a H2 database. */ public class OSHDBH2 extends OSHDBJdbc { - private JdbcConnectionPool connectionPool; + private final JdbcConnectionPool connectionPool; /** * Opens a connection to oshdb data stored in a H2 database file. @@ -50,6 +53,7 @@ public OSHDBH2(Path path, String user, String password) { public OSHDBH2(DataSource ds) { super(ds); + this.connectionPool = null; } private OSHDBH2(JdbcConnectionPool ds) { @@ -76,7 +80,7 @@ public OSHDBH2 multithreading(boolean useMultithreading) { @Override public void close() throws Exception { try { - connectionPool.dispose(); + ofNullable(connectionPool).ifPresent(JdbcConnectionPool::dispose); } finally { super.close(); } From bf2ca5867f38c78b39ef7bdf04bc46ee571df4a7 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:04:38 +0100 Subject: [PATCH 11/34] #470: simplify method implementation --- .../api/mapreducer/backend/MapReducerJdbc.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java index fdede0ce7..60592aa7c 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java @@ -1,5 +1,7 @@ package org.heigit.ohsome.oshdb.api.mapreducer.backend; +import static java.util.stream.Collectors.joining; + import java.io.IOException; import java.io.ObjectInputStream; import java.sql.ResultSet; @@ -58,13 +60,13 @@ protected GridOSHEntity readOshCellRawData(ResultSet oshCellsRawData) } protected String sqlQuery() { - var sqlQuery = this.typeFilter.stream() - .map( - osmType -> TableNames.forOSMType(osmType).map(tn -> tn.toString(this.oshdb.prefix()))) - .filter(Optional::isPresent).map(Optional::get) + return this.typeFilter.stream() + .map(osmType -> TableNames.forOSMType(osmType) + .map(tn -> tn.toString(this.oshdb.prefix()))) + .filter(Optional::isPresent) + .map(Optional::get) .map(tn -> "(select data from " + tn + " where level = ?1 and id between ?2 and ?3)") - .collect(Collectors.joining(" union all ")); - return sqlQuery; + .collect(joining(" union all ")); } @Nonnull From 31c2bb4d6188211ed85cafb5613f29b82dbad212 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 13:51:50 +0100 Subject: [PATCH 12/34] #470: refactor to smaller methods --- .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 71 +++++++++++++------ 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index ba1a87e90..6bf7502e0 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -1,13 +1,16 @@ package org.heigit.ohsome.oshdb.api.db; +import static java.util.stream.Collectors.toList; + import com.google.common.base.Joiner; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; +import java.util.HashMap; +import java.util.HashSet; import java.util.Optional; -import java.util.stream.Collectors; +import java.util.Set; import java.util.stream.Stream; import javax.sql.DataSource; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; @@ -20,6 +23,7 @@ import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.jetbrains.annotations.NotNull; /** * OSHDB database backend connector to a JDBC database file. @@ -56,31 +60,52 @@ public OSHDBJdbc prefix(String prefix) { @Override public MapReducer createMapReducer(Class forClass) { try { - Collection expectedTables = Stream.of(OSMType.values()).map(TableNames::forOSMType) - .filter(Optional::isPresent).map(Optional::get) - .map(t -> t.toString(this.prefix()).toLowerCase()).collect(Collectors.toList()); - List allTables = new LinkedList<>(); - try (var conn = getConnection()) { - var metaData = conn.getMetaData(); - try (var rs = metaData.getTables(null, null, "%", new String[] {"TABLE"})) { - while (rs.next()) { - allTables.add(rs.getString("TABLE_NAME").toLowerCase()); - } - if (!allTables.containsAll(expectedTables)) { - throw new OSHDBTableNotFoundException(Joiner.on(", ").join(expectedTables)); - } - } - } + Collection expectedTables = getExpectedTables(); + checkTables(expectedTables); } catch (SQLException e) { throw new OSHDBException(e); } - MapReducer mapReducer; + + return mapReducer(forClass); + } + + @NotNull + private Collection getExpectedTables() { + return Stream.of(OSMType.values()) + .map(TableNames::forOSMType) + .filter(Optional::isPresent) + .map(Optional::get) + .map(table -> table.toString(this.prefix()).toLowerCase()) + .collect(toList()); + } + + private void checkTables(Collection expectedTables) throws SQLException { + try (var conn = getConnection()) { + try (var rs = getTables(conn)) { + var allTables = new HashSet(); + while (rs.next()) { + allTables.add(rs.getString("TABLE_NAME").toLowerCase()); + } + if (!allTables.containsAll(expectedTables)) { + throw new OSHDBTableNotFoundException(Joiner.on(", ").join(expectedTables)); + } + } + } + } + + private static ResultSet getTables(Connection connection) throws SQLException { + return connection + .getMetaData() + .getTables(null, null, "%", new String[]{"TABLE"}); + } + + @NotNull + private MapReducer mapReducer(Class forClass) { if (this.useMultithreading) { - mapReducer = new MapReducerJdbcMultithread<>(this, forClass); - } else { - mapReducer = new MapReducerJdbcSinglethread<>(this, forClass); + return new MapReducerJdbcMultithread<>(this, forClass); } - return mapReducer; + + return new MapReducerJdbcSinglethread<>(this, forClass); } @Override From da15d5d4accaf9f9fb52a9b0c9c8813b6f044d64 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:23:53 +0100 Subject: [PATCH 13/34] #470: refactor anonymous implementation to inner class --- .../mapreducer/backend/MapReducerJdbc.java | 95 +++++++++++-------- 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java index 60592aa7c..011e4c3da 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java @@ -1,16 +1,16 @@ package org.heigit.ohsome.oshdb.api.mapreducer.backend; +import static java.util.Spliterators.spliteratorUnknownSize; import static java.util.stream.Collectors.joining; import java.io.IOException; import java.io.ObjectInputStream; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Optional; -import java.util.Spliterators; -import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nonnull; @@ -82,49 +82,60 @@ protected Stream getOshCellsStream(CellIdRange cellIdRange) { pstmt.setLong(2, cellIdRange.getStart().getId()); pstmt.setLong(3, cellIdRange.getEnd().getId()); var oshCellsRawData = pstmt.executeQuery(); - return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - new Iterator() { - GridOSHEntity next; - @Override - public boolean hasNext() { - return next != null || (next = getNext()) != null; - } - - @Override - public GridOSHEntity next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - var grid = next; - next = null; - return grid; - } - - private GridOSHEntity getNext() { - try { - if (!oshCellsRawData.next()) { - try { - oshCellsRawData.close(); - } finally { - conn.close(); - } - return null; - } - return readOshCellRawData(oshCellsRawData); - } catch (IOException | ClassNotFoundException | SQLException e) { - var exception = new OSHDBException(e); - try { - conn.close(); - } catch (Exception e2) { - exception.addSuppressed(e2); - } - throw exception; - } - } - }, 0 + return StreamSupport.stream(spliteratorUnknownSize( + new GridOSHEntityIterator(oshCellsRawData, conn), 0 ), false); } catch (SQLException e) { throw new OSHDBException(e); } } + + private class GridOSHEntityIterator implements Iterator { + + private final ResultSet oshCellsRawData; + private final Connection conn; + GridOSHEntity next; + + public GridOSHEntityIterator(ResultSet oshCellsRawData, Connection conn) { + this.oshCellsRawData = oshCellsRawData; + this.conn = conn; + } + + @Override + public boolean hasNext() { + return next != null || (next = getNext()) != null; + } + + @Override + public GridOSHEntity next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + var grid = next; + next = null; + return grid; + } + + private GridOSHEntity getNext() { + try { + if (!oshCellsRawData.next()) { + try { + oshCellsRawData.close(); + } finally { + conn.close(); + } + return null; + } + return readOshCellRawData(oshCellsRawData); + } catch (IOException | ClassNotFoundException | SQLException e) { + var exception = new OSHDBException(e); + try { + conn.close(); + } catch (Exception e2) { + exception.addSuppressed(e2); + } + throw exception; + } + } + } } From 912fd41cb8f83a8ec8cf57b044845aa434474bde Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:33:16 +0100 Subject: [PATCH 14/34] #470: remove redundant exception --- .../java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java index 9ebe2a9d9..b406dedb8 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java @@ -29,7 +29,7 @@ class TestCollect { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestCollect() throws Exception { + TestCollect() { oshdb = new OSHDBH2("../data/test-data"); } From fa4e90cd675f66d4ee2af312b70213b63e84a454 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:41:14 +0100 Subject: [PATCH 15/34] #470: rename all test names to postfix-convention --- ...ReduceOSHDBIgnite.java => MapReduceTestOSHDBIgnite.java} | 4 ++-- ...yCall.java => MapReduceTestOSHDBIgniteAffinityCall.java} | 4 ++-- ...ocalPeek.java => MapReduceTestOSHDBIgniteLocalPeek.java} | 4 ++-- ...Cache.java => MapReduceTestOSHDBIgniteMissingCache.java} | 4 ++-- ...canQuery.java => MapReduceTestOSHDBIgniteScanQuery.java} | 4 ++-- .../oshdb/api/tests/{TestCollect.java => CollectTest.java} | 5 ++--- ...Entity.java => FlatMapAggregateGroupedByEntityTest.java} | 4 ++-- ...{TestFlatMapAggregate.java => FlatMapAggregateTest.java} | 4 ++-- ...FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest.java} | 6 +++--- ...latMapReduceGroupedByEntityOSHDBH2SinglethreadTest.java} | 6 +++--- ...dByEntity.java => FlatMapReduceGroupedByEntityTest.java} | 4 ++-- .../{TestFlatMapReduce.java => FlatMapReduceTest.java} | 4 ++-- .../oshdb/api/tests/{TestForEach.java => ForEachTest.java} | 4 ++-- ...ibutionView.java => HelpersOSMContributionViewTest.java} | 4 ++-- ...pshotView.java => HelpersOSMEntitySnapshotViewTest.java} | 4 ++-- .../tests/{TestLambdaFilter.java => LambdaFilterTest.java} | 4 ++-- ...egateByGeometry.java => MapAggregateByGeometryTest.java} | 4 ++-- ...apAggregateByIndex.java => MapAggregateByIndexTest.java} | 4 ++-- ...ateByTimestamp.java => MapAggregateByTimestampTest.java} | 4 ++-- ...ultithread.java => MapReduceOSHDBH2MultithreadTest.java} | 4 ++-- ...glethread.java => MapReduceOSHDBH2SinglethreadTest.java} | 4 ++-- ...Tables.java => MapReduceOSHDBJdbcMissingTablesTest.java} | 4 ++-- .../api/tests/{TestMapReduce.java => MapReduceTest.java} | 5 ++--- .../tests/{TestOSHDBFilter.java => OSHDBFilterTest.java} | 4 ++-- ...Id.java => OSMContributionGetContributorUserIdTest.java} | 4 ++-- .../{TestOSMDataFilters.java => OSMDataFiltersTest.java} | 4 ++-- .../api/tests/{TestQuantiles.java => QuantilesTest.java} | 4 ++-- .../oshdb/api/tests/{TestStream.java => StreamTest.java} | 4 ++-- 28 files changed, 58 insertions(+), 60 deletions(-) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBIgnite.java => MapReduceTestOSHDBIgnite.java} (96%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBIgniteAffinityCall.java => MapReduceTestOSHDBIgniteAffinityCall.java} (90%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBIgniteLocalPeek.java => MapReduceTestOSHDBIgniteLocalPeek.java} (77%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBIgniteMissingCache.java => MapReduceTestOSHDBIgniteMissingCache.java} (92%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBIgniteScanQuery.java => MapReduceTestOSHDBIgniteScanQuery.java} (85%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestCollect.java => CollectTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapAggregateGroupedByEntity.java => FlatMapAggregateGroupedByEntityTest.java} (96%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapAggregate.java => FlatMapAggregateTest.java} (97%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapReduceGroupedByEntityOSHDBH2Multithread.java => FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest.java} (71%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread.java => FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest.java} (71%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapReduceGroupedByEntity.java => FlatMapReduceGroupedByEntityTest.java} (96%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestFlatMapReduce.java => FlatMapReduceTest.java} (97%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestForEach.java => ForEachTest.java} (97%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestHelpersOSMContributionView.java => HelpersOSMContributionViewTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestHelpersOSMEntitySnapshotView.java => HelpersOSMEntitySnapshotViewTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestLambdaFilter.java => LambdaFilterTest.java} (97%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapAggregateByGeometry.java => MapAggregateByGeometryTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapAggregateByIndex.java => MapAggregateByIndexTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapAggregateByTimestamp.java => MapAggregateByTimestampTest.java} (99%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBH2Multithread.java => MapReduceOSHDBH2MultithreadTest.java} (77%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBH2Singlethread.java => MapReduceOSHDBH2SinglethreadTest.java} (77%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduceOSHDBJdbcMissingTables.java => MapReduceOSHDBJdbcMissingTablesTest.java} (92%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestMapReduce.java => MapReduceTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestOSHDBFilter.java => OSHDBFilterTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestOSMContributionGetContributorUserId.java => OSMContributionGetContributorUserIdTest.java} (98%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestOSMDataFilters.java => OSMDataFiltersTest.java} (96%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestQuantiles.java => QuantilesTest.java} (99%) rename oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/{TestStream.java => StreamTest.java} (97%) diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java similarity index 96% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java index 11bbe5b4a..fcb827a52 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgnite.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java @@ -29,7 +29,7 @@ import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.TableNames; -abstract class TestMapReduceOSHDBIgnite extends TestMapReduce { +abstract class MapReduceTestOSHDBIgnite extends MapReduceTest { static final Ignite ignite; static { @@ -98,7 +98,7 @@ private static OSHDBDatabase initOshdb(Consumer computeMode) { return oshdb; } - TestMapReduceOSHDBIgnite(Consumer computeMode) throws Exception { + MapReduceTestOSHDBIgnite(Consumer computeMode) throws Exception { super(initOshdb(computeMode)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java similarity index 90% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java index 8e5d5040c..cf36fe35f 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteAffinityCall.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java @@ -13,13 +13,13 @@ * *

Runs the tests using the "affinity call" Ignite backend.

*/ -class TestMapReduceOSHDBIgniteAffinityCall extends TestMapReduceOSHDBIgnite { +class MapReduceTestOSHDBIgniteAffinityCall extends MapReduceTestOSHDBIgnite { /** * Creates the test runner using the ignite affinitycall backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBIgniteAffinityCall() throws Exception { + MapReduceTestOSHDBIgniteAffinityCall() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL)); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java similarity index 77% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java index 3eb69e6d4..c3bc778ba 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteLocalPeek.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java @@ -7,13 +7,13 @@ * *

Runs the tests using the "local peek" Ignite backend.

*/ -class TestMapReduceOSHDBIgniteLocalPeek extends TestMapReduceOSHDBIgnite { +class MapReduceTestOSHDBIgniteLocalPeek extends MapReduceTestOSHDBIgnite { /** * Creates the test runner using the ignite localpeak backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBIgniteLocalPeek() throws Exception { + MapReduceTestOSHDBIgniteLocalPeek() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java similarity index 92% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java index f3f79dadf..ebfb4e990 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteMissingCache.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java @@ -8,13 +8,13 @@ /** * Tests for proper error messages is caches are not pnt on ignite. */ -class TestMapReduceOSHDBIgniteMissingCache extends TestMapReduceOSHDBIgnite { +class MapReduceTestOSHDBIgniteMissingCache extends MapReduceTestOSHDBIgnite { /** * Creates the test runner using an Ignite backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBIgniteMissingCache() throws Exception { + MapReduceTestOSHDBIgniteMissingCache() throws Exception { super(oshdb -> {}); this.oshdb.prefix(""); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java similarity index 85% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java index 75fa3006a..173a869d0 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBIgniteScanQuery.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java @@ -10,13 +10,13 @@ * *

Runs the tests using the "scan query" Ignite backend.

*/ -class TestMapReduceOSHDBIgniteScanQuery extends TestMapReduceOSHDBIgnite { +class MapReduceTestOSHDBIgniteScanQuery extends MapReduceTestOSHDBIgnite { /** * Creates the test runner using the ignite scanquery backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBIgniteScanQuery() throws Exception { + MapReduceTestOSHDBIgniteScanQuery() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY)); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/CollectTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/CollectTest.java index b406dedb8..b16afb7b4 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/CollectTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.Iterables; -import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.SortedMap; @@ -21,7 +20,7 @@ /** * Tests the collect method of the OSHDB API. */ -class TestCollect { +class CollectTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -29,7 +28,7 @@ class TestCollect { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestCollect() { + CollectTest() { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateGroupedByEntityTest.java similarity index 96% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateGroupedByEntityTest.java index ba0d69306..d1dca9656 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateGroupedByEntityTest.java @@ -21,7 +21,7 @@ /** * Test flat map method with groupByEntity of the MapAggregator class of the OSHDB API. */ -class TestFlatMapAggregateGroupedByEntity { +class FlatMapAggregateGroupedByEntityTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -30,7 +30,7 @@ class TestFlatMapAggregateGroupedByEntity { private static final double DELTA = 1e-8; - TestFlatMapAggregateGroupedByEntity() throws Exception { + FlatMapAggregateGroupedByEntityTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateTest.java similarity index 97% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateTest.java index 2efaadcf5..67f6aa37f 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapAggregateTest.java @@ -23,7 +23,7 @@ /** * Test flat map method of the MapAggregator class of the OSHDB API. */ -class TestFlatMapAggregate { +class FlatMapAggregateTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -32,7 +32,7 @@ class TestFlatMapAggregate { private static final double DELTA = 1e-8; - TestFlatMapAggregate() throws Exception { + FlatMapAggregateTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Multithread.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest.java similarity index 71% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Multithread.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest.java index e017d1d60..cfa2fcabe 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Multithread.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest.java @@ -7,14 +7,14 @@ * *

Runs the tests using the multithreaded H2 backend.

*/ -class TestFlatMapReduceGroupedByEntityOSHDBH2Multithread extends - TestFlatMapReduceGroupedByEntity { +class FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest extends + FlatMapReduceGroupedByEntityTest { /** * Creates the test runner using the multithreaded H2 backend. * * @throws Exception if something goes wrong */ - TestFlatMapReduceGroupedByEntityOSHDBH2Multithread() throws Exception { + FlatMapReduceGroupedByEntityOSHDBH2MultithreadTest() throws Exception { super( (new OSHDBH2("../data/test-data")).multithreading(true) ); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest.java similarity index 71% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest.java index b3f35c1fd..6022e98d2 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest.java @@ -7,14 +7,14 @@ * *

Runs the tests using the singlethreaded H2 backend.

*/ -class TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread extends - TestFlatMapReduceGroupedByEntity { +class FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest extends + FlatMapReduceGroupedByEntityTest { /** * Creates the test runner using the singlethreaded "dummy" H2 backend. * * @throws Exception if something goes wrong */ - TestFlatMapReduceGroupedByEntityOSHDBH2Singlethread() throws Exception { + FlatMapReduceGroupedByEntityOSHDBH2SinglethreadTest() throws Exception { super( (new OSHDBH2("../data/test-data")).multithreading(false) ); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityTest.java similarity index 96% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityTest.java index 350fc46bf..28e76d33a 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceGroupedByEntityTest.java @@ -19,7 +19,7 @@ /** * Test flatMap method with groupByEntity of the MapReducer class of the OSHDB API. */ -abstract class TestFlatMapReduceGroupedByEntity { +abstract class FlatMapReduceGroupedByEntityTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -28,7 +28,7 @@ abstract class TestFlatMapReduceGroupedByEntity { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestFlatMapReduceGroupedByEntity(OSHDBDatabase oshdb) throws Exception { + FlatMapReduceGroupedByEntityTest(OSHDBDatabase oshdb) throws Exception { this.oshdb = oshdb; } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceTest.java similarity index 97% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceTest.java index 6e51b82ec..ea5581939 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/FlatMapReduceTest.java @@ -23,14 +23,14 @@ /** * Test flat map method of the MapReducer class of the OSHDB API. */ -class TestFlatMapReduce { +class FlatMapReduceTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestFlatMapReduce() throws Exception { + FlatMapReduceTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/ForEachTest.java similarity index 97% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/ForEachTest.java index d803517e9..9ff62f586 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/ForEachTest.java @@ -15,7 +15,7 @@ /** * Test forEach method of the OSHDB API. */ -class TestForEach { +class ForEachTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -23,7 +23,7 @@ class TestForEach { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestForEach() throws Exception { + ForEachTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java index 596d24ff7..328da748d 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java @@ -21,7 +21,7 @@ /** * Test special reducers of the OSHDB API when using the contribution view. */ -class TestHelpersOSMContributionView { +class HelpersOSMContributionViewTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -32,7 +32,7 @@ class TestHelpersOSMContributionView { private static final double DELTA = 1e-8; - TestHelpersOSMContributionView() throws Exception { + HelpersOSMContributionViewTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMEntitySnapshotViewTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMEntitySnapshotViewTest.java index b0d9ed93c..6b574d19b 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMEntitySnapshotViewTest.java @@ -18,7 +18,7 @@ /** * Test special reducers of the OSHDB API when using the contribution view. */ -class TestHelpersOSMEntitySnapshotView { +class HelpersOSMEntitySnapshotViewTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -29,7 +29,7 @@ class TestHelpersOSMEntitySnapshotView { private static final double DELTA = 1e-8; - TestHelpersOSMEntitySnapshotView() throws Exception { + HelpersOSMEntitySnapshotViewTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/LambdaFilterTest.java similarity index 97% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/LambdaFilterTest.java index da4f6fe3f..3b4e76053 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/LambdaFilterTest.java @@ -18,7 +18,7 @@ /** * Tests lambda functions as filters. */ -class TestLambdaFilter { +class LambdaFilterTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -27,7 +27,7 @@ class TestLambdaFilter { private static final double DELTA = 1e-8; - TestLambdaFilter() throws Exception { + LambdaFilterTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java index b2de3ed2f..4d1a629b2 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java @@ -30,7 +30,7 @@ /** * Test aggregateByGeometry method of the OSHDB API. */ -class TestMapAggregateByGeometry { +class MapAggregateByGeometryTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -39,7 +39,7 @@ class TestMapAggregateByGeometry { private static final double DELTA = 1e-4; - TestMapAggregateByGeometry() throws Exception { + MapAggregateByGeometryTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByIndexTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByIndexTest.java index 21a9c2b7f..336e6aa4b 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByIndexTest.java @@ -23,7 +23,7 @@ /** * Test aggregate by custom index method of the OSHDB API. */ -class TestMapAggregateByIndex { +class MapAggregateByIndexTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -34,7 +34,7 @@ class TestMapAggregateByIndex { private static final double DELTA = 1e-8; - TestMapAggregateByIndex() throws Exception { + MapAggregateByIndexTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByTimestampTest.java similarity index 99% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByTimestampTest.java index 031992c9c..4d77175ce 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByTimestampTest.java @@ -24,7 +24,7 @@ /** * Test aggregateByTimestamp method of the OSHDB API. */ -class TestMapAggregateByTimestamp { +class MapAggregateByTimestampTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -34,7 +34,7 @@ class TestMapAggregateByTimestamp { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestMapAggregateByTimestamp() throws Exception { + MapAggregateByTimestampTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Multithread.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2MultithreadTest.java similarity index 77% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Multithread.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2MultithreadTest.java index c6a72722a..5fa3bd1d9 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Multithread.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2MultithreadTest.java @@ -7,13 +7,13 @@ * *

Runs the tests using the multithreaded H2 backend.

*/ -class TestMapReduceOSHDBH2Multithread extends TestMapReduce { +class MapReduceOSHDBH2MultithreadTest extends MapReduceTest { /** * Creates the test runner using the multithreaded H2 backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBH2Multithread() throws Exception { + MapReduceOSHDBH2MultithreadTest() throws Exception { super( (new OSHDBH2("../data/test-data")).multithreading(true) ); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Singlethread.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2SinglethreadTest.java similarity index 77% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Singlethread.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2SinglethreadTest.java index a0f2b490e..1352e3e9d 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBH2Singlethread.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBH2SinglethreadTest.java @@ -7,13 +7,13 @@ * *

Runs the tests using the singlethreaded H2 backend.

*/ -class TestMapReduceOSHDBH2Singlethread extends TestMapReduce { +class MapReduceOSHDBH2SinglethreadTest extends MapReduceTest { /** * Creates the test runner using the singlethreaded "dummy" H2 backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBH2Singlethread() throws Exception { + MapReduceOSHDBH2SinglethreadTest() throws Exception { super( (new OSHDBH2("../data/test-data")).multithreading(false) ); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBJdbcMissingTables.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBJdbcMissingTablesTest.java similarity index 92% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBJdbcMissingTables.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBJdbcMissingTablesTest.java index ccf549760..988501009 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduceOSHDBJdbcMissingTables.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBJdbcMissingTablesTest.java @@ -9,13 +9,13 @@ /** * Tests for proper error messages if tables are not present on H2. */ -class TestMapReduceOSHDBJdbcMissingTables extends TestMapReduce { +class MapReduceOSHDBJdbcMissingTablesTest extends MapReduceTest { /** * Creates the test runner using an H2 backend. * * @throws Exception if something goes wrong */ - TestMapReduceOSHDBJdbcMissingTables() throws Exception { + MapReduceOSHDBJdbcMissingTablesTest() throws Exception { super((new OSHDBH2("../data/test-data")) .prefix("") ); diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTest.java index c8bf7d5fc..81414ad53 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTest.java @@ -9,7 +9,6 @@ import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.OSHDBBoundingBox; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; -import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; @@ -23,7 +22,7 @@ /** * Base class for testing the map-reducer backend implementations of the OSHDB API. */ -abstract class TestMapReduce { +abstract class MapReduceTest { final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = bboxWgs84Coordinates(8.0, 49.0, 9.0, 50.0); @@ -32,7 +31,7 @@ abstract class TestMapReduce { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestMapReduce(OSHDBDatabase oshdb) throws Exception { + MapReduceTest(OSHDBDatabase oshdb) throws Exception { this.oshdb = oshdb; } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSHDBFilterTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSHDBFilterTest.java index 0ecab66a4..0b153aa92 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSHDBFilterTest.java @@ -26,7 +26,7 @@ * of unit tests. *

*/ -class TestOSHDBFilter { +class OSHDBFilterTest { private final OSHDBJdbc oshdb; private final FilterParser filterParser; @@ -38,7 +38,7 @@ class TestOSHDBFilter { * * @throws Exception if something goes wrong. */ - TestOSHDBFilter() throws Exception { + OSHDBFilterTest() throws Exception { OSHDBH2 oshdb = new OSHDBH2("../data/test-data"); filterParser = new FilterParser(oshdb.getTagTranslator()); this.oshdb = oshdb; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMContributionGetContributorUserIdTest.java similarity index 98% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMContributionGetContributorUserIdTest.java index 5acdddf26..c2cff019d 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMContributionGetContributorUserIdTest.java @@ -26,8 +26,8 @@ /** * Tests the get contributor user id method of the OSHDB API. */ -class TestOSMContributionGetContributorUserId { - TestOSMContributionGetContributorUserId() throws Exception {} +class OSMContributionGetContributorUserIdTest { + OSMContributionGetContributorUserIdTest() throws Exception {} private final OSHEntity dummyOshEntity = OSHNodeImpl.build(Collections.singletonList( OSM.node(-1L, 1, 0L, 1L, 1, new int[]{}, 0, 0) diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMDataFiltersTest.java similarity index 96% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMDataFiltersTest.java index 54744973d..2d6a90ca9 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/OSMDataFiltersTest.java @@ -18,14 +18,14 @@ /** * Tests osm data filters. */ -class TestOSMDataFilters { +class OSMDataFiltersTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = OSHDBBoundingBox.bboxWgs84Coordinates(8.651133, 49.387611, 8.6561, 49.390513); private final OSHDBTimestamps timestamps1 = new OSHDBTimestamps("2014-01-01"); - TestOSMDataFilters() throws Exception { + OSMDataFiltersTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/QuantilesTest.java similarity index 99% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/QuantilesTest.java index 82fe5787f..a9cc611c3 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/QuantilesTest.java @@ -21,7 +21,7 @@ /** * Tests the quantiles reducer of the OSHDB API. */ -class TestQuantiles { +class QuantilesTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -31,7 +31,7 @@ class TestQuantiles { private static final double REQUIRED_ACCURACY = 1E-4; - TestQuantiles() throws Exception { + QuantilesTest() throws Exception { oshdb = new OSHDBH2("../data/test-data"); } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/StreamTest.java similarity index 97% rename from oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java rename to oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/StreamTest.java index 0cc29d9df..ce6d19c3b 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/StreamTest.java @@ -15,7 +15,7 @@ /** * Tests the stream method of the OSHDB API. */ -class TestStream { +class StreamTest { private final OSHDBDatabase oshdb; private final OSHDBBoundingBox bbox = @@ -23,7 +23,7 @@ class TestStream { private final OSHDBTimestamps timestamps72 = new OSHDBTimestamps("2010-01-01", "2015-12-01", OSHDBTimestamps.Interval.MONTHLY); - TestStream() throws Exception { + StreamTest() throws Exception { oshdb = new OSHDBH2("../data/test-data").multithreading(false); } From 11d2cb4b74e263b45dff364960634ab8d9324505 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:43:23 +0100 Subject: [PATCH 16/34] #470: rename all test names to postfix-convention --- ...ityCall.java => MapReduceOSHDBIgniteAffinityCallTest.java} | 4 ++-- ...eLocalPeek.java => MapReduceOSHDBIgniteLocalPeekTest.java} | 4 ++-- ...ngCache.java => MapReduceOSHDBIgniteMissingCacheTest.java} | 4 ++-- ...eScanQuery.java => MapReduceOSHDBIgniteScanQueryTest.java} | 4 ++-- ...duceTestOSHDBIgnite.java => MapReduceOSHDBIgniteTest.java} | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{MapReduceTestOSHDBIgniteAffinityCall.java => MapReduceOSHDBIgniteAffinityCallTest.java} (89%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{MapReduceTestOSHDBIgniteLocalPeek.java => MapReduceOSHDBIgniteLocalPeekTest.java} (75%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{MapReduceTestOSHDBIgniteMissingCache.java => MapReduceOSHDBIgniteMissingCacheTest.java} (91%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{MapReduceTestOSHDBIgniteScanQuery.java => MapReduceOSHDBIgniteScanQueryTest.java} (83%) rename oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/{MapReduceTestOSHDBIgnite.java => MapReduceOSHDBIgniteTest.java} (96%) diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java similarity index 89% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java index cf36fe35f..2b6861859 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteAffinityCall.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java @@ -13,13 +13,13 @@ * *

Runs the tests using the "affinity call" Ignite backend.

*/ -class MapReduceTestOSHDBIgniteAffinityCall extends MapReduceTestOSHDBIgnite { +class MapReduceOSHDBIgniteAffinityCallTest extends MapReduceOSHDBIgniteTest { /** * Creates the test runner using the ignite affinitycall backend. * * @throws Exception if something goes wrong */ - MapReduceTestOSHDBIgniteAffinityCall() throws Exception { + MapReduceOSHDBIgniteAffinityCallTest() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL)); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java similarity index 75% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java index c3bc778ba..2038a7ac0 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteLocalPeek.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java @@ -7,13 +7,13 @@ * *

Runs the tests using the "local peek" Ignite backend.

*/ -class MapReduceTestOSHDBIgniteLocalPeek extends MapReduceTestOSHDBIgnite { +class MapReduceOSHDBIgniteLocalPeekTest extends MapReduceOSHDBIgniteTest { /** * Creates the test runner using the ignite localpeak backend. * * @throws Exception if something goes wrong */ - MapReduceTestOSHDBIgniteLocalPeek() throws Exception { + MapReduceOSHDBIgniteLocalPeekTest() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java similarity index 91% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java index ebfb4e990..b3c0c86ad 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteMissingCache.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java @@ -8,13 +8,13 @@ /** * Tests for proper error messages is caches are not pnt on ignite. */ -class MapReduceTestOSHDBIgniteMissingCache extends MapReduceTestOSHDBIgnite { +class MapReduceOSHDBIgniteMissingCacheTest extends MapReduceOSHDBIgniteTest { /** * Creates the test runner using an Ignite backend. * * @throws Exception if something goes wrong */ - MapReduceTestOSHDBIgniteMissingCache() throws Exception { + MapReduceOSHDBIgniteMissingCacheTest() throws Exception { super(oshdb -> {}); this.oshdb.prefix(""); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java similarity index 83% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java index 173a869d0..73682f134 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgniteScanQuery.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java @@ -10,13 +10,13 @@ * *

Runs the tests using the "scan query" Ignite backend.

*/ -class MapReduceTestOSHDBIgniteScanQuery extends MapReduceTestOSHDBIgnite { +class MapReduceOSHDBIgniteScanQueryTest extends MapReduceOSHDBIgniteTest { /** * Creates the test runner using the ignite scanquery backend. * * @throws Exception if something goes wrong */ - MapReduceTestOSHDBIgniteScanQuery() throws Exception { + MapReduceOSHDBIgniteScanQueryTest() throws Exception { super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY)); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java similarity index 96% rename from oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java rename to oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java index fcb827a52..f727366dd 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceTestOSHDBIgnite.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java @@ -29,7 +29,7 @@ import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.TableNames; -abstract class MapReduceTestOSHDBIgnite extends MapReduceTest { +abstract class MapReduceOSHDBIgniteTest extends MapReduceTest { static final Ignite ignite; static { @@ -98,7 +98,7 @@ private static OSHDBDatabase initOshdb(Consumer computeMode) { return oshdb; } - MapReduceTestOSHDBIgnite(Consumer computeMode) throws Exception { + MapReduceOSHDBIgniteTest(Consumer computeMode) throws Exception { super(initOshdb(computeMode)); } } From 382d3166edbac0b2ba4172721042941ca3a5835f Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:53:23 +0100 Subject: [PATCH 17/34] #470: fix misleading method name --- .../util/geometry/helpers/OSMXmlReaderTagInterpreter.java | 4 ++-- .../ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java | 4 ++-- .../src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java | 4 ++-- .../test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java | 8 ++++---- .../java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java | 4 ++-- .../java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java | 4 ++-- .../test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java index ff20e5279..b4e692ba7 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java @@ -38,10 +38,10 @@ public boolean isArea(OSMEntity e) { if (e instanceof OSMWay) { OSMMember[] nds = ((OSMWay) e).getMembers(); return nds.length >= 4 && nds[0].getId() == nds[nds.length - 1].getId() - && e.getTags().hasTagValue(area, areaYes); + && e.getTags().hasTag(area, areaYes); } if (e instanceof OSMRelation) { - return e.getTags().hasTagValue(type, typeMultipolygon); + return e.getTags().hasTag(type, typeMultipolygon); } return true; } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java index 98cbd4b21..38eef6b15 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java @@ -58,7 +58,7 @@ void testGetModificationTimestampsNodeWithFilter() throws IOException { )); List tss = - OSHEntityTimeUtils.getModificationTimestamps(hnode, e -> e.getTags().hasTagValue(1, 1)); + OSHEntityTimeUtils.getModificationTimestamps(hnode, e -> e.getTags().hasTag(1, 1)); assertNotNull(tss); assertEquals(2, tss.size()); @@ -176,7 +176,7 @@ void testGetModificationTimestampsWayWithFilter() throws IOException { tss = OSHEntityTimeUtils.getModificationTimestamps( hway, - osmEntity -> osmEntity.getTags().hasTagValue(2, 1) + osmEntity -> osmEntity.getTags().hasTag(2, 1) ); assertNotNull(tss); assertEquals(5, tss.size()); diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java index 62eec5908..d644b91ef 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java @@ -46,7 +46,7 @@ public boolean hasTagKey(OSHDBTagKey key) { * @param value the value to search for. * @return true, if key/value combination is present. */ - public abstract boolean hasTagValue(int key, int value); + public abstract boolean hasTag(int key, int value); /** * Test for a certain key/value combination. @@ -136,7 +136,7 @@ public boolean hasTagKeyExcluding(int key, int[] uninterestingValues) { } @Override - public boolean hasTagValue(int key, int value) { + public boolean hasTag(int key, int value) { return hasTag(new OSHDBTag(key, value)); } diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java index 1e6de22da..329495267 100644 --- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java +++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java @@ -45,11 +45,11 @@ void testArrayHasTagKeyExcluding() { void testArrayHasTagValue() { var tags = OSHDBTags.of(kvs); - assertTrue(tags.hasTagValue(1, 2)); + assertTrue(tags.hasTag(1, 2)); - assertFalse(tags.hasTagValue(2, 2)); - assertFalse(tags.hasTagValue(3, 4)); - assertFalse(tags.hasTagValue(5, 6)); + assertFalse(tags.hasTag(2, 2)); + assertFalse(tags.hasTag(3, 4)); + assertFalse(tags.hasTag(5, 6)); } @Test() diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java index 58ea80540..b6850bf3d 100644 --- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java +++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java @@ -172,12 +172,12 @@ void testHasTagValue() { OSMNode instance = OSM.node(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, 1000000000, 1000000000); boolean expResult = false; - boolean result = instance.getTags().hasTagValue(1, 1); + boolean result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); instance = OSM.node(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, 1000000000, 1000000000); expResult = true; - result = instance.getTags().hasTagValue(1, 1); + result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); } diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java index 0ab3db4fd..a4d9a2d60 100644 --- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java +++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java @@ -142,7 +142,7 @@ void testHasTagValue() { OSMRelation instance = OSM.relation(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, new OSMMember[] {part, part}); boolean expResult = false; - boolean result = instance.getTags().hasTagValue(1, 1); + boolean result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); } @@ -152,7 +152,7 @@ void testHasTagValue2() { OSMRelation instance = OSM.relation(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, new OSMMember[] {part, part}); boolean expResult = true; - boolean result = instance.getTags().hasTagValue(1, 1); + boolean result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); } } diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java index 18a24fbf5..528fc4233 100644 --- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java +++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java @@ -141,13 +141,13 @@ void testHasTagValue() { OSMWay instance = OSM.way(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, new OSMMember[] {part, part}); boolean expResult = false; - boolean result = instance.getTags().hasTagValue(1, 1); + boolean result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); part = new OSMMember(1L, OSMType.NODE, 1); instance = OSM.way(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, new OSMMember[] {part, part}); expResult = true; - result = instance.getTags().hasTagValue(1, 1); + result = instance.getTags().hasTag(1, 1); assertEquals(expResult, result); } From 390ecff722857342b8e2768b0de63f69e1d4792d Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Thu, 3 Nov 2022 14:58:51 +0100 Subject: [PATCH 18/34] #470: improve method implementation --- oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java index d644b91ef..5c53e25ba 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java @@ -143,8 +143,8 @@ public boolean hasTag(int key, int value) { @Override public boolean hasTag(OSHDBTag tag) { for (int i = 0; i < tags.length; i += 2) { - if (tags[i] == tag.getKey() && tags[i + 1] == tag.getValue()) { - return true; + if (tags[i] == tag.getKey()) { + return tags[i + 1] == tag.getValue(); } } return false; From 2265db827eda0338c33077f5ffacd03709534e08 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Fri, 4 Nov 2022 11:26:16 +0100 Subject: [PATCH 19/34] #470: remove obsolete imports --- .../heigit/ohsome/oshdb/api/db/OSHDBH2.java | 20 ++----------------- .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 2 -- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java index 14f626e58..630e4beb0 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java @@ -1,26 +1,10 @@ package org.heigit.ohsome.oshdb.api.db; +import static java.util.Optional.ofNullable; + import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.sql.DataSource; import org.h2.jdbcx.JdbcConnectionPool; -import org.h2.jdbcx.JdbcDataSource; -import org.heigit.ohsome.oshdb.OSHDB; -import org.heigit.ohsome.oshdb.util.TableNames; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; - -import static java.util.Optional.ofNullable; /** * OSHDB database backend connector to a H2 database. diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 6bf7502e0..e040660cb 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -7,10 +7,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.Optional; -import java.util.Set; import java.util.stream.Stream; import javax.sql.DataSource; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; From 66cc4488d30a9f3f69e5de9a803884d32b94703f Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Fri, 4 Nov 2022 12:00:37 +0100 Subject: [PATCH 20/34] #470: create tag-translator eagerly --- .../main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index e040660cb..640985f43 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -30,7 +30,7 @@ public class OSHDBJdbc extends OSHDBDatabase { protected final DataSource dataSource; protected final DataSource keytablesSource; - protected JdbcTagTranslator tagTranslator; + protected final TagTranslator tagTranslator; private boolean useMultithreading = true; public OSHDBJdbc(DataSource source) { @@ -40,13 +40,11 @@ public OSHDBJdbc(DataSource source) { public OSHDBJdbc(DataSource source, DataSource keytables) { this.dataSource = source; this.keytablesSource = keytables; + this.tagTranslator = new JdbcTagTranslator(keytablesSource); } @Override public TagTranslator getTagTranslator() { - if (tagTranslator == null) { - tagTranslator = new JdbcTagTranslator(keytablesSource); - } return tagTranslator; } From ae31cedbeae376a80996a2df9d3dfcda7d8df27b Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Fri, 4 Nov 2022 14:09:50 +0100 Subject: [PATCH 21/34] #470: cleanup: introduce static imports, remove redundant throws-clauses --- .../tests/MapReduceOSHDBIgniteAffinityCallTest.java | 4 ++-- .../api/tests/MapReduceOSHDBIgniteLocalPeekTest.java | 4 ++-- .../tests/MapReduceOSHDBIgniteMissingCacheTest.java | 10 +++++----- .../api/tests/MapReduceOSHDBIgniteScanQueryTest.java | 6 +++--- .../oshdb/api/tests/MapReduceOSHDBIgniteTest.java | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java index 2b6861859..37dda5b47 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteAffinityCallTest.java @@ -1,10 +1,10 @@ package org.heigit.ohsome.oshdb.api.tests; +import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.AFFINITY_CALL; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Set; import java.util.stream.Collectors; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.jupiter.api.Test; @@ -20,7 +20,7 @@ class MapReduceOSHDBIgniteAffinityCallTest extends MapReduceOSHDBIgniteTest { * @throws Exception if something goes wrong */ MapReduceOSHDBIgniteAffinityCallTest() throws Exception { - super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL)); + super(oshdb -> oshdb.computeMode(AFFINITY_CALL)); } @Test diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java index 2038a7ac0..860e62497 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteLocalPeekTest.java @@ -1,6 +1,6 @@ package org.heigit.ohsome.oshdb.api.tests; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; +import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.LOCAL_PEEK; /** * {@inheritDoc} @@ -14,6 +14,6 @@ class MapReduceOSHDBIgniteLocalPeekTest extends MapReduceOSHDBIgniteTest { * @throws Exception if something goes wrong */ MapReduceOSHDBIgniteLocalPeekTest() throws Exception { - super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK)); + super(oshdb -> oshdb.computeMode(LOCAL_PEEK)); } } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java index b3c0c86ad..ba5327e5f 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteMissingCacheTest.java @@ -21,31 +21,31 @@ class MapReduceOSHDBIgniteMissingCacheTest extends MapReduceOSHDBIgniteTest { @Override @Test() - void testOSMContributionView() throws Exception { + void testOSMContributionView() { assertThrows(OSHDBTableNotFoundException.class, super::testOSMContributionView); } @Override @Test() - void testOSMEntitySnapshotView() throws Exception { + void testOSMEntitySnapshotView() { assertThrows(OSHDBTableNotFoundException.class, super::testOSMEntitySnapshotView); } @Override @Test() - void testOSMContributionViewStream() throws Exception { + void testOSMContributionViewStream() { assertThrows(OSHDBTableNotFoundException.class, super::testOSMContributionViewStream); } @Override @Test() - void testOSMEntitySnapshotViewStream() throws Exception { + void testOSMEntitySnapshotViewStream() { assertThrows(OSHDBTableNotFoundException.class, super::testOSMEntitySnapshotViewStream); } @Override @Test() - void testTimeoutMapReduce() throws Exception { + void testTimeoutMapReduce() { assertThrows(OSHDBTableNotFoundException.class, this::timeoutMapReduce); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java index 73682f134..dcae4d3b6 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteScanQueryTest.java @@ -1,8 +1,8 @@ package org.heigit.ohsome.oshdb.api.tests; +import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.SCAN_QUERY; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.junit.jupiter.api.Test; /** @@ -17,12 +17,12 @@ class MapReduceOSHDBIgniteScanQueryTest extends MapReduceOSHDBIgniteTest { * @throws Exception if something goes wrong */ MapReduceOSHDBIgniteScanQueryTest() throws Exception { - super(oshdb -> oshdb.computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY)); + super(oshdb -> oshdb.computeMode(SCAN_QUERY)); } @Override @Test - void testTimeoutStream() throws Exception { + void testTimeoutStream() { // ignore this test -> scanquery backend currently doesn't support timeouts for stream() assertTrue(true); } diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java index f727366dd..9c094d9ee 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java @@ -72,7 +72,7 @@ private static OSHDBDatabase initOshdb(Consumer computeMode) { streamer.allowOverwrite(true); try (final ResultSet rst = - h2Stmt.executeQuery("select level, id, data from " + TableNames.T_NODES.toString())) { + h2Stmt.executeQuery("select level, id, data from " + TableNames.T_NODES)) { while (rst.next()) { final int level = rst.getInt(1); final long id = rst.getLong(2); From e61994f7032b439c6aaebd904ec13584e201f37c Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Fri, 4 Nov 2022 14:13:56 +0100 Subject: [PATCH 22/34] #470: do not store keytables datasource anymore --- .../main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 640985f43..232b1955c 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -29,7 +29,6 @@ public class OSHDBJdbc extends OSHDBDatabase { protected final DataSource dataSource; - protected final DataSource keytablesSource; protected final TagTranslator tagTranslator; private boolean useMultithreading = true; @@ -39,8 +38,7 @@ public OSHDBJdbc(DataSource source) { public OSHDBJdbc(DataSource source, DataSource keytables) { this.dataSource = source; - this.keytablesSource = keytables; - this.tagTranslator = new JdbcTagTranslator(keytablesSource); + this.tagTranslator = new JdbcTagTranslator(keytables); } @Override From b4a127608fa66d3180ea5f36e54dd2424af71242 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Fri, 4 Nov 2022 14:34:35 +0100 Subject: [PATCH 23/34] #470: use datasource instead of OSHDBDatabase --- .../ohsome/oshdb/api/db/OSHDBIgnite.java | 19 +++++++++++-------- .../api/tests/MapReduceOSHDBIgniteTest.java | 15 ++++++++++----- .../heigit/ohsome/oshdb/api/db/H2Support.java | 12 ++++++++++++ .../heigit/ohsome/oshdb/api/db/OSHDBH2.java | 7 +------ 4 files changed, 34 insertions(+), 19 deletions(-) create mode 100644 oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java diff --git a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java index 735260cb2..b77182f87 100644 --- a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java +++ b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java @@ -6,6 +6,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.lang.IgniteRunnable; @@ -18,6 +19,7 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; /** @@ -41,7 +43,7 @@ public enum ComputeMode { } private final Ignite ignite; - private final OSHDBDatabase keytables; + private final TagTranslator tagTranslator; private final boolean owner; private ComputeMode computeMode = ComputeMode.LOCAL_PEEK; @@ -52,7 +54,7 @@ public enum ComputeMode { * * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(OSHDBDatabase keytables) { + public OSHDBIgnite(DataSource keytables) { this(new File("ignite-config.xml"), keytables); } @@ -62,7 +64,7 @@ public OSHDBIgnite(OSHDBDatabase keytables) { * @param ignite Ignite instance to use. * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(Ignite ignite, OSHDBDatabase keytables) { + public OSHDBIgnite(Ignite ignite, DataSource keytables) { this(ignite, false, keytables); } @@ -72,7 +74,7 @@ public OSHDBIgnite(Ignite ignite, OSHDBDatabase keytables) { * @param igniteConfigFilePath ignite configuration file * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(String igniteConfigFilePath, OSHDBDatabase keytables) { + public OSHDBIgnite(String igniteConfigFilePath, DataSource keytables) { this(new File(igniteConfigFilePath), keytables); } @@ -82,14 +84,15 @@ public OSHDBIgnite(String igniteConfigFilePath, OSHDBDatabase keytables) { * @param igniteConfig ignite configuration file * @throws OSHDBException if cluster state is not active. */ - public OSHDBIgnite(File igniteConfig, OSHDBDatabase keytables) { + public OSHDBIgnite(File igniteConfig, DataSource keytables) { this(startClient(igniteConfig), true, keytables); } - public OSHDBIgnite(Ignite ignite, boolean owner, OSHDBDatabase keytables) { + public OSHDBIgnite(Ignite ignite, boolean owner, DataSource keytables) { this.ignite = ignite; this.owner = owner; - this.keytables = keytables; + this.tagTranslator = new JdbcTagTranslator(keytables); + checkStateActive(); } @@ -107,7 +110,7 @@ private void checkStateActive() { @Override public TagTranslator getTagTranslator() { - return keytables.getTagTranslator(); + return tagTranslator; } @Override diff --git a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java index 9c094d9ee..3fe1449f4 100644 --- a/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java +++ b/oshdb-api-ignite/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapReduceOSHDBIgniteTest.java @@ -1,9 +1,11 @@ package org.heigit.ohsome.oshdb.api.tests; +import static org.heigit.ohsome.oshdb.api.db.H2Support.pathToUrl; import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.ObjectInputStream; +import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -22,8 +24,8 @@ import org.apache.ignite.logger.slf4j.Slf4jLogger; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; -import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.grid.GridOSHNodes; import org.heigit.ohsome.oshdb.util.CellId; @@ -63,11 +65,14 @@ private static OSHDBDatabase initOshdb(Consumer computeMode) { ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_WAYS.toString(prefix))); ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_RELATIONS.toString(prefix))); - OSHDBH2 oshdbH2 = new OSHDBH2("../data/test-data"); + JdbcConnectionPool oshdbH2 = JdbcConnectionPool.create(pathToUrl(Path.of("../data/test-data")), "sa", + ""); + // load test data into ignite cache - try (IgniteDataStreamer streamer = ignite.dataStreamer(cache.getName())) { - Connection h2Conn = oshdbH2.getConnection(); - Statement h2Stmt = h2Conn.createStatement(); + try (IgniteDataStreamer streamer = ignite.dataStreamer(cache.getName()); + Connection h2Conn = oshdbH2.getConnection(); + Statement h2Stmt = h2Conn.createStatement(); + ) { streamer.allowOverwrite(true); diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java new file mode 100644 index 000000000..2593123f6 --- /dev/null +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java @@ -0,0 +1,12 @@ +package org.heigit.ohsome.oshdb.api.db; + +import java.nio.file.Path; + +public class H2Support { + + public static String pathToUrl(Path path) { + var absolutePath = path.toAbsolutePath().toString(); + absolutePath = absolutePath.replaceAll("\\.mv\\.db$", ""); + return String.format("jdbc:h2:%s;ACCESS_MODE_DATA=r", absolutePath); + } +} diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java index 630e4beb0..71fe0224b 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBH2.java @@ -1,6 +1,7 @@ package org.heigit.ohsome.oshdb.api.db; import static java.util.Optional.ofNullable; +import static org.heigit.ohsome.oshdb.api.db.H2Support.pathToUrl; import java.nio.file.Path; import javax.sql.DataSource; @@ -45,12 +46,6 @@ private OSHDBH2(JdbcConnectionPool ds) { this.connectionPool = ds; } - private static String pathToUrl(Path path) { - var absolutePath = path.toAbsolutePath().toString(); - absolutePath = absolutePath.replaceAll("\\.mv\\.db$", ""); - return String.format("jdbc:h2:%s;ACCESS_MODE_DATA=r", absolutePath); - } - @Override public OSHDBH2 prefix(String prefix) { return (OSHDBH2) super.prefix(prefix); From c710f8d75862111f916d45ed2b1914b3ab860e84 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 10:42:53 +0100 Subject: [PATCH 24/34] #470: simplify nullability handling with Optional --- .../java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java index b77182f87..19c894ba1 100644 --- a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java +++ b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java @@ -207,10 +207,7 @@ public OSHDBIgnite onClose(IgniteRunnable action) { * @return the currently set onClose callback */ public Optional onClose() { - if (this.onCloseCallback == null) { - return Optional.empty(); - } else { - return Optional.of(this.onCloseCallback); - } + return Optional.ofNullable(this.onCloseCallback); } + } From 5f8d2d4fe33773bd02eaca7becd65e5c6011bbc0 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 11:23:31 +0100 Subject: [PATCH 25/34] #470: specify pooling requirements for constructor argument Datasource --- .../ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index bbe6eeb1f..27263fdf1 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -48,6 +48,12 @@ public class JdbcTagTranslator implements TagTranslator { private final DataSource source; private final Cache cacheKeys; + /** + * Attention: + * This tag translator relies on a pooled datasource for thread-safety. + * + * @param source the (pooled) datasource + */ public JdbcTagTranslator(DataSource source) { this.source = source; cacheKeys = Caffeine.newBuilder() From 4ba9380f5343d630672779647eb30b1588eb18e6 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 11:29:59 +0100 Subject: [PATCH 26/34] #470: cleanup naming and imports --- .../util/celliterator/IterateByContributionTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java index 09e2dd28d..4b9673aad 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.ObjectInputStream; -import java.sql.SQLException; import java.util.List; import java.util.TreeSet; import java.util.stream.Collectors; @@ -15,7 +14,6 @@ import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -30,7 +28,7 @@ class IterateByContributionTest { * Set up of test framework, loading H2 driver and connection via jdbc. */ @BeforeAll - static void setUpClass() throws ClassNotFoundException, SQLException { + static void setUpClass() { // connect to the "Big"DB source = JdbcConnectionPool.create( "jdbc:h2:../data/test-data;ACCESS_MODE_DATA=r", @@ -53,7 +51,7 @@ void testIssue108() throws Exception { int countCreated = 0; int countOther = 0; - var tt = new JdbcTagTranslator(source); + var tagTranslator = new JdbcTagTranslator(source); try (var conn = source.getConnection(); var stmt = conn.createStatement(); var oshCellsRawData = stmt.executeQuery("select data from " + TableNames.T_NODES)) { @@ -70,7 +68,7 @@ void testIssue108() throws Exception { List result = (new CellIterator( timestamps, OSHDBBoundingBox.bboxWgs84Coordinates(8.0, 9.0, 49.0, 50.0), - new DefaultTagInterpreter(tt), + new DefaultTagInterpreter(tagTranslator), oshEntity -> oshEntity.getId() == 617308093, osmEntity -> true, false From f275a7e1d5b7bfaea92eaa34a3a0a9799f9ee2cf Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 11:53:18 +0100 Subject: [PATCH 27/34] #470: remove redundant join and cleanup whitespace in sql --- .../oshdb/util/tagtranslator/JdbcTagTranslator.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 27263fdf1..46d71bab1 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -29,20 +29,19 @@ public class JdbcTagTranslator implements TagTranslator { private static final String OSHDB_OSM_KEY = String.format("SELECT txt, id" - + " from %s " + + " from %s" + " where id = any(?)", TableNames.E_KEY); - private static final String OSHDB_OSM_TAG = String.format("SELECT kv.txt, valueid" - + " from %s k" - + " left join %s kv on k.id = kv.keyid" - + " where k.id = ? and kv.valueid = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); + private static final String OSHDB_OSM_TAG = String.format("SELECT txt, valueid" + + " from %s" + + " where keyid = ? and valueid = any (?)", TableNames.E_KEYVALUE); private static final String OSM_OSHDB_ROLE = String.format("SELECT id, txt" - + " from %s " + + " from %s" + " where txt = any (?)", TableNames.E_ROLE); private static final String OSHDB_OSM_ROLE = String.format("SELECT txt, id" - + " from %s " + + " from %s" + " where id = any (?)", TableNames.E_ROLE); private final DataSource source; From 5c18dac512d469523e5cfe621c94c947185434bb Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 11:55:42 +0100 Subject: [PATCH 28/34] #470: cleanup formatting --- .../ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 46d71bab1..e856bf5a9 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -18,6 +18,7 @@ import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; public class JdbcTagTranslator implements TagTranslator { + private static final String OSM_OSHDB_KEY = String.format("SELECT id, txt" + " from %s k" + " where k.txt = ?", TableNames.E_KEY); @@ -27,7 +28,6 @@ public class JdbcTagTranslator implements TagTranslator { + " left join %s kv on k.id = kv.keyid" + " where k.txt = ? and kv.txt = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); - private static final String OSHDB_OSM_KEY = String.format("SELECT txt, id" + " from %s" + " where id = any(?)", TableNames.E_KEY); From ff2d1c6c923179e26ea9d6ee53c77236f60c630c Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 12:06:30 +0100 Subject: [PATCH 29/34] #470: extract abstract superclass for tag translator tests --- ...st.java => AbstractTagTranslatorTest.java} | 32 ++++++++----------- .../tagtranslator/JdbcTagTranslatorTest.java | 13 ++++++++ 2 files changed, 27 insertions(+), 18 deletions(-) rename oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/{TagTranslatorTest.java => AbstractTagTranslatorTest.java} (84%) create mode 100644 oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java similarity index 84% rename from oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java rename to oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java index ea207f83b..3735204e7 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java @@ -15,15 +15,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -/** - * Tests the {@link TagTranslator} class. - */ -class TagTranslatorTest { - private static JdbcConnectionPool source; +public abstract class AbstractTagTranslatorTest { + + protected static JdbcConnectionPool source; /** * Initialize tests by loading the H2 driver and open a connection via jdbc. - * */ @BeforeAll static void setUpClass() { @@ -35,12 +32,12 @@ static void breakDownClass() { source.dispose(); } - TagTranslatorTest() {} + abstract TagTranslator getTranslator(); @Test void testTag2Int() throws OSHDBKeytablesNotFoundException { OSMTag tag = new OSMTag("building", "yes"); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); OSHDBTag expResult = new OSHDBTag(1, 0); OSHDBTag result = instance.getOSHDBTagOf(tag).get(); assertEquals(expResult, result); @@ -53,7 +50,7 @@ void testTags2Int() throws OSHDBKeytablesNotFoundException { new OSMTag("building", "residential"), new OSHDBTag(1, 2), new OSMTag("highway", "primary"), new OSHDBTag(2, 7)); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); var result = instance.getOSHDBTagOf(tags.keySet()); assertEquals(tags.size(), result.size()); for (var entry : tags.entrySet()) { @@ -62,12 +59,11 @@ void testTags2Int() throws OSHDBKeytablesNotFoundException { } } - @Test void testTag2String() throws OSHDBKeytablesNotFoundException { OSHDBTag tag = new OSHDBTag(1, 2); OSMTag expResult = new OSMTag("building", "residential"); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); OSMTag result = instance.lookupTag(tag); assertEquals(expResult, result); } @@ -78,7 +74,7 @@ void testTags2String() { new OSHDBTag(1, 0), new OSMTag("building", "yes"), new OSHDBTag(1, 2), new OSMTag("building", "residential"), new OSHDBTag(2, 7), new OSMTag("highway", "primary")); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); var result = instance.lookupTag(tags.keySet()); assertEquals(tags.size(), result.size()); for (var entry : tags.entrySet()) { @@ -89,8 +85,8 @@ void testTags2String() { @Test void testOSMEntityTag2String() { - var osm = OSM.node(123, 1, 1000L, 100L, 1, new int[] {1, 0, 2, 7}, 0, 0); - var instance = new JdbcTagTranslator(source); + var osm = OSM.node(123, 1, 1000L, 100L, 1, new int[]{1, 0, 2, 7}, 0, 0); + TagTranslator instance = getTranslator(); var tags = instance.lookupTag(osm.getTags()); assertEquals(2, tags.size()); assertEquals(new OSMTag("building", "yes"), tags.get(new OSHDBTag(1, 0))); @@ -100,7 +96,7 @@ void testOSMEntityTag2String() { @Test void testKey2Int() throws OSHDBKeytablesNotFoundException { OSMTagKey key = new OSMTagKey("highway"); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); OSHDBTagKey expResult = new OSHDBTagKey(2); OSHDBTagKey result = instance.getOSHDBTagKeyOf(key).get(); assertEquals(expResult, result); @@ -109,7 +105,7 @@ void testKey2Int() throws OSHDBKeytablesNotFoundException { @Test void testRole2Int() throws OSHDBKeytablesNotFoundException { OSMRole role = new OSMRole("from"); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); OSHDBRole expResult = OSHDBRole.of(4); OSHDBRole result = instance.getOSHDBRoleOf(role).get(); assertEquals(expResult, result); @@ -118,7 +114,7 @@ void testRole2Int() throws OSHDBKeytablesNotFoundException { @Test void testRole2String() throws OSHDBKeytablesNotFoundException { OSHDBRole role = OSHDBRole.of(1); - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); OSMRole expResult = new OSMRole("inner"); OSMRole result = instance.lookupRole(role); assertEquals(expResult, result); @@ -126,7 +122,7 @@ void testRole2String() throws OSHDBKeytablesNotFoundException { @Test void testKeysIdentity() { - var instance = new JdbcTagTranslator(source); + TagTranslator instance = getTranslator(); var tags = new ArrayList(10); for (var i = 0; i < 10; i++) { tags.add(instance.lookupTag(new OSHDBTag(1, i))); diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java new file mode 100644 index 000000000..d092d9e66 --- /dev/null +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java @@ -0,0 +1,13 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +/** + * Tests the {@link JdbcTagTranslator} class. + */ +class JdbcTagTranslatorTest extends AbstractTagTranslatorTest { + + @Override + TagTranslator getTranslator() { + return new JdbcTagTranslator(source); + } + +} From fe056117863d3a2e955394ea9f3dd1f225dafebf Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 12:09:51 +0100 Subject: [PATCH 30/34] #470: add test subclass to validate cached tag translator --- .../util/tagtranslator/CachedTagTranslatorTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslatorTest.java diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslatorTest.java new file mode 100644 index 000000000..2fce06dc5 --- /dev/null +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslatorTest.java @@ -0,0 +1,13 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +/** + * Tests the {@link CachedTagTranslator} class. + */ +class CachedTagTranslatorTest extends AbstractTagTranslatorTest { + + @Override + TagTranslator getTranslator() { + return new CachedTagTranslator(new JdbcTagTranslator(source), 1000, 1000); + } + +} From 95aba03804d1fa8e6bf8abdc2e7018b16d365645 Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 12:17:53 +0100 Subject: [PATCH 31/34] #470: fix various warnings --- .../heigit/ohsome/oshdb/api/db/H2Support.java | 3 +++ .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 1 + .../tests/HelpersOSMContributionViewTest.java | 17 +++++++++-------- .../api/tests/MapAggregateByGeometryTest.java | 14 +++++++------- .../util/tagtranslator/JdbcTagTranslator.java | 6 +++--- .../AbstractTagTranslatorTest.java | 4 ++-- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java index 2593123f6..7117632dd 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java @@ -4,6 +4,9 @@ public class H2Support { + private H2Support() { + } + public static String pathToUrl(Path path) { var absolutePath = path.toAbsolutePath().toString(); absolutePath = absolutePath.replaceAll("\\.mv\\.db$", ""); diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index 232b1955c..4092a0c29 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -135,5 +135,6 @@ public boolean multithreading() { @Override public void close() throws Exception { + //nothing to do } } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java index 328da748d..a5b004c67 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/HelpersOSMContributionViewTest.java @@ -1,6 +1,7 @@ package org.heigit.ohsome.oshdb.api.tests; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.util.EnumSet; import java.util.HashSet; @@ -32,11 +33,11 @@ class HelpersOSMContributionViewTest { private static final double DELTA = 1e-8; - HelpersOSMContributionViewTest() throws Exception { + HelpersOSMContributionViewTest() { oshdb = new OSHDBH2("../data/test-data"); } - private MapReducer createMapReducer() throws Exception { + private MapReducer createMapReducer() { return OSMContributionView .on(oshdb) .areaOfInterest(bbox) @@ -85,7 +86,7 @@ void testSum() throws Exception { .sum(contribution -> 1); assertEquals(42, result4.get(EnumSet.of(ContributionType.CREATION).toString())); - assertEquals(null, result4.get(EnumSet.of(ContributionType.DELETION).toString())); + assertNull(result4.get(EnumSet.of(ContributionType.DELETION).toString())); } @Test @@ -135,7 +136,7 @@ void testAverage() throws Exception { contribution.getContributionTypes().contains(ContributionType.TAG_CHANGE) ? 1 : 0) .average(); - assertEquals(1.0, result1.doubleValue(), DELTA); + assertEquals(1.0, result1, DELTA); // many timestamps SortedMap result2 = this.createMapReducer() @@ -161,7 +162,7 @@ void testAverage() throws Exception { .contains(ContributionType.CREATION)) .average(contribution -> contribution.getEntityAfter().getId() % 2); - assertEquals(0.5, result4.get(true).doubleValue(), DELTA); + assertEquals(0.5, result4.get(true), DELTA); } @Test @@ -174,7 +175,7 @@ void testWeightedAverage() throws Exception { 2 * (contribution.getEntityAfter().getId() % 2) )); - assertEquals(1.0, result1.doubleValue(), DELTA); + assertEquals(1.0, result1, DELTA); // many timestamps SortedMap result2 = this.createMapReducer() @@ -203,7 +204,7 @@ void testWeightedAverage() throws Exception { 2 * (contribution.getEntityAfter().getId() % 2) )); - assertEquals(1.0, result4.get(true).doubleValue(), DELTA); + assertEquals(1.0, result4.get(true), DELTA); } @Test @@ -252,7 +253,7 @@ void testUniq() throws Exception { .timestamps(timestamps2) .map(x -> null) .uniq(); - assertEquals(result5.size(), 1); + assertEquals(1, result5.size() ); } } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java index 4d1a629b2..c89fdd7ed 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/MapAggregateByGeometryTest.java @@ -39,18 +39,18 @@ class MapAggregateByGeometryTest { private static final double DELTA = 1e-4; - MapAggregateByGeometryTest() throws Exception { + MapAggregateByGeometryTest() { oshdb = new OSHDBH2("../data/test-data"); } - private MapReducer createMapReducerOSMContribution() throws Exception { + private MapReducer createMapReducerOSMContribution() { return OSMContributionView .on(oshdb) .areaOfInterest(bbox) .filter("type:way and highway=*"); } - private MapReducer createMapReducerOSMEntitySnapshot() throws Exception { + private MapReducer createMapReducerOSMEntitySnapshot() { return OSMEntitySnapshotView .on(oshdb) .areaOfInterest(bbox) @@ -205,13 +205,13 @@ void testCombinedWithAggregateByTimestampOrder() throws Exception { @SuppressWarnings("ResultOfMethodCallIgnored") // we test for a thrown exception here @Test() - void testCombinedWithAggregateByTimestampUnsupportedOrder3() throws Exception { - assertThrows(UnsupportedOperationException.class, () -> { + void testCombinedWithAggregateByTimestampUnsupportedOrder3() { + assertThrows(UnsupportedOperationException.class, () -> createMapReducerOSMEntitySnapshot() .timestamps(timestamps1) .groupByEntity() .aggregateByGeometry(getSubRegions()) - .collect(); - }); + .collect() + ); } } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index e856bf5a9..201a3c165 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -29,8 +29,8 @@ public class JdbcTagTranslator implements TagTranslator { + " where k.txt = ? and kv.txt = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); private static final String OSHDB_OSM_KEY = String.format("SELECT txt, id" - + " from %s" - + " where id = any(?)", TableNames.E_KEY); + + " from %s" + + " where id = any(?)", TableNames.E_KEY); private static final String OSHDB_OSM_TAG = String.format("SELECT txt, valueid" + " from %s" @@ -166,7 +166,7 @@ public Map lookupTag(Set tags) { return result; } - private Map lookupKeys(Set osm) { + private Map lookupKeys(Set osm) { try (var conn = source.getConnection(); var sqlArray = createArray(conn, "int", osm); var pstmt = conn.prepareStatement(OSHDB_OSM_KEY)) { diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java index 3735204e7..452f4d19a 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/AbstractTagTranslatorTest.java @@ -1,7 +1,7 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertSame; import java.util.ArrayList; import java.util.Map; @@ -128,7 +128,7 @@ void testKeysIdentity() { tags.add(instance.lookupTag(new OSHDBTag(1, i))); } for (var i = 1; i < tags.size(); i++) { - assertTrue(tags.get(i - 1).getKey() == tags.get(i).getKey()); + assertSame(tags.get(i - 1).getKey(), tags.get(i).getKey()); } } } From a4e9b5b0f61be422c9299562af91319c6d76d6fe Mon Sep 17 00:00:00 2001 From: Matthias Merdes Date: Wed, 9 Nov 2022 12:45:22 +0100 Subject: [PATCH 32/34] #470: fixe resource leak and copy-constructor issues --- .../api/mapreducer/backend/MapReducerJdbc.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java index 011e4c3da..740e5e4eb 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Iterator; @@ -38,8 +39,9 @@ abstract class MapReducerJdbc extends MapReducer implements CancelableProc } // copy constructor - MapReducerJdbc(MapReducerJdbc obj) { - super(obj); + MapReducerJdbc(MapReducerJdbc source) { + super(source); + this.executionStartTimeMillis = source.executionStartTimeMillis; } @Override @@ -83,7 +85,7 @@ protected Stream getOshCellsStream(CellIdRange cellIdRange) { pstmt.setLong(3, cellIdRange.getEnd().getId()); var oshCellsRawData = pstmt.executeQuery(); return StreamSupport.stream(spliteratorUnknownSize( - new GridOSHEntityIterator(oshCellsRawData, conn), 0 + new GridOSHEntityIterator(oshCellsRawData, pstmt, conn), 0 ), false); } catch (SQLException e) { throw new OSHDBException(e); @@ -93,11 +95,13 @@ protected Stream getOshCellsStream(CellIdRange cellIdRange) { private class GridOSHEntityIterator implements Iterator { private final ResultSet oshCellsRawData; + private final PreparedStatement preparedStatement; private final Connection conn; GridOSHEntity next; - public GridOSHEntityIterator(ResultSet oshCellsRawData, Connection conn) { + public GridOSHEntityIterator(ResultSet oshCellsRawData, PreparedStatement pstmt, Connection conn) { this.oshCellsRawData = oshCellsRawData; + this.preparedStatement = pstmt; this.conn = conn; } @@ -121,6 +125,7 @@ private GridOSHEntity getNext() { if (!oshCellsRawData.next()) { try { oshCellsRawData.close(); + preparedStatement.close(); } finally { conn.close(); } From 4c19d20efc55590764059081d2d23d794405b058 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 9 Nov 2022 15:04:14 +0100 Subject: [PATCH 33/34] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e940453..77d8bad00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog * remove Comparable interface from OSMEntity ([#449]) * remove deprecated filter methods `osmType`, `osmTag` and `osmEntityFlter` (which were replaced by [OSHDB `filter`s](https://github.com/GIScience/oshdb/blob/0.7/documentation/manual/filters.md#oshdb-filter)) ([#451]) * `OSMType.toString` returns lower-case name. ([#459]) +* rework of handling separate keytables, see [#470] ### new features @@ -30,6 +31,7 @@ Changelog * update ignite dependency to version 2.14.0 ([#459], [#467]) * add natural order to `OSHDBTag` ([#454]) +[#470]: https://github.com/GIScience/oshdb/pull/470 [#419]: https://github.com/GIScience/oshdb/pull/419 [#433]: https://github.com/GIScience/oshdb/issues/433 [#438]: https://github.com/GIScience/oshdb/pull/438 From dae299615572dea58e0759c3d414b84901846a91 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 9 Nov 2022 15:07:14 +0100 Subject: [PATCH 34/34] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77d8bad00..c063a8395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Changelog * remove Comparable interface from OSMEntity ([#449]) * remove deprecated filter methods `osmType`, `osmTag` and `osmEntityFlter` (which were replaced by [OSHDB `filter`s](https://github.com/GIScience/oshdb/blob/0.7/documentation/manual/filters.md#oshdb-filter)) ([#451]) * `OSMType.toString` returns lower-case name. ([#459]) -* rework of handling separate keytables, see [#470] +* rework of handling separate keytables, see [#470] ([#470]) ### new features