Merge pull request #470 from GIScience/rework_tag-translator

Rework tag translator
GIScience · Nov 9, 2022 · 20bd2c8 · 20bd2c8
2 parents fd51513 + e6a6517
commit 20bd2c8
Show file tree

Hide file tree

Showing 66 changed files with 1,045 additions and 955 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ Changelog
 * remove Comparable interface from OSMEntity ([#449])
 * remove deprecated filter methods `osmType`, `osmTag` and `osmEntityFlter` (which were replaced by [OSHDB `filter`s](https://github.com/GIScience/oshdb/blob/0.7/documentation/manual/filters.md#oshdb-filter)) ([#451])
 * `OSMType.toString` returns lower-case name. ([#459])
+* rework of handling separate keytables, see [#470] ([#470])
 
 ### new features
 
@@ -48,7 +49,7 @@ Changelog
 [#454]: https://github.com/GIScience/oshdb/pull/454
 [#459]: https://github.com/GIScience/oshdb/pull/459
 [#467]: https://github.com/GIScience/oshdb/pull/467
-
+[#470]: https://github.com/GIScience/oshdb/pull/470
 
 
 ## 0.7.2

diff --git a/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java
@@ -6,6 +6,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.sql.DataSource;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.lang.IgniteRunnable;
@@ -18,6 +19,8 @@
 import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException;
 import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException;
 import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible;
+import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator;
+import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
 
 /**
  * OSHDB database backend connector to a Ignite system.
@@ -40,6 +43,7 @@ public enum ComputeMode {
   }
 
   private final Ignite ignite;
+  private final TagTranslator tagTranslator;
   private final boolean owner;
   private ComputeMode computeMode = ComputeMode.LOCAL_PEEK;
 
@@ -50,8 +54,8 @@ public enum ComputeMode {
    *
    * @throws OSHDBException if cluster state is not active.
    */
-  public OSHDBIgnite() {
-    this(new File("ignite-config.xml"));
+  public OSHDBIgnite(DataSource keytables) {
+    this(new File("ignite-config.xml"), keytables);
   }
 
   /**
@@ -60,8 +64,8 @@ public OSHDBIgnite() {
    * @param ignite Ignite instance to use.
    * @throws OSHDBException if cluster state is not active.
    */
-  public OSHDBIgnite(Ignite ignite) {
-    this(ignite, false);
+  public OSHDBIgnite(Ignite ignite, DataSource keytables) {
+    this(ignite, false, keytables);
   }
 
   /**
@@ -70,8 +74,8 @@ public OSHDBIgnite(Ignite ignite) {
    * @param igniteConfigFilePath ignite configuration file
    * @throws OSHDBException if cluster state is not active.
    */
-  public OSHDBIgnite(String igniteConfigFilePath) {
-    this(new File(igniteConfigFilePath));
+  public OSHDBIgnite(String igniteConfigFilePath, DataSource keytables) {
+    this(new File(igniteConfigFilePath), keytables);
   }
 
   /**
@@ -80,13 +84,15 @@ public OSHDBIgnite(String igniteConfigFilePath) {
    * @param igniteConfig ignite configuration file
    * @throws OSHDBException if cluster state is not active.
    */
-  public OSHDBIgnite(File igniteConfig) {
-    this(startClient(igniteConfig), true);
+  public OSHDBIgnite(File igniteConfig, DataSource keytables) {
+    this(startClient(igniteConfig), true, keytables);
   }
 
-  private OSHDBIgnite(Ignite ignite, boolean owner) {
+  public OSHDBIgnite(Ignite ignite, boolean owner, DataSource keytables) {
     this.ignite = ignite;
     this.owner = owner;
+    this.tagTranslator = new JdbcTagTranslator(keytables);
+
     checkStateActive();
   }
 
@@ -102,6 +108,11 @@ private void checkStateActive() {
     }
   }
 
+  @Override
+  public TagTranslator getTagTranslator() {
+    return tagTranslator;
+  }
+
   @Override
   public OSHDBIgnite prefix(String prefix) {
     return (OSHDBIgnite) super.prefix(prefix);
@@ -196,10 +207,7 @@ public OSHDBIgnite onClose(IgniteRunnable action) {
    * @return the currently set onClose callback
    */
   public Optional<IgniteRunnable> onClose() {
-    if (this.onCloseCallback == null) {
-      return Optional.empty();
-    } else {
-      return Optional.of(this.onCloseCallback);
-    }
+    return Optional.ofNullable(this.onCloseCallback);
   }
+
 }
diff --git a/...TestMapReduceOSHDBIgniteAffinityCall.java → ...MapReduceOSHDBIgniteAffinityCallTest.java b/...TestMapReduceOSHDBIgniteAffinityCall.java → ...MapReduceOSHDBIgniteAffinityCallTest.java
@@ -1,10 +1,10 @@
 package org.heigit.ohsome.oshdb.api.tests;
 
+import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.AFFINITY_CALL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.util.Set;
 import java.util.stream.Collectors;
-import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
 import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps;
 import org.junit.jupiter.api.Test;
 
@@ -13,14 +13,14 @@
  *
  * <p>Runs the tests using the "affinity call" Ignite backend.</p>
  */
-class TestMapReduceOSHDBIgniteAffinityCall extends TestMapReduceOSHDBIgnite {
+class MapReduceOSHDBIgniteAffinityCallTest extends MapReduceOSHDBIgniteTest {
   /**
    * Creates the test runner using the ignite affinitycall backend.
    *
    * @throws Exception if something goes wrong
    */
-  TestMapReduceOSHDBIgniteAffinityCall() throws Exception {
-    super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.AFFINITY_CALL));
+  MapReduceOSHDBIgniteAffinityCallTest() throws Exception {
+    super(oshdb -> oshdb.computeMode(AFFINITY_CALL));
   }
 
   @Test

diff --git a/...ts/TestMapReduceOSHDBIgniteLocalPeek.java → ...ts/MapReduceOSHDBIgniteLocalPeekTest.java b/...ts/TestMapReduceOSHDBIgniteLocalPeek.java → ...ts/MapReduceOSHDBIgniteLocalPeekTest.java
@@ -1,19 +1,19 @@
 package org.heigit.ohsome.oshdb.api.tests;
 
-import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
+import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.LOCAL_PEEK;
 
 /**
  * {@inheritDoc}
  *
  * <p>Runs the tests using the "local peek" Ignite backend.</p>
  */
-class TestMapReduceOSHDBIgniteLocalPeek extends TestMapReduceOSHDBIgnite {
+class MapReduceOSHDBIgniteLocalPeekTest extends MapReduceOSHDBIgniteTest {
   /**
    * Creates the test runner using the ignite localpeak backend.
    *
    * @throws Exception if something goes wrong
    */
-  TestMapReduceOSHDBIgniteLocalPeek() throws Exception {
-    super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.LOCAL_PEEK));
+  MapReduceOSHDBIgniteLocalPeekTest() throws Exception {
+    super(oshdb -> oshdb.computeMode(LOCAL_PEEK));
   }
 }
diff --git a/...TestMapReduceOSHDBIgniteMissingCache.java → ...MapReduceOSHDBIgniteMissingCacheTest.java b/...TestMapReduceOSHDBIgniteMissingCache.java → ...MapReduceOSHDBIgniteMissingCacheTest.java
@@ -2,51 +2,50 @@
 
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
 import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException;
 import org.junit.jupiter.api.Test;
 
 /**
  * Tests for proper error messages is caches are not pnt on ignite.
  */
-class TestMapReduceOSHDBIgniteMissingCache extends TestMapReduceOSHDBIgnite {
+class MapReduceOSHDBIgniteMissingCacheTest extends MapReduceOSHDBIgniteTest {
   /**
    * Creates the test runner using an Ignite backend.
    *
    * @throws Exception if something goes wrong
    */
-  TestMapReduceOSHDBIgniteMissingCache() throws Exception {
-    super(new OSHDBIgnite(ignite));
+  MapReduceOSHDBIgniteMissingCacheTest() throws Exception {
+    super(oshdb -> {});
     this.oshdb.prefix("<test caches not present>");
   }
 
   @Override
   @Test()
-  void testOSMContributionView() throws Exception {
+  void testOSMContributionView() {
     assertThrows(OSHDBTableNotFoundException.class, super::testOSMContributionView);
   }
 
   @Override
   @Test()
-  void testOSMEntitySnapshotView() throws Exception {
+  void testOSMEntitySnapshotView() {
     assertThrows(OSHDBTableNotFoundException.class, super::testOSMEntitySnapshotView);
   }
 
   @Override
   @Test()
-  void testOSMContributionViewStream() throws Exception {
+  void testOSMContributionViewStream() {
     assertThrows(OSHDBTableNotFoundException.class, super::testOSMContributionViewStream);
   }
 
   @Override
   @Test()
-  void testOSMEntitySnapshotViewStream() throws Exception {
+  void testOSMEntitySnapshotViewStream() {
     assertThrows(OSHDBTableNotFoundException.class, super::testOSMEntitySnapshotViewStream);
   }
 
   @Override
   @Test()
-  void testTimeoutMapReduce() throws Exception {
+  void testTimeoutMapReduce() {
     assertThrows(OSHDBTableNotFoundException.class, this::timeoutMapReduce);
   }
 

diff --git a/...ts/TestMapReduceOSHDBIgniteScanQuery.java → ...ts/MapReduceOSHDBIgniteScanQueryTest.java b/...ts/TestMapReduceOSHDBIgniteScanQuery.java → ...ts/MapReduceOSHDBIgniteScanQueryTest.java
@@ -1,28 +1,28 @@
 package org.heigit.ohsome.oshdb.api.tests;
 
+import static org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode.SCAN_QUERY;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
 import org.junit.jupiter.api.Test;
 
 /**
  * {@inheritDoc}
  *
  * <p>Runs the tests using the "scan query" Ignite backend.</p>
  */
-class TestMapReduceOSHDBIgniteScanQuery extends TestMapReduceOSHDBIgnite {
+class MapReduceOSHDBIgniteScanQueryTest extends MapReduceOSHDBIgniteTest {
   /**
    * Creates the test runner using the ignite scanquery backend.
    *
    * @throws Exception if something goes wrong
    */
-  TestMapReduceOSHDBIgniteScanQuery() throws Exception {
-    super(new OSHDBIgnite(ignite).computeMode(OSHDBIgnite.ComputeMode.SCAN_QUERY));
+  MapReduceOSHDBIgniteScanQueryTest() throws Exception {
+    super(oshdb -> oshdb.computeMode(SCAN_QUERY));
   }
 
   @Override
   @Test
-  void testTimeoutStream() throws Exception {
+  void testTimeoutStream() {
     // ignore this test -> scanquery backend currently doesn't support timeouts for stream()
     assertTrue(true);
   }

diff --git a/...b/api/tests/TestMapReduceOSHDBIgnite.java → ...b/api/tests/MapReduceOSHDBIgniteTest.java b/...b/api/tests/TestMapReduceOSHDBIgnite.java → ...b/api/tests/MapReduceOSHDBIgniteTest.java
@@ -1,14 +1,17 @@
 package org.heigit.ohsome.oshdb.api.tests;
 
+import static org.heigit.ohsome.oshdb.api.db.H2Support.pathToUrl;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
+import java.nio.file.Path;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
+import java.util.function.Consumer;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
@@ -21,13 +24,14 @@
 import org.apache.ignite.logger.slf4j.Slf4jLogger;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.heigit.ohsome.oshdb.api.db.OSHDBH2;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase;
 import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
 import org.heigit.ohsome.oshdb.grid.GridOSHNodes;
 import org.heigit.ohsome.oshdb.util.CellId;
 import org.heigit.ohsome.oshdb.util.TableNames;
 
-abstract class TestMapReduceOSHDBIgnite extends TestMapReduce {
+abstract class MapReduceOSHDBIgniteTest extends MapReduceTest {
   static final Ignite ignite;
 
   static {
@@ -46,16 +50,8 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce {
     ignite = Ignition.start(cfg);
   }
 
-  TestMapReduceOSHDBIgnite(OSHDBIgnite oshdb) throws Exception {
-    super(oshdb);
-
+  private static OSHDBDatabase initOshdb(Consumer<OSHDBIgnite> computeMode) {
     final String prefix = "tests";
-    oshdb.prefix(prefix);
-
-    OSHDBH2 oshdbH2 = new OSHDBH2("../data/test-data");
-    this.keytables = oshdbH2;
-
-    Ignite ignite = ((OSHDBIgnite) this.oshdb).getIgnite();
     ignite.cluster().state(ClusterState.ACTIVE);
 
     CacheConfiguration<Long, GridOSHNodes> cacheCfg =
@@ -69,15 +65,19 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce {
     ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_WAYS.toString(prefix)));
     ignite.getOrCreateCache(new CacheConfiguration<>(TableNames.T_RELATIONS.toString(prefix)));
 
+    JdbcConnectionPool oshdbH2 = JdbcConnectionPool.create(pathToUrl(Path.of("../data/test-data")), "sa",
+        "");
+
     // load test data into ignite cache
-    try (IgniteDataStreamer<Long, GridOSHNodes> streamer = ignite.dataStreamer(cache.getName())) {
-      Connection h2Conn = oshdbH2.getConnection();
-      Statement h2Stmt = h2Conn.createStatement();
+    try (IgniteDataStreamer<Long, GridOSHNodes> streamer = ignite.dataStreamer(cache.getName());
+        Connection h2Conn = oshdbH2.getConnection();
+        Statement h2Stmt = h2Conn.createStatement();
+    ) {
 
       streamer.allowOverwrite(true);
 
       try (final ResultSet rst =
-          h2Stmt.executeQuery("select level, id, data from " + TableNames.T_NODES.toString())) {
+          h2Stmt.executeQuery("select level, id, data from " + TableNames.T_NODES)) {
         while (rst.next()) {
           final int level = rst.getInt(1);
           final long id = rst.getLong(2);
@@ -96,5 +96,14 @@ abstract class TestMapReduceOSHDBIgnite extends TestMapReduce {
     }
 
     ignite.cluster().state(ClusterState.ACTIVE_READ_ONLY);
+
+    var oshdb = new OSHDBIgnite(ignite, oshdbH2);
+    oshdb.prefix(prefix);
+    computeMode.accept(oshdb);
+    return oshdb;
+  }
+
+  MapReduceOSHDBIgniteTest(Consumer<OSHDBIgnite> computeMode) throws Exception {
+    super(initOshdb(computeMode));
   }
 }
diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/H2Support.java
@@ -0,0 +1,15 @@
+package org.heigit.ohsome.oshdb.api.db;
+
+import java.nio.file.Path;
+
+public class H2Support {
+
+  private H2Support() {
+  }
+
+  public static String pathToUrl(Path path) {
+    var absolutePath = path.toAbsolutePath().toString();
+    absolutePath = absolutePath.replaceAll("\\.mv\\.db$", "");
+    return String.format("jdbc:h2:%s;ACCESS_MODE_DATA=r", absolutePath);
+  }
+}
diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java
@@ -4,6 +4,7 @@
 import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
 import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException;
 import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible;
+import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
 
 /**
  * OSHDB database backend connector.
@@ -12,6 +13,8 @@ public abstract class OSHDBDatabase implements AutoCloseable {
   private String prefix = "";
   private Long timeout = null;
 
+  public abstract TagTranslator getTagTranslator();
+
   /**
    * Factory function that creates a mapReducer object of the appropriate data type class for this
    * oshdb backend implemenation.