diff --git a/CHANGELOG.md b/CHANGELOG.md index 09fc1626..5ed8c8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ Changelog ## 1.8.0-SNAPSHOT (current master) +### Breaking Changes +* remove caching command line parameter ([#281]) +* update to OSHDB 1.0.0-SNAPSHOT ([#281]) + +### New Features +* tag translator parameter are now configurable via CLI parameter: `tt.maxbytesvalue`, `tt.maxnumroles` ([#281]) + +[#281]: https://github.com/GIScience/ohsome-api/pull/281 + ## 1.7.0 diff --git a/README.md b/README.md index 6d56a59b..e66c23c8 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,6 @@ Now you should have a running local API, which is ready for receiving requests u *Note:* * additionally you can add optional run-parameters: * to disable multithreading: `--database.multithreading=false` - * to enable in-memory-caching: `--database.caching=true` (caution.. enabling this option requires quite some memory, but makes processing much faster) * if you want to run the maven project in your IDE, you need to set the paths to your data in the run configurations * in Eclipse: *Run As --> Run Configurations --> (x)= Arguments --> Program arguments: 'enter the parameters here'* * if you want to get information about the code directly, you can access the [Javadoc](https://docs.ohsome.org/java/ohsome-api/), which gets updated daily. diff --git a/pom.xml b/pom.xml index 000698e2..897f8d0b 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 0.17.0 3.2.0 5.6 - 1.0.0-beta-1 + 1.0.0-RC1 1.18.16 src/main/lombok 2.5.14 diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java index cb768197..b74d91fc 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java @@ -1,21 +1,11 @@ package org.heigit.ohsome.ohsomeapi; -import com.zaxxer.hikari.HikariConfig; import java.io.IOException; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Properties; import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; -import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; -import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; -import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; -import org.heigit.ohsome.ohsomeapi.utils.RequestUtils; -import org.heigit.ohsome.oshdb.api.db.OSHDBH2; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; -import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; +import org.heigit.ohsome.ohsomeapi.utils.ConfigureApplication; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.DefaultApplicationArguments; @@ -84,125 +74,7 @@ public static void main(String[] args) { */ public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - final String dbProperty = "database.db"; - boolean multithreading = true; - boolean caching = false; - String dbPrefix = null; - long timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS; - int numberOfClusterNodes = DEFAULT_NUMBER_OF_CLUSTER_NODES; - int numberOfDataExtractionThreads = DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; - // only used when tests are executed directly in Eclipse - if (System.getProperty(dbProperty) != null) { - DbConnData.db = new OSHDBH2(System.getProperty(dbProperty)); - } - for (String paramName : args.getOptionNames()) { - switch (paramName) { - case dbProperty: - DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0)); - break; - case "database.jdbc": - String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); - break; - case "database.ignite": - if (DbConnData.db != null) { - break; - } - DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0)); - break; - case "database.keytables": - DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0)); - break; - case "database.keytables.jdbc": - jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.keytables = - new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); - DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { - try { - Class.forName(jdbcParam[0]); - return new TagTranslator( - DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3])); - } catch (ClassNotFoundException e) { - throw new RuntimeException("A class with this specific name could not be found"); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - HikariConfig hikariConfig = new HikariConfig(); - hikariConfig.setJdbcUrl(jdbcParam[1]); - hikariConfig.setUsername(jdbcParam[2]); - hikariConfig.setPassword(jdbcParam[3]); - hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads); - DbConnData.keytablesDbPoolConfig = hikariConfig; - break; - case "database.multithreading": - if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { - multithreading = false; - } - break; - case "database.caching": - if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) { - caching = true; - } - break; - case "database.prefix": - dbPrefix = args.getOptionValues(paramName).get(0); - break; - case "database.timeout": - timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0)); - break; - case "cluster.servernodes.count": - numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0)); - break; - case "cluster.dataextraction.threadcount": - numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0)); - break; - default: - break; - } - } - if (DbConnData.db == null) { - throw new RuntimeException( - "You have to define one of the following three database parameters: '--database.db', " - + "'--database.ignite', or '--database.jdbc'."); - } - ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0); - DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds); - ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes); - ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads); - if (DbConnData.db instanceof OSHDBJdbc) { - DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading); - } - if (DbConnData.db instanceof OSHDBH2) { - DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching); - } - if (DbConnData.keytables != null) { - DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection()); - } else { - if (!(DbConnData.db instanceof OSHDBJdbc)) { - throw new DatabaseAccessException("Missing keytables."); - } - DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection()); - } - RequestUtils.extractOSHDBMetadata(); - if (DbConnData.mapTagTranslator == null) { - DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator); - } - if (DbConnData.db instanceof OSHDBIgnite) { - RemoteTagTranslator mtt = DbConnData.mapTagTranslator; - ((OSHDBIgnite) DbConnData.db).onClose(() -> { - try { - if (mtt.wasEvaluated()) { - mtt.get().getConnection().close(); - } - } catch (SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - } - if (dbPrefix != null) { - DbConnData.db = DbConnData.db.prefix(dbPrefix); - } + ConfigureApplication.parseArguments(args); } @Override diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index 5b506488..95df128d 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -54,6 +54,7 @@ import org.heigit.ohsome.oshdb.filter.FilterParser; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.geometry.Geo; @@ -180,10 +181,10 @@ public static

Response aggregateGroupByBoundary TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -217,7 +218,7 @@ public static

Response aggregateGroupByBoundary String tagIdentifier; // check for non-remainder objects (which do have the defined key and value) if (entry.getKey().getSecondIndex().getKey() != -1 && tagValue != -1) { - tagIdentifier = tt.getOSMTagOf(keysInt, tagValue).toString(); + tagIdentifier = tt.lookupTag(keysInt, tagValue).toString(); } else { tagIdentifier = "remainder"; } @@ -287,10 +288,10 @@ public static Response aggregateGroupByTag(RequestResource requestResource, TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -307,7 +308,7 @@ public static Response aggregateGroupByTag(RequestResource requestResource, entry.getValue(), requestParameters.isDensity(), df, geom); // check for non-remainder objects (which do have the defined key and value) if (entry.getKey().getKey() != -1 && entry.getKey().getValue() != -1) { - groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString(); + groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString(); } else { groupByName = "remainder"; } @@ -430,7 +431,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource, TagTranslator tt = DbConnData.tagTranslator; Integer[] keysInt = new Integer[groupByKeys.length]; for (int i = 0; i < groupByKeys.length; i++) { - keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt(); + keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i); } MapAggregator, OSMEntitySnapshot> preResult = mapRed.flatMap(f -> { @@ -460,7 +461,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource, entry.getValue(), requestParameters.isDensity(), df, null); // check for non-remainder objects (which do have the defined key) if (entry.getKey() != -1) { - groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString(); + groupByName = tt.lookupTag(entry.getKey(), 0).getKey(); } else { groupByName = "remainder"; } @@ -530,17 +531,18 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc Integer[] keysInt2 = new Integer[keys2.length]; Integer[] valuesInt2 = new Integer[values2.length]; for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt(); + keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) + .orElse(-i); if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { valuesInt1[i] = tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .getValue(); + .map(OSHDBTag::getValue).orElse(-i); } } for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt(); + keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue(); + valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); } } EnumSet osmTypes1 = @@ -780,17 +782,18 @@ public static

Response aggregateBasicFiltersRat Integer[] valuesInt2 = new Integer[values2.length]; TagTranslator tt = DbConnData.tagTranslator; for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt(); + keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) + .orElse(-i); if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { valuesInt1[i] = tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .getValue(); + .map(OSHDBTag::getValue).orElse(-i); } } for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt(); + keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue(); + valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); } } EnumSet osmTypes1 = (EnumSet) processingData.getOsmTypes(); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java index 39438224..471b4b02 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -10,12 +10,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import com.opencsv.CSVWriter; -import com.zaxxer.hikari.HikariDataSource; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; -import java.sql.SQLException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; @@ -46,7 +44,6 @@ import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry; import org.heigit.ohsome.ohsomeapi.exception.BadRequestException; import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; @@ -78,15 +75,14 @@ import org.heigit.ohsome.oshdb.api.mapreducer.Mappable; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.function.SerializablePredicate; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.geometry.Geo; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -140,7 +136,7 @@ public static boolean snapshotMatches(OSMEntitySnapshot snapshot, Set o for (int i = 0; i < keysInt.length; i++) { boolean matchesTag; if (i < valuesInt.length) { - matchesTag = entity.getTags().hasTagValue(keysInt[i], valuesInt[i]); + matchesTag = entity.getTags().hasTag(keysInt[i], valuesInt[i]); } else { matchesTag = entity.getTags().hasTagKey(keysInt[i]); } @@ -856,20 +852,7 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, final ServletOutputStream outputStream) throws ExecutionException, InterruptedException, IOException { ThreadLocal tts; - HikariDataSource keytablesConnectionPool; - if (DbConnData.keytablesDbPoolConfig != null) { - keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig); - tts = ThreadLocal.withInitial(() -> { - try { - return new TagTranslator(keytablesConnectionPool.getConnection()); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - } else { - keytablesConnectionPool = null; - tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator); - } + tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator); ReentrantLock lock = new ReentrantLock(); AtomicBoolean errored = new AtomicBoolean(false); ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads()); @@ -879,11 +862,10 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, Map props = data.getProperties(); OSHDBTag[] tags = (OSHDBTag[]) props.remove("@tags"); if (tags != null) { - for (OSHDBTag tag : tags) { - OSMTag osmTag = tts.get().getOSMTagOf(tag); + tts.get().lookupTag(Set.of(tags)).forEach((oshdbTag, osmTag) -> { String key = osmTag.getKey(); props.put(key.startsWith("@") ? "@" + key : key, osmTag.getValue()); - } + }); } // 1. convert features to geojson try { @@ -921,9 +903,6 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, })).get(); } finally { threadPool.shutdown(); - if (keytablesConnectionPool != null) { - keytablesConnectionPool.close(); - } outputStream.flush(); } } @@ -987,7 +966,7 @@ static Set keysToKeysInt(String[] keys, TagTranslator tt) { if (keys.length != 0) { keysInt = new HashSet<>(keys.length); for (String key : keys) { - keysInt.add(tt.getOSHDBTagKeyOf(key).toInt()); + tt.getOSHDBTagKeyOf(key).map(OSHDBTagKey::toInt).ifPresent(keysInt::add); } } else { keysInt = Collections.emptySet(); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java index 0c294cd2..6e995c8c 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java @@ -30,6 +30,7 @@ import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; @@ -119,10 +120,11 @@ public static Response countGroupByTag(HttpServletRequest servletRequest, TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue) + .orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -172,7 +174,7 @@ public static Response countGroupByTag(HttpServletRequest servletRequest, } else if (entry.getKey().getKey() == -1 && entry.getKey().getValue() == -1) { groupByName = "remainder"; } else { - groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString(); + groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString(); } resultSet[count] = new GroupByResult(groupByName, results); count++; @@ -214,7 +216,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, TagTranslator tt = DbConnData.tagTranslator; Integer[] keysInt = new Integer[groupByKeys.length]; for (int i = 0; i < groupByKeys.length; i++) { - keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt(); + keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i); } SortedMap, Integer> result = null; result = mapRed @@ -256,7 +258,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, } else if (entry.getKey() == -1) { groupByName = "remainder"; } else { - groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString(); + groupByName = tt.lookupTag(entry.getKey(), 0).getKey(); } resultSet[count] = new GroupByResult(groupByName, results); count++; diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java index 25fae145..08070545 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java @@ -359,11 +359,12 @@ public boolean isSimpleFeatureType(String type) { public MapReducer filterOnPlanarRelations(MapReducer mapRed) { // further filtering to not look at all relations TagTranslator tt = DbConnData.tagTranslator; - OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon"); - OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary"); + OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon") + .orElse(new OSHDBTag(-1, -1)); + OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary").orElse(new OSHDBTag(-1, -1)); return mapRed.filter(Filter.byOSMEntity(entity -> !entity.getType().equals(OSMType.RELATION) - || entity.getTags().hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue()) - || entity.getTags().hasTagValue(typeBoundary.getKey(), typeBoundary.getValue()))); + || entity.getTags().hasTag(typeMultipolygon.getKey(), typeMultipolygon.getValue()) + || entity.getTags().hasTag(typeBoundary.getKey(), typeBoundary.getValue()))); } /** diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java index 3cbd6a82..9a9734dc 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java @@ -196,17 +196,9 @@ public MapReducer processParameters(ComputeMode } DbConnData.db.timeout(timeout); if (isSnapshot) { - if (DbConnData.keytables == null) { - mapRed = OSMEntitySnapshotView.on(DbConnData.db); - } else { - mapRed = OSMEntitySnapshotView.on(DbConnData.db).keytables(DbConnData.keytables); - } + mapRed = OSMEntitySnapshotView.on(DbConnData.db); } else { - if (DbConnData.keytables == null) { - mapRed = OSMContributionView.on(DbConnData.db); - } else { - mapRed = OSMContributionView.on(DbConnData.db).keytables(DbConnData.keytables); - } + mapRed = OSMContributionView.on(DbConnData.db); } if (boundary.isRectangle()) { mapRed = diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java index 97d5e6ed..547d3de9 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java @@ -1,6 +1,6 @@ package org.heigit.ohsome.ohsomeapi.oshdb; -import com.zaxxer.hikari.HikariConfig; +import javax.sql.DataSource; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; @@ -11,8 +11,8 @@ public class DbConnData { public static OSHDBDatabase db = null; public static OSHDBJdbc keytables = null; public static TagTranslator tagTranslator = null; - public static RemoteTagTranslator mapTagTranslator = null; - public static HikariConfig keytablesDbPoolConfig = null; + public static DataSource dbSource = null; + public static DataSource keytablesDbSource = null; private DbConnData() { throw new IllegalStateException("Utility class"); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java index ad3a586e..7ca6e48d 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java @@ -4,6 +4,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; + +// TODO: Remove + /** * A tag-translator that can be used in map/flatMap/aggregateBy functions that may be executed on * remote machines. diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java new file mode 100644 index 00000000..62cd0015 --- /dev/null +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -0,0 +1,170 @@ +package org.heigit.ohsome.ohsomeapi.utils; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.io.IOException; +import org.heigit.ohsome.ohsomeapi.Application; +import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; +import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; +import org.heigit.ohsome.oshdb.api.db.H2Support; +import org.heigit.ohsome.oshdb.api.db.OSHDBH2; +import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; +import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; +import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; +import org.springframework.boot.ApplicationArguments; + +/** + * Utility Class for parsing and configuration the ohsome API application. + */ +public class ConfigureApplication { + + private enum DatabaseType { + H2, JDBC, IGNITE, NONE + } + + private boolean multithreading = true; + private String dbPrefix = ""; + private long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; + private int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; + private int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; + private String databaseUser; + private String databasePassword; + private DatabaseType databaseType = DatabaseType.NONE; + private String databaseUrl; + private String keytablesUser; + private String keytablesPassword; + private DatabaseType keytablesType = DatabaseType.NONE; + private String keytablesUrl; + private long ttMaxBytesValue = 512L * 1024L * 1024L; + private int ttMaxNumRoles = Integer.MAX_VALUE; + + private ConfigureApplication(ApplicationArguments args) { + for (String paramName : args.getOptionNames()) { + switch (paramName) { + // TODO change to "database.h2" for a future stable version + case "database.db": + databaseType = DatabaseType.H2; + databaseUrl = args.getOptionValues(paramName).get(0); + break; + case "database.jdbc": + databaseType = DatabaseType.JDBC; + String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); + databaseUrl = jdbcParam[1]; + databaseUser = jdbcParam[2]; + databasePassword = jdbcParam[3]; + break; + case "database.ignite": + databaseType = DatabaseType.IGNITE; + databaseUrl = args.getOptionValues(paramName).get(0); + break; + case "database.keytables": + keytablesType = DatabaseType.H2; + keytablesUrl = args.getOptionValues(paramName).get(0); + break; + case "database.keytables.jdbc": + keytablesType = DatabaseType.JDBC; + String[] keytablesJdbcParam = args.getOptionValues(paramName).get(0).split(";"); + keytablesUrl = keytablesJdbcParam[1]; + keytablesUser = keytablesJdbcParam[2]; + keytablesPassword = keytablesJdbcParam[3]; + break; + case "database.multithreading": + if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { + multithreading = false; + } + break; + case "database.prefix": + dbPrefix = args.getOptionValues(paramName).get(0); + break; + case "database.timeout": + timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0)); + break; + case "cluster.servernodes.count": + numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; + case "cluster.dataextraction.threadcount": + numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; + case "tt.maxbytesvalue": + ttMaxBytesValue = Long.parseLong(args.getOptionValues(paramName).get(0)); + break; + case "tt.maxnumroles": + ttMaxNumRoles = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; + default: + break; + } + } + } + + /** + * Method run by the Application class to parse incoming command line arguments + * @param args ApplicationArguments from spring to be parsed. + */ + public static void parseArguments(ApplicationArguments args) + throws OSHDBKeytablesNotFoundException, IOException { + var config = new ConfigureApplication(args); + switch (config.keytablesType) { + case H2: + DbConnData.keytablesDbSource = H2Support.createJdbcPoolFromPath(config.keytablesUrl); + break; + case JDBC: + var keytablesHc = new HikariConfig(); + keytablesHc.setJdbcUrl(config.keytablesUrl); + keytablesHc.setUsername(config.keytablesUser); + keytablesHc.setPassword(config.keytablesPassword); + keytablesHc.setMaximumPoolSize(config.numberOfDataExtractionThreads); + DbConnData.keytablesDbSource = new HikariDataSource(keytablesHc); + break; + default: + break; + } + switch (config.databaseType) { + case H2: + DbConnData.db = new OSHDBH2(config.databaseUrl); + break; + case JDBC: + var hc = new HikariConfig(); + hc.setJdbcUrl(config.databaseUrl); + hc.setUsername(config.databaseUser); + hc.setPassword(config.databasePassword); + DbConnData.dbSource = new HikariDataSource(hc); + DbConnData.db = new OSHDBJdbc(DbConnData.dbSource, config.dbPrefix, + DbConnData.keytablesDbSource == null ? DbConnData.dbSource + : DbConnData.keytablesDbSource); + break; + case IGNITE: + if (DbConnData.keytablesDbSource == null) { + throw new IllegalArgumentException("Keytables parameter missing"); + } + DbConnData.db = new OSHDBIgnite(config.databaseUrl, config.dbPrefix, + DbConnData.keytablesDbSource); + break; + default: + throw new IllegalArgumentException( + "You have to define one of the following three database parameters: '--database.db', " + + "'--database.ignite', or '--database.jdbc'."); + } + ProcessingData.setTimeout(config.timeoutInMilliseconds / 1000.0); + DbConnData.db.timeoutInMilliseconds(config.timeoutInMilliseconds); + ProcessingData.setNumberOfClusterNodes(config.numberOfClusterNodes); + ProcessingData.setNumberOfDataExtractionThreads(config.numberOfDataExtractionThreads); + if (DbConnData.db instanceof OSHDBJdbc) { + DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(config.multithreading); + } + + // initialize TagTranslator + DbConnData.tagTranslator = new CachedTagTranslator(DbConnData.db.getTagTranslator(), + config.ttMaxBytesValue, config.ttMaxNumRoles); + + // extract metadata + RequestUtils.extractOSHDBMetadata(); + + if (DbConnData.db instanceof OSHDBIgnite) { + ((OSHDBIgnite) DbConnData.db).onClose(() -> { + // TODO add connections to close + }); + } + } +} \ No newline at end of file