From ae0e7bbee6a8024f4dbcba374d4b002ad59e6980 Mon Sep 17 00:00:00 2001
From: Martin Raifer
Date: Wed, 22 Dec 2021 17:49:19 +0100
Subject: [PATCH 01/31] never return negative values for areas (#438)
invalid polygons can potentially in some certain situation have a bigger inner rings area than the outer ring area. The area method should however never return a value smaller than zero. This clips it to a minimum value of zero in such (errror) cases.
---
CHANGELOG.md | 2 ++
.../ohsome/oshdb/util/geometry/Geo.java | 7 ++++++-
.../ohsome/oshdb/util/geometry/GeoTest.java | 20 +++++++++++++++++++
3 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 430176bd0..38e77d36c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ Changelog
### bugfixes
* change geometry filters to be based on full (unclipped) geometries ([#433])
+* make sure area computation never returns negative results (instead zero is returned for the invalid geometries which previously resulted in negative values) ([#438])
### other changes
@@ -17,6 +18,7 @@ Changelog
[#419]: https://github.com/GIScience/oshdb/pull/419
[#433]: https://github.com/GIScience/oshdb/issues/433
+[#438]: https://github.com/GIScience/oshdb/pull/438
## 0.7.2
diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/geometry/Geo.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/geometry/Geo.java
index afff8e563..f0bae4cdc 100644
--- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/geometry/Geo.java
+++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/geometry/Geo.java
@@ -155,6 +155,11 @@ public static double lengthOf(Geometry geom) {
* https://trs.jpl.nasa.gov/handle/2014/40409
*
*
+ *
+ * This method will never return a negative number. For invalid polygons with a larger inner
+ * rings area than the outer ring encompasses, zero is returned instead.
+ *
+ *
* @param poly the polygon for which the area should be calculated. coordinates must be in WGS84
* @return The approximate signed geodesic area of the polygon in square meters.
*/
@@ -164,7 +169,7 @@ public static double areaOf(Polygon poly) {
for (int i = 0; i < poly.getNumInteriorRing(); i++) {
area -= Math.abs(ringArea((LinearRing) poly.getInteriorRingN(i)));
}
- return area;
+ return Math.max(0, area);
}
/**
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/GeoTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/GeoTest.java
index 9925ceadb..13383b804 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/GeoTest.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/GeoTest.java
@@ -1,6 +1,7 @@
package org.heigit.ohsome.oshdb.util.geometry;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import org.junit.Test;
import org.locationtech.jts.geom.Coordinate;
@@ -153,6 +154,25 @@ public void testAreaRealFeatures() {
assertEquals(1.0, Geo.areaOf(polygon) / expectedResult, relativeDelta);
}
+ @Test
+ public void testAreaNotNegative() {
+ Polygon poly = gf.createPolygon(constructRing(
+ 0, 0,
+ 0, 1,
+ 1, 1,
+ 1, 0,
+ 0, 0
+ ), new LinearRing[] { constructRing(
+ 0, 0,
+ 0, 3,
+ 3, 3,
+ 3, 0,
+ 0, 0
+ )});
+ // check that area is not negative
+ assertFalse(Geo.areaOf(poly) < 0);
+ }
+
// Geo.lengthOf
From 64bc0d8a434ee7bc04dcdefe2e1d0084e01806fa Mon Sep 17 00:00:00 2001
From: Martin Raifer
Date: Wed, 22 Dec 2021 15:54:45 +0100
Subject: [PATCH 02/31] add section about geometry building to manual
---
documentation/manual/README.md | 1 +
documentation/manual/api.md | 1 +
documentation/manual/filters.md | 4 ++--
documentation/manual/geometries.md | 26 ++++++++++++++++++++++++++
documentation/manual/views.md | 2 +-
5 files changed, 31 insertions(+), 3 deletions(-)
create mode 100644 documentation/manual/geometries.md
diff --git a/documentation/manual/README.md b/documentation/manual/README.md
index 685a965ac..3d9509c3a 100644
--- a/documentation/manual/README.md
+++ b/documentation/manual/README.md
@@ -16,6 +16,7 @@ Contents
* [OSHDB API Manual](api.md)
Shows the different features of the OSHDB API and how they can be used to efficiently query the OSM history data.
* [Data Views](views.md)
+ * [OSM Feature Geometries](geometries.md)
* [Filtering of OSM data](filters.md)
* [Map and Reduce](map-reduce.md)
* [Data Aggregation](aggregation.md)
diff --git a/documentation/manual/api.md b/documentation/manual/api.md
index eaa552782..e9928dd1d 100644
--- a/documentation/manual/api.md
+++ b/documentation/manual/api.md
@@ -6,6 +6,7 @@ The OSHDB API provides a library that can be used to write custom analysis queri
The [first steps tutorial](../first-steps) explains the main components of the OSHDB API and how they can be used to perform a simple analysis query. On the following sub-pages, the different parts of the API are described in more detail.
* [Data Views](views.md)
+* [OSM Feature Geometries](geometries.md)
* [Filtering of OSM data](filters.md)
* [Map and Reduce](map-reduce.md)
* [Data Aggregation](aggregation.md)
diff --git a/documentation/manual/filters.md b/documentation/manual/filters.md
index 22686e9cb..f47626041 100644
--- a/documentation/manual/filters.md
+++ b/documentation/manual/filters.md
@@ -15,9 +15,9 @@ The output of this filter will keep only OSM entities whose geometry lie within
> For example, a large forest polygon in OSM that completely encompasses a small area of interest _is_ returned by the OSHDB API.
-The resulting geometries produced by the different OSHDB [views](views.md) are by default clipped to the specified area of interest. This makes it possible to directly calculate the length or area of linear or polygonal OSM features within the given query region, without having to consider the fact that some features might only partially lie within the region. It is, at the same time, still possible to access full extent of the respective OSM features' [unclipped](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.html#getGeometryUnclipped()) [geometries](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.html#getGeometryUnclippedBefore()).
+The resulting geometries produced by the different OSHDB [views](views.md) are by default clipped to the specified area of interest. This makes it possible to directly calculate the length or area of linear or polygonal OSM features within the given query region, without having to consider the fact that some features might only partially lie within the region. It is, at the same time, still possible to access full extent of the respective OSM features' [unclipped](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.html#getGeometryUnclipped()) [geometries](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.html#getGeometryUnclippedBefore()). You can find further information in the section about how the OSHDB [builds geometries](geometries.md) from OSM data.
-The OSHDB is able to cope well even with complex polygons that have many vertices, but keep in mind that using simpler geometries will generally result in higher query performance: For example a bounding-box query is executed slightly faster than a polygon-areaOfInterest query with a rectangular polygon.
+The OSHDB is able to cope well even with complex polygons that have many vertices as areas of interest, but keep in mind that using simpler geometries will generally result in higher query performance: For example a bounding-box query is executed slightly faster than a polygon-areaOfInterest query with a rectangular polygon.
diff --git a/documentation/manual/geometries.md b/documentation/manual/geometries.md
new file mode 100644
index 000000000..72a7b0de3
--- /dev/null
+++ b/documentation/manual/geometries.md
@@ -0,0 +1,26 @@
+OSM Feature Geometries
+======================
+
+The [_views_](views.md) provided by the OSHDB API provide direct access to the OSM entities, but also allow to get the [JTS](https://en.wikipedia.org/wiki/JTS_Topology_Suite#Geometry_model) [geometries](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/Geometry.html) of the OSM features corresponding to their state at the requested _snapshot_ or _contribution_ time(s). With this geometry, further operations such as [_filtering_](filters.md#areaOfInterest) or [_aggregation_](aggregation.md#aggregateByGeometry) can be performed. It is also irreplaceable during the [_map-reduce_](map-reduce.md#geometry-helpers) step to generate useful results, such as the length of a road network for example.
+
+For some OSM elements, such as nodes, generating geometries is straight forward, for others the conversion requires further knowledge of the [data model](https://wiki.openstreetmap.org/wiki/Elements) and [tagging schema](https://wiki.openstreetmap.org/wiki/Tags) used by OSM. The following document gives an overview of how the OSHDB handles the building of geometries of different OSM object types.
+
+Nodes
+-----
+
+Nodes are always presented as [`Point`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/Point.html)s. Note that nodes which have never had a tag and are a part of a way or relation are considered to be structural-only points (sometimes called _vertices_), and thus not returned in an OSHDB query when querying all nodes. This is because the OSHDB does not consider them to not be _map features_ by their own. They can however be fetched as members of their parent way or relation objects, if needed.
+
+Ways
+----
+
+Ways are converted to either [`LineString`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/LineString.html) or [`Polygon`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/Polygon.html) geometries depending on their composition and their tags: A not closed way is always represented as a line, while it depends for a closed one. The [`TagInterpreter`](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/util/taginterpreter/TagInterpreter.html) component of the OSHDB is responsible for deciding whether a closed way results in a line or a polygon: A (closed) OSM way with the tag `building=yes` will be converted to a polygon geometry, while a `junction=roundabout` one will not.
+
+Relations
+---------
+
+Relations are handled by the OSHDB in two different ways: [Multipolygons](https://wiki.openstreetmap.org/wiki/Multipolygon) are converted to either [`Polygon`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/Polygon.html) or [`MultiPolygon`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/MultiPolygon.html) geometries (depending on the number of outer rings), while all [other relation types](https://wiki.openstreetmap.org/wiki/Types_of_relation) result in a [`GeometryCollection`](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/GeometryCollection.html).
+
+Invalid OSM Data
+----------------
+
+There are situations where a part of OSM's entities have either incomplete or invalid data, for example a broken multipolygon resulting from a mapping error. The OSHDB makes the best effort to return a (potentially [invalid](https://locationtech.github.io/jts/javadoc/org/locationtech/jts/geom/Geometry.html#isValid--)) geometry also for these objects. For performance reasons the OSHDB cannot check for every possible error in the input (OSM) data, and for similar reasons it also cannot correct all errors it does find. This means that the OSHDB does not provide any guaranteed outcome for all possible errors and might return an invalid or valid geometry as a result in such cases or even no result at all.
diff --git a/documentation/manual/views.md b/documentation/manual/views.md
index 4d35dc005..6f499c2bc 100644
--- a/documentation/manual/views.md
+++ b/documentation/manual/views.md
@@ -1,7 +1,7 @@
Views
=====
-Two different ways of querying OSM data are available, which determine how the OSM history data is actually analyzed in a given OSHDB query:
+Two different ways of querying OSM data are available, which determine how the OSM history data is actually analyzed in a given OSHDB query:
* The **snapshot view** ([`OSMEntitySnapshotView`](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/api/mapreducer/OSMEntitySnapshotView.html)) returns the state of the OSM history data at specific given points in time.
* The **contribution view** ([`OSMContributionView`](https://docs.ohsome.org/java/oshdb/0.7.2/aggregated/org/heigit/ohsome/oshdb/api/mapreducer/OSMContributionView.html)) returns all modifications (e.g., creations, modifications or deletions) to the OSM elements within a given time period.
From 76ca3b530445c7d15c5e3b6ac8e1767f69917186 Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Wed, 13 Apr 2022 15:27:43 +0200
Subject: [PATCH 03/31] free up OHSDB.java to get used as a common utility
class
---
.../heigit/ohsome/oshdb/api/db/OSHDBDatabase.java | 3 +--
.../main/java/org/heigit/ohsome/oshdb/OSHDB.java | 13 ++-----------
2 files changed, 3 insertions(+), 13 deletions(-)
diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java
index da0a37a45..600247699 100644
--- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java
+++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java
@@ -1,7 +1,6 @@
package org.heigit.ohsome.oshdb.api.db;
import java.util.OptionalLong;
-import org.heigit.ohsome.oshdb.OSHDB;
import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException;
import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible;
@@ -9,7 +8,7 @@
/**
* OSHDB database backend connector.
*/
-public abstract class OSHDBDatabase extends OSHDB implements AutoCloseable {
+public abstract class OSHDBDatabase implements AutoCloseable {
private String prefix = "";
private Long timeout = null;
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDB.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDB.java
index 900ab7e64..8cb57d85f 100644
--- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDB.java
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDB.java
@@ -1,17 +1,8 @@
package org.heigit.ohsome.oshdb;
-public abstract class OSHDB {
+public class OSHDB {
+ private OSHDB() {}
public static final int MAXZOOM = 14;
- /**
- * Returns various metadata properties of this OSHDB instance.
- *
- * For example, metadata("extract.region") returns the geographic region for which the current
- * oshdb extract has been generated in GeoJSON format.
- *
- * @param property the metadata property to request
- * @return the value of the requested metadata field
- */
- public abstract String metadata(String property);
}
From 2e91ea24ecdd6d8e08bdfeaa92779754397b9516 Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Wed, 13 Apr 2022 15:28:56 +0200
Subject: [PATCH 04/31] remove deprecated OSHEntity.getRawTagKeys
---
.../org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java | 4 ++--
.../java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java | 5 -----
.../src/main/java/org/heigit/ohsome/oshdb/osh/OSHEntity.java | 3 ---
3 files changed, 2 insertions(+), 10 deletions(-)
diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
index b2b7e9e1a..f6645b509 100644
--- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
+++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
@@ -673,8 +673,8 @@ public MapReducer osmTag(Collection extends OSMTagInterface> tags) {
}
MapReducer ret = this.copy();
ret.preFilters.add(oshEntity -> {
- for (int key : oshEntity.getRawTagKeys()) {
- if (preKeyIds.contains(key)) {
+ for (var key : oshEntity.getTagKeys()) {
+ if (preKeyIds.contains(key.toInt())) {
return true;
}
}
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
index 178b73d69..f6b4eba13 100644
--- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
@@ -431,11 +431,6 @@ public OSHDBTagKey next() {
};
}
- @Override
- public int[] getRawTagKeys() {
- return keys;
- }
-
@Override
public boolean hasTagKey(OSHDBTagKey tag) {
return this.hasTagKey(tag.toInt());
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/osh/OSHEntity.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/osh/OSHEntity.java
index 63c7e809b..fb88fbade 100644
--- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/osh/OSHEntity.java
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/osh/OSHEntity.java
@@ -13,9 +13,6 @@ public interface OSHEntity extends OSHDBBoundable {
OSMType getType();
- @Deprecated(since = "0.7.0", forRemoval = true)
- int[] getRawTagKeys();
-
Iterable getTagKeys();
boolean hasTagKey(OSHDBTagKey tag);
From e502413e06ccb9c8b9f6699cd9827297f44bccef Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Thu, 14 Apr 2022 19:00:42 +0200
Subject: [PATCH 05/31] Update CHANGELOG.md
---
CHANGELOG.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 38e77d36c..0edfd9f2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,12 +13,13 @@ Changelog
* make sure area computation never returns negative results (instead zero is returned for the invalid geometries which previously resulted in negative values) ([#438])
### other changes
-
+ * remove deprecated method `OSHEntity.getRawTagKeys` ([#441])
* update jts dependency to version 1.18.2
[#419]: https://github.com/GIScience/oshdb/pull/419
[#433]: https://github.com/GIScience/oshdb/issues/433
[#438]: https://github.com/GIScience/oshdb/pull/438
+[#441]: https://github.com/GIScience/oshdb/pull/441
## 0.7.2
From 68f2197ca9eb5ff8d9d8134f4604c3db991957ea Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Thu, 14 Apr 2022 19:03:08 +0200
Subject: [PATCH 06/31] Update CHANGELOG.md
Co-authored-by: Johannes Visintini
---
CHANGELOG.md | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0edfd9f2b..cd80d2018 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,8 +13,9 @@ Changelog
* make sure area computation never returns negative results (instead zero is returned for the invalid geometries which previously resulted in negative values) ([#438])
### other changes
- * remove deprecated method `OSHEntity.getRawTagKeys` ([#441])
- * update jts dependency to version 1.18.2
+
+* remove deprecated method `OSHEntity.getRawTagKeys` ([#441])
+* update jts dependency to version 1.18.2
[#419]: https://github.com/GIScience/oshdb/pull/419
[#433]: https://github.com/GIScience/oshdb/issues/433
From 83b0290a65745792c2655c44f211b85226807183 Mon Sep 17 00:00:00 2001
From: Johannes Visintini
Date: Thu, 14 Apr 2022 20:22:00 +0200
Subject: [PATCH 07/31] Jenkins: use SonarCloud PR feature (#444)
---
Jenkinsfile | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 722a371db..9df7e1712 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,7 +66,17 @@ pipeline {
script {
// START CUSTOM oshdb
withSonarQubeEnv('sonarcloud GIScience/ohsome') {
- sh "mvn $MAVEN_GENERAL_OPTIONS sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.projectName=OSHDB"
+ SONAR_CLI_PARAMETER = "-Dsonar.projectName=OSHDB"
+ if (env.CHANGE_ID) {
+ SONAR_CLI_PARAMETER += " " +
+ "-Dsonar.pullrequest.key=${env.CHANGE_ID} " +
+ "-Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} " +
+ "-Dsonar.pullrequest.base=${env.CHANGE_TARGET}"
+ } else {
+ SONAR_CLI_PARAMETER += " " +
+ "-Dsonar.branch.name=${env.BRANCH_NAME}"
+ }
+ sh "mvn $MAVEN_GENERAL_OPTIONS sonar:sonar ${SONAR_CLI_PARAMETER}"
}
// END CUSTOM oshdb
report_basedir = "/srv/reports/${REPO_NAME}/${VERSION}_${env.BRANCH_NAME}/${env.BUILD_NUMBER}_${LATEST_COMMIT_ID}"
From ad9227c4891f1cf6359ee63855da5b23cbd62037 Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Tue, 19 Apr 2022 14:12:39 +0200
Subject: [PATCH 08/31] Remove deprecated OSMEntity.getRawTags (#443)
deprecated remove OSMEntity.getRawTags
Co-authored-by: Johannes Visintini
---
CHANGELOG.md | 2 +
.../oshdb/api/mapreducer/MapReducer.java | 26 ++--
.../oshdb/tool/importer/load/LoaderNode.java | 2 +-
.../importer/transform/oshdb/OSHEntity2.java | 20 ++-
.../oshdb/filter/GeometryTypeFilter.java | 4 +-
.../ohsome/oshdb/filter/TagFilterEquals.java | 2 +-
.../oshdb/filter/TagFilterEqualsAny.java | 2 +-
.../oshdb/filter/TagFilterNotEquals.java | 2 +-
.../oshdb/filter/TagFilterNotEqualsAny.java | 2 +-
.../oshdb/util/celliterator/CellIterator.java | 13 +-
.../taginterpreter/BaseTagInterpreter.java | 34 ++---
.../taginterpreter/DefaultTagInterpreter.java | 12 +-
.../IterateByContributionNodesTest.java | 24 +--
.../IterateByContributionWaysTest.java | 4 +-
.../IterateByTimestampsNodesTest.java | 48 +++---
.../IterateByTimestampsWaysTest.java | 14 +-
.../helpers/OSMXmlReaderTagInterpreter.java | 4 +-
.../util/osh/TestOSHEntityTimeUtils.java | 4 +-
.../org/heigit/ohsome/oshdb/OSHDBTags.java | 142 ++++++++++++++++++
.../ohsome/oshdb/impl/osh/OSHEntityImpl.java | 19 ++-
.../heigit/ohsome/oshdb/osm/OSMEntity.java | 109 +-------------
.../heigit/ohsome/oshdb/OSHDBTagsTest.java | 82 ++++++++++
.../heigit/ohsome/oshdb/osm/OSMNodeTest.java | 21 +--
.../ohsome/oshdb/osm/OSMRelationTest.java | 33 ++--
.../heigit/ohsome/oshdb/osm/OSMWayTest.java | 41 ++---
25 files changed, 372 insertions(+), 294 deletions(-)
create mode 100644 oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java
create mode 100644 oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd80d2018..648d16f63 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,12 +15,14 @@ Changelog
### other changes
* remove deprecated method `OSHEntity.getRawTagKeys` ([#441])
+* remove deprecated method `OSMEntity.getRawTags` ([#443])
* update jts dependency to version 1.18.2
[#419]: https://github.com/GIScience/oshdb/pull/419
[#433]: https://github.com/GIScience/oshdb/issues/433
[#438]: https://github.com/GIScience/oshdb/pull/438
[#441]: https://github.com/GIScience/oshdb/pull/441
+[#443]: https://github.com/GIScience/oshdb/pull/443
## 0.7.2
diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
index f6645b509..c4003c810 100644
--- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
+++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java
@@ -584,13 +584,10 @@ public MapReducer osmTag(String key, Collection values) {
MapReducer ret = this.copy();
ret.preFilters.add(oshEntity -> oshEntity.hasTagKey(keyId));
ret.filters.add(osmEntity -> {
- int[] tags = osmEntity.getRawTags();
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] > keyId) {
- break;
- }
- if (tags[i] == keyId) {
- return valueIds.contains(tags[i + 1]);
+ var tags = osmEntity.getTags();
+ for (var tag : tags) {
+ if (tag.getKey() == keyId) {
+ return valueIds.contains(tag.getValue());
}
}
return false;
@@ -618,13 +615,10 @@ public MapReducer osmTag(String key, Pattern valuePattern) {
MapReducer ret = this.copy();
ret.preFilters.add(oshEntity -> oshEntity.hasTagKey(keyId));
ret.filters.add(osmEntity -> {
- int[] tags = osmEntity.getRawTags();
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] > keyId) {
- return false;
- }
- if (tags[i] == keyId) {
- String value = this.getTagTranslator().getOSMTagOf(keyId, tags[i + 1]).getValue();
+ var tags = osmEntity.getTags();
+ for (var tag : tags) {
+ if (tag.getKey() == keyId) {
+ String value = this.getTagTranslator().getOSMTagOf(keyId, tag.getValue()).getValue();
return valuePattern.matcher(value).matches();
}
}
@@ -695,7 +689,7 @@ public MapReducer osmTag(Collection extends OSMTagInterface> tags) {
private MapReducer osmTag(OSHDBTag tag) {
MapReducer ret = this.copy();
ret.preFilters.add(oshEntity -> oshEntity.hasTagKey(tag.getKey()));
- ret.filters.add(osmEntity -> osmEntity.hasTagValue(tag.getKey(), tag.getValue()));
+ ret.filters.add(osmEntity -> osmEntity.getTags().hasTagValue(tag.getKey(), tag.getValue()));
return ret;
}
@@ -703,7 +697,7 @@ private MapReducer osmTag(OSHDBTag tag) {
private MapReducer osmTag(OSHDBTagKey tagKey) {
MapReducer ret = this.copy();
ret.preFilters.add(oshEntity -> oshEntity.hasTagKey(tagKey));
- ret.filters.add(osmEntity -> osmEntity.hasTagKey(tagKey));
+ ret.filters.add(osmEntity -> osmEntity.getTags().hasTagKey(tagKey));
return ret;
}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
index f7a252947..659cb3126 100644
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
+++ b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
@@ -98,7 +98,7 @@ public void load(long cellId2, boolean all) {
if (onlyNodesWithTags) {
grid.entities = nodes.stream().filter(osh -> {
- return osh.stream().anyMatch(osm -> osm.getRawTags().length > 0);
+ return osh.stream().anyMatch(osm -> osm.getTags().size() > 0);
}).collect(Collectors.toList());
} else {
grid.entities = new ArrayList<>(nodes);
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity2.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity2.java
index cca24543d..e388424dd 100644
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity2.java
+++ b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity2.java
@@ -1,7 +1,6 @@
package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -9,6 +8,7 @@
import java.util.Set;
import java.util.TreeSet;
import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.heigit.ohsome.oshdb.osm.OSMEntity;
import org.heigit.ohsome.oshdb.util.bytearray.ByteArrayOutputWrapper;
import org.heigit.ohsome.oshdb.util.bytearray.ByteArrayWrapper;
@@ -90,8 +90,7 @@ public void build(ByteArrayOutputWrapper out, ByteArrayOutputWrapper aux,
long timestamp = 0;
long changeset = 0;
int userId = -1;
- int[] tags = new int[0];
-
+ var tags = OSHDBTags.empty();
for (OSMEntity version : versions) {
final int visible = version.isVisible() ? 1 : -1;
@@ -106,15 +105,14 @@ public void build(ByteArrayOutputWrapper out, ByteArrayOutputWrapper aux,
changed |= CHANGED_USER_ID;
userId = aux.writeS32Delta(version.getUserId(), userId);
}
- if (!Arrays.equals(version.getRawTags(), tags)) {
+ if (!version.getTags().equals(tags)) {
changed |= CHANGED_TAGS;
- tags = version.getRawTags();
- aux.writeU32(tags.length);
- for (int i = 0; i < tags.length; i++) {
- aux.writeU32(tags[i]);
- if (i % 2 == 0) {
- keySet.add(Integer.valueOf(tags[i]));
- }
+ tags = version.getTags();
+ aux.writeU32(tags.size() * 2);
+ for (var tag : tags) {
+ aux.writeU32(tag.getKey());
+ aux.writeU32(tag.getValue());
+ keySet.add(tag.getKey());
}
}
if (extension(aux, version, baseLongitude, baseLatitude, nodeOffsets, wayOffsets,
diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java
index 0605b7efd..69fc25085 100644
--- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java
+++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryTypeFilter.java
@@ -150,8 +150,8 @@ public boolean applyOSM(OSMEntity entity) {
OSMMember[] wayNodes = ((OSMWay) entity).getMembers();
return wayNodes.length >= 4 && wayNodes[0].getId() == wayNodes[wayNodes.length - 1].getId();
} else if (osmType == OSMType.RELATION) {
- return entity.hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue())
- || entity.hasTagValue(typeBoundary.getKey(), typeBoundary.getValue());
+ return entity.getTags().hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue())
+ || entity.getTags().hasTagValue(typeBoundary.getKey(), typeBoundary.getValue());
}
}
return true;
diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java
index 124cef36f..1f3065dcb 100644
--- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java
+++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEquals.java
@@ -21,7 +21,7 @@ public OSHDBTag getTag() {
@Override
public boolean applyOSM(OSMEntity entity) {
- return entity.hasTagValue(tag.getKey(), tag.getValue());
+ return entity.getTags().hasTagValue(tag.getKey(), tag.getValue());
}
@Override
diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEqualsAny.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEqualsAny.java
index d737c2728..30228a3de 100644
--- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEqualsAny.java
+++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterEqualsAny.java
@@ -21,7 +21,7 @@ public OSHDBTagKey getTag() {
@Override
public boolean applyOSM(OSMEntity entity) {
- return entity.hasTagKey(tag.toInt());
+ return entity.getTags().hasTagKey(tag.toInt());
}
@Override
diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java
index 4f9664a57..84b444a0a 100644
--- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java
+++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java
@@ -27,7 +27,7 @@ public boolean applyOSH(OSHEntity entity) {
@Override
public boolean applyOSM(OSMEntity entity) {
- return !entity.hasTagValue(tag.getKey(), tag.getValue());
+ return !entity.getTags().hasTagValue(tag.getKey(), tag.getValue());
}
@Override
diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java
index 821a30ff6..38778c93f 100644
--- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java
+++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java
@@ -27,7 +27,7 @@ public boolean applyOSH(OSHEntity entity) {
@Override
public boolean applyOSM(OSMEntity entity) {
- return !entity.hasTagKey(tag.toInt());
+ return !entity.getTags().hasTagKey(tag.toInt());
}
@Override
diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/CellIterator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/CellIterator.java
index 3f295a92e..4f8d840a2 100644
--- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/CellIterator.java
+++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/celliterator/CellIterator.java
@@ -724,18 +724,7 @@ private IterateAllEntry getNext() {
switch (contributionType) {
case TAG_CHANGE:
// look if tags have been changed between versions
- boolean tagsChange = false;
- if (prevEntity.getRawTags().length != osmEntity.getRawTags().length) {
- tagsChange = true;
- } else {
- for (int i = 0; i < prevEntity.getRawTags().length; i++) {
- if (prevEntity.getRawTags()[i] != osmEntity.getRawTags()[i]) {
- tagsChange = true;
- break;
- }
- }
- }
- return tagsChange;
+ return !prevEntity.getTags().equals(osmEntity.getTags());
case GEOMETRY_CHANGE:
// look if geometry has been changed between versions
return !prevGeometry.equals(geom);
diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java
index 105fc3528..22429dbe4 100644
--- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java
+++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/BaseTagInterpreter.java
@@ -1,7 +1,9 @@
package org.heigit.ohsome.oshdb.util.taginterpreter;
+import static java.util.Collections.emptySet;
import java.util.Map;
import java.util.Set;
+import org.heigit.ohsome.oshdb.OSHDBTag;
import org.heigit.ohsome.oshdb.osh.OSHWay;
import org.heigit.ohsome.oshdb.osm.OSMEntity;
import org.heigit.ohsome.oshdb.osm.OSMMember;
@@ -45,13 +47,13 @@ class BaseTagInterpreter implements TagInterpreter {
}
private boolean evaluateWayForArea(OSMWay entity) {
- int[] tags = entity.getRawTags();
- if (entity.hasTagValue(areaNoTagKeyId, areaNoTagValueId)) {
+ var tags = entity.getTags();
+ if (tags.hasTagValue(areaNoTagKeyId, areaNoTagValueId)) {
return false;
}
- for (int i = 0; i < tags.length; i += 2) {
- if (wayAreaTags.containsKey(tags[i])
- && wayAreaTags.get(tags[i]).contains(tags[i + 1])) {
+ for (var tag : entity.getTags()) {
+ if (wayAreaTags.getOrDefault(tag.getKey(), emptySet())
+ .contains(tag.getValue())) {
return true;
}
}
@@ -59,11 +61,10 @@ private boolean evaluateWayForArea(OSMWay entity) {
}
protected boolean evaluateRelationForArea(OSMRelation entity) {
- int[] tags = entity.getRawTags();
// skip area=no check, since that doesn't make much sense for multipolygon relations (does it??)
- for (int i = 0; i < tags.length; i += 2) {
- if (relationAreaTags.containsKey(tags[i])
- && relationAreaTags.get(tags[i]).contains(tags[i + 1])) {
+ for (var tag : entity.getTags()) {
+ if (relationAreaTags.getOrDefault(tag.getKey(), emptySet())
+ .contains(tag.getValue())) {
return true;
}
}
@@ -97,9 +98,8 @@ public boolean isLine(OSMEntity entity) {
@Override
public boolean hasInterestingTagKey(OSMEntity osm) {
- int[] tags = osm.getRawTags();
- for (int i = 0; i < tags.length; i += 2) {
- if (!uninterestingTagKeys.contains(tags[i])) {
+ for(var tag : osm.getTags()) {
+ if (!uninterestingTagKeys.contains(tag.getKey())) {
return true;
}
}
@@ -121,13 +121,13 @@ public boolean isOldStyleMultipolygon(OSMRelation osmRelation) {
if (outerWayCount != 1) {
return false;
}
- int[] tags = osmRelation.getRawTags();
- for (int i = 0; i < tags.length; i += 2) {
- if (relationAreaTags.containsKey(tags[i])
- && relationAreaTags.get(tags[i]).contains(tags[i + 1])) {
+ var tags = osmRelation.getTags();
+ for (var tag : tags) {
+ if (relationAreaTags.getOrDefault(tag.getKey(), emptySet())
+ .contains(tag.getValue())) {
continue;
}
- if (!uninterestingTagKeys.contains(tags[i])) {
+ if (!uninterestingTagKeys.contains(tag.getKey())) {
return false;
}
}
diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java
index 9811a3a08..86708d23e 100644
--- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java
+++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/taginterpreter/DefaultTagInterpreter.java
@@ -176,14 +176,14 @@ public boolean isLine(OSMEntity entity) {
// checks if the relation has the tag "type=multipolygon"
@Override
protected boolean evaluateRelationForArea(OSMRelation entity) {
- int[] tags = entity.getRawTags();
+ var tags = entity.getTags();
// skip area=no check, since that doesn't make much sense for multipolygon relations (does it??)
// the following is slightly faster than running
// `return entity.hasTagValue(k1,v1) || entity.hasTagValue(k2,v2);`
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] == typeKey) {
- return tags[i + 1] == typeMultipolygonValue || tags[i + 1] == typeBoundaryValue;
- } else if (tags[i] > typeKey) {
+ for (var tag : tags) {
+ if (tag.getKey() == typeKey) {
+ return tag.getValue() == typeMultipolygonValue || tag.getValue() == typeBoundaryValue;
+ } else if (tag.getKey() > typeKey) {
return false;
}
}
@@ -192,6 +192,6 @@ protected boolean evaluateRelationForArea(OSMRelation entity) {
// checks if the relation has the tag "type=route"
private boolean evaluateRelationForLine(OSMRelation entity) {
- return entity.hasTagValue(typeKey, typeRouteValue);
+ return entity.getTags().hasTagValue(typeKey, typeRouteValue);
}
}
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionNodesTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionNodesTest.java
index f61ff88e1..19cd1c6b4 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionNodesTest.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionNodesTest.java
@@ -81,7 +81,7 @@ public void testGeometryChange() {
assertTrue(geom instanceof Point);
assertEquals(result.get(0).geometry.get(), result.get(1).previousGeometry.get());
assertNotEquals(result.get(1).geometry.get(), result.get(1).previousGeometry.get());
- assertEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
+ assertEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
}
@Test
@@ -116,8 +116,8 @@ public void testTagChange() {
result.get(2).activities.get()
);
assertEquals(3, result.get(0).changeset);
- assertNotEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertNotEquals(result.get(2).osmEntity.getRawTags(), result.get(1).osmEntity.getRawTags());
+ assertNotEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertNotEquals(result.get(2).osmEntity.getTags(), result.get(1).osmEntity.getTags());
}
@Test
@@ -211,10 +211,10 @@ public void testMultipleChanges() {
result.get(5).activities.get()
);
assertEquals(11, result.get(0).changeset);
- assertNotEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertNotEquals(result.get(3).osmEntity.getRawTags(), result.get(1).osmEntity.getRawTags());
- assertEquals(result.get(4).osmEntity.getRawTags(), result.get(3).osmEntity.getRawTags());
- assertNotEquals(result.get(5).osmEntity.getRawTags(), result.get(4).osmEntity.getRawTags());
+ assertNotEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertNotEquals(result.get(3).osmEntity.getTags(), result.get(1).osmEntity.getTags());
+ assertEquals(result.get(4).osmEntity.getTags(), result.get(3).osmEntity.getTags());
+ assertNotEquals(result.get(5).osmEntity.getTags(), result.get(4).osmEntity.getTags());
}
@Test
@@ -309,7 +309,7 @@ public void testTagChangeTagFilterWithSuccess() {
OSHDBBoundingBox.bboxWgs84Coordinates(-180.0, -90.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 5,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
false
)).iterateByContribution(
oshdbDataGridCell
@@ -345,7 +345,7 @@ public void testTagChangeTagFilterDisused() {
OSHDBBoundingBox.bboxWgs84Coordinates(-180.0, -90.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 7,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("disused:shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("disused:shop")),
false
)).iterateByContribution(
oshdbDataGridCell
@@ -377,7 +377,7 @@ public void testMoreComplicatedFilter() {
OSHDBBoundingBox.bboxWgs84Coordinates(0.0, 0.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 8,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
false
)).iterateByContribution(
oshdbDataGridCell
@@ -410,7 +410,7 @@ public void testTagChangeTagFilterWithoutSuccess() {
OSHDBBoundingBox.bboxWgs84Coordinates(-180.0, -90.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 5,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().getOrDefault("amenity", -1)),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().getOrDefault("amenity", -1)),
false
)).iterateByContribution(
oshdbDataGridCell
@@ -468,7 +468,7 @@ public void testTagFilterAndPolygonIntersectingDataPartly() {
areaDecider,
oshEntity -> oshEntity.getId() == 6,
// filter entity for tag = shop
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
false
)).iterateByContribution(
oshdbDataGridCell
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionWaysTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionWaysTest.java
index d6bbcda5f..e7c189da1 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionWaysTest.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByContributionWaysTest.java
@@ -312,8 +312,8 @@ public void testMultipleChanges() {
);
assertEquals(44, result.get(0).changeset);
- assertNotEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertEquals(result.get(5).osmEntity.getRawTags(), result.get(3).osmEntity.getRawTags());
+ assertNotEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertEquals(result.get(5).osmEntity.getTags(), result.get(3).osmEntity.getTags());
}
@Test
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsNodesTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsNodesTest.java
index 127d25a71..8935a5c53 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsNodesTest.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsNodesTest.java
@@ -87,17 +87,17 @@ public void testTagChange() {
oshdbDataGridCell
).collect(Collectors.toList());
assertEquals(12, result.size());
- assertNotEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertEquals(result.get(2).osmEntity.getRawTags(), result.get(1).osmEntity.getRawTags());
- assertEquals(result.get(3).osmEntity.getRawTags(), result.get(2).osmEntity.getRawTags());
- assertEquals(result.get(4).osmEntity.getRawTags(), result.get(3).osmEntity.getRawTags());
- assertEquals(result.get(5).osmEntity.getRawTags(), result.get(4).osmEntity.getRawTags());
- assertEquals(result.get(6).osmEntity.getRawTags(), result.get(5).osmEntity.getRawTags());
- assertNotEquals(result.get(7).osmEntity.getRawTags(), result.get(6).osmEntity.getRawTags());
- assertEquals(result.get(8).osmEntity.getRawTags(), result.get(7).osmEntity.getRawTags());
- assertEquals(result.get(9).osmEntity.getRawTags(), result.get(8).osmEntity.getRawTags());
- assertEquals(result.get(10).osmEntity.getRawTags(), result.get(9).osmEntity.getRawTags());
- assertEquals(result.get(11).osmEntity.getRawTags(), result.get(10).osmEntity.getRawTags());
+ assertNotEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertEquals(result.get(2).osmEntity.getTags(), result.get(1).osmEntity.getTags());
+ assertEquals(result.get(3).osmEntity.getTags(), result.get(2).osmEntity.getTags());
+ assertEquals(result.get(4).osmEntity.getTags(), result.get(3).osmEntity.getTags());
+ assertEquals(result.get(5).osmEntity.getTags(), result.get(4).osmEntity.getTags());
+ assertEquals(result.get(6).osmEntity.getTags(), result.get(5).osmEntity.getTags());
+ assertNotEquals(result.get(7).osmEntity.getTags(), result.get(6).osmEntity.getTags());
+ assertEquals(result.get(8).osmEntity.getTags(), result.get(7).osmEntity.getTags());
+ assertEquals(result.get(9).osmEntity.getTags(), result.get(8).osmEntity.getTags());
+ assertEquals(result.get(10).osmEntity.getTags(), result.get(9).osmEntity.getTags());
+ assertEquals(result.get(11).osmEntity.getTags(), result.get(10).osmEntity.getTags());
}
@Test
@@ -158,16 +158,16 @@ public void testMultipleChanges() {
result.get(3).geometry.get().getCoordinates());
assertArrayEquals(result.get(9).geometry.get().getCoordinates(),
result.get(6).geometry.get().getCoordinates());
- assertNotEquals(result.get(1).osmEntity.getRawTags(),
- result.get(0).osmEntity.getRawTags());
- assertEquals(result.get(2).osmEntity.getRawTags(),
- result.get(1).osmEntity.getRawTags());
- assertNotEquals(result.get(3).osmEntity.getRawTags(),
- result.get(2).osmEntity.getRawTags());
- assertEquals(result.get(5).osmEntity.getRawTags(),
- result.get(4).osmEntity.getRawTags());
- assertNotEquals(result.get(9).osmEntity.getRawTags(),
- result.get(6).osmEntity.getRawTags());
+ assertNotEquals(result.get(1).osmEntity.getTags(),
+ result.get(0).osmEntity.getTags());
+ assertEquals(result.get(2).osmEntity.getTags(),
+ result.get(1).osmEntity.getTags());
+ assertNotEquals(result.get(3).osmEntity.getTags(),
+ result.get(2).osmEntity.getTags());
+ assertEquals(result.get(5).osmEntity.getTags(),
+ result.get(4).osmEntity.getTags());
+ assertNotEquals(result.get(9).osmEntity.getTags(),
+ result.get(6).osmEntity.getTags());
}
@Test
@@ -182,7 +182,7 @@ public void testTagChangeTagFilterWithSuccess() {
OSHDBBoundingBox.bboxWgs84Coordinates(-180.0, -90.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 5,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
false
)).iterateByTimestamps(
oshdbDataGridCell
@@ -202,7 +202,7 @@ public void testTagChangeTagFilterWithoutSuccess() {
OSHDBBoundingBox.bboxWgs84Coordinates(-180.0, -90.0, 180.0, 90.0),
areaDecider,
oshEntity -> oshEntity.getId() == 5,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().getOrDefault("amenity", -1)),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().getOrDefault("amenity", -1)),
false
)).iterateByTimestamps(
oshdbDataGridCell
@@ -231,7 +231,7 @@ public void testTagFilterAndPolygonIntersectingDataPartly() {
polygonFromCoordinates,
areaDecider,
oshEntity -> oshEntity.getId() == 6,
- osmEntity -> osmEntity.hasTagKey(osmXmlTestData.keys().get("shop")),
+ osmEntity -> osmEntity.getTags().hasTagKey(osmXmlTestData.keys().get("shop")),
false
)).iterateByTimestamps(
oshdbDataGridCell
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsWaysTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsWaysTest.java
index 96d778624..94fccb76c 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsWaysTest.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/celliterator/IterateByTimestampsWaysTest.java
@@ -60,7 +60,7 @@ public void testGeometryChange() {
oshdbDataGridCell
).collect(Collectors.toList());
assertEquals(10, result.size());
- assertEquals(result.get(1).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
+ assertEquals(result.get(1).osmEntity.getTags(), result.get(0).osmEntity.getTags());
assertEquals(4, result.get(0).geometry.get().getNumPoints());
assertEquals(8, result.get(1).geometry.get().getNumPoints());
assertEquals(9, result.get(2).geometry.get().getNumPoints());
@@ -161,8 +161,8 @@ public void testTagChange() {
assertTrue(geom2 instanceof LineString);
Geometry geom3 = result.get(10).geometry.get();
assertTrue(geom3 instanceof LineString);
- assertNotEquals(result.get(2).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertNotEquals(result.get(10).osmEntity.getRawTags(), result.get(2).osmEntity.getRawTags());
+ assertNotEquals(result.get(2).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertNotEquals(result.get(10).osmEntity.getTags(), result.get(2).osmEntity.getTags());
assertNotEquals(result.get(2).geometry.get(), result.get(0).geometry.get());
assertEquals(result.get(10).geometry.get(), result.get(2).geometry.get());
}
@@ -217,8 +217,8 @@ public void testMultipleChanges() {
).collect(Collectors.toList());
assertEquals(7, result.size());
- assertNotEquals(result.get(2).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
- assertEquals(result.get(6).osmEntity.getRawTags(), result.get(2).osmEntity.getRawTags());
+ assertNotEquals(result.get(2).osmEntity.getTags(), result.get(0).osmEntity.getTags());
+ assertEquals(result.get(6).osmEntity.getTags(), result.get(2).osmEntity.getTags());
assertEquals(result.get(1).geometry.get(), result.get(0).geometry.get());
assertNotEquals(result.get(3).geometry.get(), result.get(1).geometry.get());
assertNotEquals(result.get(6).geometry.get(), result.get(3).geometry.get());
@@ -250,7 +250,7 @@ public void testPolygonAreaYesTagDisappears() {
assertTrue(geom instanceof Polygon);
Geometry geom2 = result.get(8).geometry.get();
assertTrue(geom2 instanceof LineString);
- assertNotEquals(result.get(8).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
+ assertNotEquals(result.get(8).osmEntity.getTags(), result.get(0).osmEntity.getTags());
assertNotEquals(result.get(8).geometry.get(), result.get(0).geometry.get());
}
@@ -279,7 +279,7 @@ public void testPolygonAreaYesNodeDisappears() {
assertTrue(geom instanceof Polygon);
Geometry geom2 = result.get(8).geometry.get();
assertTrue(geom2 instanceof LineString);
- assertEquals(result.get(8).osmEntity.getRawTags(), result.get(0).osmEntity.getRawTags());
+ assertEquals(result.get(8).osmEntity.getTags(), result.get(0).osmEntity.getTags());
assertNotEquals(result.get(8).geometry.get(), result.get(0).geometry.get());
}
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java
index 2e1cb6e81..b48ad263c 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/geometry/helpers/OSMXmlReaderTagInterpreter.java
@@ -38,10 +38,10 @@ public boolean isArea(OSMEntity e) {
if (e instanceof OSMWay) {
OSMMember[] nds = ((OSMWay) e).getMembers();
return nds.length >= 4 && nds[0].getId() == nds[nds.length - 1].getId()
- && e.hasTagValue(area, areaYes);
+ && e.getTags().hasTagValue(area, areaYes);
}
if (e instanceof OSMRelation) {
- return e.hasTagValue(type, typeMultipolygon);
+ return e.getTags().hasTagValue(type, typeMultipolygon);
}
return true;
}
diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java
index f31f5f4f1..4866af1e4 100644
--- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java
+++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/osh/TestOSHEntityTimeUtils.java
@@ -59,7 +59,7 @@ public void testGetModificationTimestampsNodeWithFilter() throws IOException {
));
List tss =
- OSHEntityTimeUtils.getModificationTimestamps(hnode, e -> e.hasTagValue(1, 1));
+ OSHEntityTimeUtils.getModificationTimestamps(hnode, e -> e.getTags().hasTagValue(1, 1));
assertNotNull(tss);
assertEquals(2, tss.size());
@@ -177,7 +177,7 @@ public void testGetModificationTimestampsWayWithFilter() throws IOException {
tss = OSHEntityTimeUtils.getModificationTimestamps(
hway,
- osmEntity -> osmEntity.hasTagValue(2, 1)
+ osmEntity -> osmEntity.getTags().hasTagValue(2, 1)
);
assertNotNull(tss);
assertEquals(5, tss.size());
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java
new file mode 100644
index 000000000..fdb909a0a
--- /dev/null
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBTags.java
@@ -0,0 +1,142 @@
+package org.heigit.ohsome.oshdb;
+
+import java.io.Serializable;
+import java.util.AbstractSet;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.stream.IntStream;
+import org.heigit.ohsome.oshdb.util.OSHDBTagKey;
+
+/**
+ * Collection class for OSHDBTag.
+ *
+ */
+public abstract class OSHDBTags extends AbstractSet implements Serializable {
+ private static final OSHDBTags EMPTY = new IntArrayOSHDBTags(new int[0]);
+
+ public static OSHDBTags empty() {
+ return EMPTY;
+ }
+
+ public boolean hasTagKey(OSHDBTagKey key) {
+ return hasTagKey(key.toInt());
+ }
+
+ /**
+ * Test this {@code OSMEntity} if it contains a certain tag key(integer).
+ */
+ public abstract boolean hasTagKey(int key);
+
+ /**
+ * Tests if any a given key is present but ignores certain values. Useful when looking for example
+ * "TagKey" != "no"
+ *
+ * @param key the key to search for
+ * @param uninterestingValues list of values, that should return false although the key is
+ * actually present
+ * @return true if the key is present and is NOT in a combination with the given values, false
+ * otherwise
+ */
+ public abstract boolean hasTagKeyExcluding(int key, int[] uninterestingValues);
+
+ /**
+ * Test for a certain key/value combination.
+ */
+ public abstract boolean hasTagValue(int key, int value);
+
+ /**
+ * KV based OSHDBTags.
+ *
+ * @param tags kv based array
+ * @return OSHDBTags instance
+ */
+ public static OSHDBTags of(int[] tags) {
+ return new IntArrayOSHDBTags(tags);
+ }
+
+ private static class IntArrayOSHDBTags extends OSHDBTags {
+
+ private final int[] tags;
+
+ private IntArrayOSHDBTags(int[] tags) {
+ this.tags = tags;
+ }
+
+ @Override
+ public int size() {
+ return tags.length / 2;
+ }
+
+ @Override
+ public Iterator iterator() {
+ return IntStream.range(0, tags.length / 2).map(i -> i * 2)
+ .mapToObj(i -> new OSHDBTag(tags[i], tags[i + 1])).iterator();
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(tags);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof IntArrayOSHDBTags) {
+ var other = (IntArrayOSHDBTags) obj;
+ return Arrays.equals(tags, other.tags);
+ }
+ return super.equals(obj);
+ }
+
+ @Override
+ public boolean hasTagKey(int key) {
+ for (int i = 0; i < tags.length; i += 2) {
+ if (tags[i] < key) {
+ continue;
+ }
+ if (tags[i] == key) {
+ return true;
+ }
+ if (tags[i] > key) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasTagKeyExcluding(int key, int[] uninterestingValues) {
+ for (int i = 0; i < tags.length; i += 2) {
+ if (tags[i] < key) {
+ continue;
+ }
+ if (tags[i] == key) {
+ final int value = tags[i + 1];
+ return !IntStream.of(uninterestingValues).anyMatch(x -> x == value);
+ }
+ if (tags[i] > key) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasTagValue(int key, int value) {
+ for (int i = 0; i < tags.length; i += 2) {
+ if (tags[i] < key) {
+ continue;
+ }
+ if (tags[i] == key) {
+ return tags[i + 1] == value;
+ }
+ if (tags[i] > key) {
+ return false;
+ }
+ }
+ return false;
+ }
+ }
+}
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
index f6b4eba13..78cd787db 100644
--- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHEntityImpl.java
@@ -5,7 +5,6 @@
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.io.Serializable;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.Locale;
import java.util.NoSuchElementException;
@@ -13,6 +12,7 @@
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nonnull;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.heigit.ohsome.oshdb.osh.OSHEntity;
import org.heigit.ohsome.oshdb.osm.OSMCoordinates;
import org.heigit.ohsome.oshdb.osm.OSMEntity;
@@ -38,7 +38,7 @@ protected static class Builder {
long lastTimestamp = 0;
long lastChangeset = 0;
int lastUserId = 0;
- int[] lastKeyValues = new int[0];
+ OSHDBTags lastKeyValues = OSHDBTags.empty();
SortedSet keySet = new TreeSet<>();
@@ -77,9 +77,9 @@ protected void build(OSMEntity version, byte changed) {
changed |= CHANGED_USER_ID;
}
- int[] keyValues = version.getRawTags();
+ var keyValues = version.getTags();
- if (version.isVisible() && !Arrays.equals(keyValues, lastKeyValues)) {
+ if (version.isVisible() && !keyValues.equals(lastKeyValues)) {
changed |= CHANGED_TAGS;
}
@@ -91,12 +91,11 @@ protected void build(OSMEntity version, byte changed) {
}
if ((changed & CHANGED_TAGS) != 0) {
- output.writeU32(keyValues.length);
- for (var kv = 0; kv < keyValues.length; kv++) {
- output.writeU32(keyValues[kv]);
- if (kv % 2 == 0) {
- keySet.add(Integer.valueOf(keyValues[kv]));
- }
+ output.writeU32(keyValues.size() * 2);
+ for (var kv : keyValues) {
+ output.writeU32(kv.getKey());
+ output.writeU32(kv.getValue());
+ keySet.add(Integer.valueOf(kv.getKey()));
}
lastKeyValues = keyValues;
}
diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/osm/OSMEntity.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/osm/OSMEntity.java
index 5bd7d06d4..388d92757 100644
--- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/osm/OSMEntity.java
+++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/osm/OSMEntity.java
@@ -1,15 +1,9 @@
package org.heigit.ohsome.oshdb.osm;
import java.io.Serializable;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
import java.util.Objects;
-import java.util.stream.IntStream;
-import javax.annotation.Nonnull;
-import org.heigit.ohsome.oshdb.OSHDBTag;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.heigit.ohsome.oshdb.OSHDBTemporal;
-import org.heigit.ohsome.oshdb.util.OSHDBTagKey;
public abstract class OSMEntity implements OSHDBTemporal, Serializable {
@@ -19,7 +13,7 @@ public abstract class OSMEntity implements OSHDBTemporal, Serializable {
protected final long timestamp;
protected final long changesetId;
protected final int userId;
- protected final int[] tags;
+ protected final OSHDBTags tags;
/**
* Constructor for a OSMEntity. Holds the basic information, every OSM-Object has.
@@ -38,7 +32,7 @@ protected OSMEntity(final long id, final int version, final long timestamp,
this.timestamp = timestamp;
this.changesetId = changesetId;
this.userId = userId;
- this.tags = tags;
+ this.tags = OSHDBTags.of(tags);
}
public long getId() {
@@ -71,102 +65,10 @@ public boolean isVisible() {
/**
* Returns a "view" of the current osm tags.
*/
- public Iterable getTags() {
- return new Iterable<>() {
- @Nonnull
- @Override
- public Iterator iterator() {
- return new Iterator<>() {
- int pos = 0;
-
- @Override
- public boolean hasNext() {
- return pos < tags.length;
- }
-
- @Override
- public OSHDBTag next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- return new OSHDBTag(tags[pos++], tags[pos++]);
- }
- };
- }
- };
- }
-
- @Deprecated(since = "0.7.0", forRemoval = true)
- public int[] getRawTags() {
+ public OSHDBTags getTags() {
return tags;
}
- public boolean hasTagKey(OSHDBTagKey key) {
- return this.hasTagKey(key.toInt());
- }
-
- /**
- * Test this {@code OSMEntity} if it contains a certain tag key(integer).
- */
- public boolean hasTagKey(int key) {
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] < key) {
- continue;
- }
- if (tags[i] == key) {
- return true;
- }
- if (tags[i] > key) {
- return false;
- }
- }
- return false;
- }
-
- /**
- * Tests if any a given key is present but ignores certain values. Useful when looking for example
- * "TagKey" != "no"
- *
- * @param key the key to search for
- * @param uninterestingValues list of values, that should return false although the key is
- * actually present
- * @return true if the key is present and is NOT in a combination with the given values, false
- * otherwise
- */
- public boolean hasTagKeyExcluding(int key, int[] uninterestingValues) {
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] < key) {
- continue;
- }
- if (tags[i] == key) {
- final int value = tags[i + 1];
- return !IntStream.of(uninterestingValues).anyMatch(x -> x == value);
- }
- if (tags[i] > key) {
- return false;
- }
- }
- return false;
- }
-
- /**
- * Test for a certain key/value combination.
- */
- public boolean hasTagValue(int key, int value) {
- for (int i = 0; i < tags.length; i += 2) {
- if (tags[i] < key) {
- continue;
- }
- if (tags[i] == key) {
- return tags[i + 1] == value;
- }
- if (tags[i] > key) {
- return false;
- }
- }
- return false;
- }
-
@Override
public int hashCode() {
return Objects.hash(getType(), id, version);
@@ -187,7 +89,6 @@ public boolean equals(Object obj) {
@Override
public String toString() {
return String.format("ID:%d V:+%d+ TS:%d CS:%d VIS:%s UID:%d TAGS:%S", getId(), getVersion(),
- getEpochSecond(), getChangesetId(), isVisible(), getUserId(),
- Arrays.toString(getRawTags()));
+ getEpochSecond(), getChangesetId(), isVisible(), getUserId(), getTags());
}
}
diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java
new file mode 100644
index 000000000..302c35ae4
--- /dev/null
+++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/OSHDBTagsTest.java
@@ -0,0 +1,82 @@
+package org.heigit.ohsome.oshdb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import java.util.Set;
+import org.heigit.ohsome.oshdb.util.OSHDBTagKey;
+import org.junit.Test;
+
+/**
+ * Test class for OSHDBTags interface.
+ *
+ */
+@SuppressWarnings("javadoc")
+public class OSHDBTagsTest {
+ int[] kvs = new int[] {1, 2, 2, 3, 4, 5};
+
+ @Test
+ public void testArrayHasTagKey() {
+ var tags = OSHDBTags.of(kvs);
+
+ var tagKey2 = new OSHDBTagKey(2);
+ var tagKey3 = new OSHDBTagKey(3);
+
+ assertTrue("tag key should exist", tags.hasTagKey(tagKey2));
+ assertFalse("tag key should not exist", tags.hasTagKey(tagKey3));
+ assertFalse("tag key should not exist", tags.hasTagKey(5));
+ }
+
+ @Test
+ public void testArrayHasTagKeyExcluding() {
+ var tags = OSHDBTags.of(kvs);
+
+ assertTrue(tags.hasTagKeyExcluding(2, new int[] {1, 2, 4}));
+
+ assertFalse(tags.hasTagKeyExcluding(2, new int[] {3}));
+ assertFalse(tags.hasTagKeyExcluding(3, new int[0]));
+ assertFalse(tags.hasTagKeyExcluding(5, new int[0]));
+
+ }
+
+ @Test
+ public void testArrayHasTagValue() {
+ var tags = OSHDBTags.of(kvs);
+
+ assertTrue(tags.hasTagValue(1, 2));
+
+ assertFalse(tags.hasTagValue(2, 2));
+ assertFalse(tags.hasTagValue(3, 4));
+ assertFalse(tags.hasTagValue(5, 6));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testImmutableAdd() {
+ var tags = OSHDBTags.of(kvs);
+
+ tags.add(new OSHDBTag(5, 6));
+ }
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testImmutableRemove() {
+ var tags = OSHDBTags.of(kvs);
+
+ tags.removeIf(tag -> tag.getKey() == 2);
+ }
+
+ @Test
+ public void testArrayEquality() {
+ var tags = OSHDBTags.of(new int[] {2, 2, 4, 4});
+
+ assertEquals(tags, tags);
+ assertEquals(tags, OSHDBTags.of(new int[] {2, 2, 4, 4}));
+ assertEquals(tags, Set.of(new OSHDBTag(2, 2), new OSHDBTag(4, 4)));
+
+ assertNotEquals(tags, OSHDBTags.of(new int[] {1, 1, 4, 4}));
+ assertNotEquals(tags, List.of(new OSHDBTag(2, 2), new OSHDBTag(4, 4)));
+ }
+
+}
diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java
index f56e9481d..0d093638e 100644
--- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java
+++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMNodeTest.java
@@ -3,6 +3,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.junit.Assert;
import org.junit.Test;
@@ -156,39 +157,39 @@ public void testisVisible2() {
@Test
public void testGetTags() {
OSMNode instance = new OSMNode(1L, 1, 1L, 1L, 1, new int[] {}, 1000000000, 1000000000);
- int[] expResult = new int[] {};
- int[] result = instance.getRawTags();
- Assert.assertArrayEquals(expResult, result);
+ var expResult = OSHDBTags.empty();
+ var result = instance.getTags();
+ Assert.assertEquals(expResult, result);
}
@Test
public void testHasTagKey() {
OSMNode instance = new OSMNode(1L, 1, 1L, 1L, 1, new int[] {}, 1000000000, 1000000000);
boolean expResult = false;
- boolean result = instance.hasTagKey(1);
+ boolean result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
instance =
new OSMNode(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3}, 1000000000, 1000000000);
expResult = true;
- result = instance.hasTagKey(1);
+ result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
instance =
new OSMNode(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 2, 3, 3}, 1000000000, 1000000000);
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
instance =
new OSMNode(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3}, 1000000000, 1000000000);
expResult = true;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
instance = new OSMNode(1L, 1, 1L, 1L, 1, new int[] {2, 1, 3, 3}, 1000000000, 1000000000);
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {1, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {1, 3});
assertEquals(expResult, result);
}
@@ -197,12 +198,12 @@ public void testHasTagValue() {
OSMNode instance =
new OSMNode(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, 1000000000, 1000000000);
boolean expResult = false;
- boolean result = instance.hasTagValue(1, 1);
+ boolean result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
instance = new OSMNode(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, 1000000000, 1000000000);
expResult = true;
- result = instance.hasTagValue(1, 1);
+ result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
}
diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java
index 64e1394e3..841ca8d45 100644
--- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java
+++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMRelationTest.java
@@ -4,6 +4,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.junit.Assert;
import org.junit.Test;
@@ -117,9 +118,9 @@ public void testGetTags() {
OSMMember part = new OSMMember(1L, OSMType.WAY, 1);
OSMRelation instance =
new OSMRelation(1L, 2, 1L, 1L, 1, new int[] {1, 1, 2, 2}, new OSMMember[] {part, part});
- int[] expResult = new int[] {1, 1, 2, 2};
- int[] result = instance.getRawTags();
- Assert.assertArrayEquals(expResult, result);
+ var expResult = OSHDBTags.of(new int[] {1, 1, 2, 2});
+ var result = instance.getTags();
+ Assert.assertEquals(expResult, result);
}
@Test
@@ -128,35 +129,35 @@ public void testHasTagKey() {
OSMRelation instance =
new OSMRelation(1L, 2, 1L, 1L, 1, new int[] {}, new OSMMember[] {part, part});
boolean expResult = false;
- boolean result = instance.hasTagKey(1);
+ boolean result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.WAY, 1);
instance = new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3},
new OSMMember[] {part, part});
expResult = true;
- result = instance.hasTagKey(1);
+ result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.WAY, 1);
instance = new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 2, 3, 3},
new OSMMember[] {part, part});
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.WAY, 1);
instance = new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3},
new OSMMember[] {part, part});
expResult = true;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.WAY, 1);
instance =
new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {2, 1, 3, 3}, new OSMMember[] {part, part});
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {1, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {1, 3});
assertEquals(expResult, result);
}
@@ -166,7 +167,7 @@ public void testHasTagValue() {
OSMRelation instance =
new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, new OSMMember[] {part, part});
boolean expResult = false;
- boolean result = instance.hasTagValue(1, 1);
+ boolean result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
}
@@ -176,19 +177,7 @@ public void testHasTagValue2() {
OSMRelation instance =
new OSMRelation(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, new OSMMember[] {part, part});
boolean expResult = true;
- boolean result = instance.hasTagValue(1, 1);
+ boolean result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
}
-
- @Test
- public void testToString() {
- OSMMember part = new OSMMember(1L, OSMType.WAY, 1);
- OSMRelation instance =
- new OSMRelation(1L, 2, 1L, 1L, 1, new int[] {1, 2}, new OSMMember[] {part, part});
- String expResult = "Relation-> ID:1 V:+2+ TS:1 CS:1 VIS:true UID:1 TAGS:[1, 2] "
- + "Mem:[T:WAY ID:1 R:1, T:WAY ID:1 R:1]";
- String result = instance.toString();
- assertEquals(expResult, result);
- }
-
}
diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java
index c394b065d..eb923643d 100644
--- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java
+++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/osm/OSMWayTest.java
@@ -4,6 +4,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.heigit.ohsome.oshdb.OSHDBTags;
import org.junit.Assert;
import org.junit.Test;
@@ -30,26 +31,6 @@ public void testGetRefs() {
assertArrayEquals(expResult, result);
}
- @Test
- public void testToString() {
- OSMMember part = new OSMMember(1L, OSMType.NODE, 1);
- OSMWay instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {}, new OSMMember[] {part, part});
- String expResult =
- "WAY-> ID:1 V:+1+ TS:1 CS:1 VIS:true UID:1 TAGS:[] Refs:[T:NODE ID:1 R:1, T:NODE ID:1 R:1]";
- String result = instance.toString();
- assertEquals(expResult, result);
-
- instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2}, new OSMMember[] {});
- expResult = "WAY-> ID:1 V:+1+ TS:1 CS:1 VIS:true UID:1 TAGS:[1, 1, 2, 2] Refs:[]";
- result = instance.toString();
- assertEquals(expResult, result);
-
- instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {}, null);
- expResult = "WAY-> ID:1 V:+1+ TS:1 CS:1 VIS:true UID:1 TAGS:[] Refs:null";
- result = instance.toString();
- assertEquals(expResult, result);
- }
-
@Test
public void testCompareTo() {
OSMMember part = new OSMMember(1L, OSMType.NODE, 1);
@@ -133,9 +114,9 @@ public void testisVisible() {
public void testGetTags() {
OSMMember part = new OSMMember(1L, OSMType.NODE, 1);
OSMWay instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 1}, new OSMMember[] {part, part});
- int[] expResult = new int[] {1, 1};
- int[] result = instance.getRawTags();
- Assert.assertArrayEquals(expResult, result);
+ var expResult = OSHDBTags.of(new int[] {1, 1});
+ var result = instance.getTags();
+ Assert.assertEquals(expResult, result);
}
@Test
@@ -143,34 +124,34 @@ public void testHasTagKey() {
OSMMember part = new OSMMember(1L, OSMType.NODE, 1);
OSMWay instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {}, new OSMMember[] {part, part});
boolean expResult = false;
- boolean result = instance.hasTagKey(1);
+ boolean result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.NODE, 1);
instance =
new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3}, new OSMMember[] {part, part});
expResult = true;
- result = instance.hasTagKey(1);
+ result = instance.getTags().hasTagKey(1);
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.NODE, 1);
instance =
new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 2, 3, 3}, new OSMMember[] {part, part});
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.NODE, 1);
instance =
new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 2, 3, 3}, new OSMMember[] {part, part});
expResult = true;
- result = instance.hasTagKeyExcluding(1, new int[] {2, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {2, 3});
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.NODE, 1);
instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {2, 1, 3, 3}, new OSMMember[] {part, part});
expResult = false;
- result = instance.hasTagKeyExcluding(1, new int[] {1, 3});
+ result = instance.getTags().hasTagKeyExcluding(1, new int[] {1, 3});
assertEquals(expResult, result);
}
@@ -180,13 +161,13 @@ public void testHasTagValue() {
OSMWay instance =
new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 2, 2, 3}, new OSMMember[] {part, part});
boolean expResult = false;
- boolean result = instance.hasTagValue(1, 1);
+ boolean result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
part = new OSMMember(1L, OSMType.NODE, 1);
instance = new OSMWay(1L, 1, 1L, 1L, 1, new int[] {1, 1, 2, 3}, new OSMMember[] {part, part});
expResult = true;
- result = instance.hasTagValue(1, 1);
+ result = instance.getTags().hasTagValue(1, 1);
assertEquals(expResult, result);
}
From da9d476365a613c93a43c3b43465e9568d2f5075 Mon Sep 17 00:00:00 2001
From: Rafael Troilo
Date: Tue, 19 Apr 2022 17:11:19 +0200
Subject: [PATCH 09/31] drop etl temporarly (#447)
* drop etl, move OSHDBToIgnite to api-ignite
---
CHANGELOG.md | 2 +
.../oshdb/api/ignite}/util/OSHDBToIgnite.java | 51 +-
oshdb-etl/README.md | 96 --
oshdb-etl/pom.xml | 106 --
.../oshdb/tool/importer/cli/CommonArgs.java | 21 -
.../tool/importer/cli/DistributableArgs.java | 18 -
.../cli/validator/DirExistValidator.java | 26 -
.../cli/validator/FileExistValidator.java | 18 -
.../cli/validator/PathExistValidator.java | 17 -
.../importer/cli/validator/TimeValidity.java | 16 -
.../oshdb/tool/importer/extract/Extract.java | 440 -------
.../importer/extract/TypeStartFinder.java | 221 ----
.../importer/extract/cli/ExtractArgs.java | 43 -
.../collector/KeyValueFrequencyCollector.java | 434 -------
.../extract/collector/RoleCollector.java | 232 ----
.../extract/collector/StatsCollector.java | 171 ---
.../extract/data/KeyValueFrequency.java | 18 -
.../importer/extract/data/OsmPbfMeta.java | 18 -
.../tool/importer/extract/data/Role.java | 40 -
.../importer/extract/data/ValueFrequency.java | 41 -
.../oshdb/tool/importer/load/Loader.java | 52 -
.../tool/importer/load/LoaderKeyTables.java | 86 --
.../oshdb/tool/importer/load/LoaderNode.java | 151 ---
.../tool/importer/load/LoaderRelation.java | 238 ----
.../oshdb/tool/importer/load/LoaderWay.java | 211 ----
.../oshdb/tool/importer/load/cli/DbH2Arg.java | 45 -
.../importer/load/handle/ConsoleHandler.java | 50 -
.../tool/importer/load/handle/H2Handler.java | 321 -----
.../importer/load/handle/LoaderHandler.java | 38 -
.../importer/load/handle/OSHDBHandler.java | 164 ---
.../tool/importer/transform/Transform.java | 220 ----
.../importer/transform/TransformReader.java | 193 ---
.../tool/importer/transform/Transformer.java | 281 -----
.../importer/transform/TransformerNode.java | 90 --
.../transform/TransformerRelation.java | 150 ---
.../transform/TransformerTagRoles.java | 196 ---
.../importer/transform/TransformerWay.java | 96 --
.../importer/transform/cli/TransformArgs.java | 34 -
.../cli/validator/TransformStepValidator.java | 25 -
.../transform/oshdb/MutableOSMEntity.java | 99 --
.../transform/oshdb/MutableOSMNode.java | 30 -
.../transform/oshdb/MutableOSMRelation.java | 16 -
.../transform/oshdb/MutableOSMWay.java | 15 -
.../tool/importer/transform/oshdb/OSH.java | 35 -
.../importer/transform/oshdb/OSHEntity.java | 23 -
.../importer/transform/oshdb/OSHEntity2.java | 191 ---
.../importer/transform/oshdb/OSHNode.java | 28 -
.../importer/transform/oshdb/OSHNode2.java | 101 --
.../transform/oshdb/OSHRelation2.java | 131 --
.../importer/transform/oshdb/OSHWay2.java | 106 --
.../importer/transform/oshdb/OSMEntity.java | 25 -
.../importer/transform/oshdb/OSMMember.java | 17 -
.../transform/oshdb/OSMMemberRelation.java | 47 -
.../transform/oshdb/OSMMemberWay.java | 39 -
.../transform/oshdb/OSMMemberWayIdOnly.java | 37 -
.../importer/transform/oshdb/OSMNode.java | 13 -
.../importer/transform/oshdb/OSMRelation.java | 14 -
.../tool/importer/transform/oshdb/OSMWay.java | 15 -
.../transform/oshdb/TransfomRelation.java | 123 --
.../transform/oshdb/TransformOSHEntity.java | 15 -
.../transform/oshdb/TransformOSHNode.java | 63 -
.../transform/oshdb/TransformOSHWay.java | 113 --
.../transform/reader/TransformReader.java | 118 --
.../reader/TransformRelationReaders.java | 93 --
.../transform/reader/TransformWayReaders.java | 92 --
.../reader/TransfromNodeReaders.java | 92 --
.../tool/importer/util/ExternalSort.java | 199 ---
.../tool/importer/util/MergeIterator.java | 73 --
.../tool/importer/util/PolyFileReader.java | 122 --
.../tool/importer/util/RoleToIdMapper.java | 8 -
.../importer/util/RoleToIdMapperImpl.java | 45 -
.../tool/importer/util/SizeEstimator.java | 74 --
.../tool/importer/util/StringToIdMapping.java | 6 -
.../importer/util/StringToIdMappingImpl.java | 96 --
.../oshdb/tool/importer/util/TagId.java | 21 -
.../tool/importer/util/TagToIdMapper.java | 13 -
.../tool/importer/util/TagToIdMapperImpl.java | 91 --
.../oshdb/tool/importer/util/ZGrid.java | 262 ----
.../importer/util/lambda/ConsumerUtil.java | 41 -
.../util/long2long/LongToLongMap.java | 7 -
.../util/long2long/SortedLong2LongMap.java | 221 ----
.../util/long2long/page/DensePage.java | 42 -
.../util/long2long/page/EmptyPage.java | 15 -
.../importer/util/long2long/page/Page.java | 7 -
.../util/long2long/page/PageLoader.java | 105 --
.../util/long2long/page/PageLocation.java | 13 -
.../util/long2long/page/SparsePage.java | 23 -
.../util/reactive/MyLambdaSubscriber.java | 113 --
.../load/handle/OSHDBHandlerTest.java | 122 --
.../importer/tests/TestPolyFileReader.java | 80 --
.../transform/oshdb/TransformOSHNodeTest.java | 37 -
.../test/resources/poly/australia-open.poly | 26 -
.../src/test/resources/poly/australia.poly | 27 -
oshdb-etl/src/test/resources/poly/russia.poly | 1119 -----------------
.../src/test/resources/poly/south-africa.poly | 645 ----------
pom.xml | 1 -
96 files changed, 9 insertions(+), 10006 deletions(-)
rename {oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer => oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/ignite}/util/OSHDBToIgnite.java (75%)
delete mode 100644 oshdb-etl/README.md
delete mode 100644 oshdb-etl/pom.xml
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/CommonArgs.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/DistributableArgs.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/DirExistValidator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/FileExistValidator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/PathExistValidator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/TimeValidity.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/Extract.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/TypeStartFinder.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/cli/ExtractArgs.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/KeyValueFrequencyCollector.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/RoleCollector.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/StatsCollector.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/KeyValueFrequency.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/OsmPbfMeta.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/Role.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/ValueFrequency.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/Loader.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderKeyTables.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/cli/DbH2Arg.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/ConsoleHandler.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/H2Handler.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/LoaderHandler.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/OSHDBHandler.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transform.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformReader.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transformer.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerTagRoles.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/TransformArgs.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/validator/TransformStepValidator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMEntity.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSH.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHEntity2.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHNode2.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHRelation2.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSHWay2.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMEntity.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMMember.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMMemberRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMMemberWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMMemberWayIdOnly.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSMWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/TransfomRelation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/TransformOSHEntity.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/TransformOSHNode.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/TransformOSHWay.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/reader/TransformReader.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/reader/TransformRelationReaders.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/reader/TransformWayReaders.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/reader/TransfromNodeReaders.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/ExternalSort.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/MergeIterator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/PolyFileReader.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/RoleToIdMapper.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/RoleToIdMapperImpl.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/SizeEstimator.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/StringToIdMapping.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/StringToIdMappingImpl.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/TagId.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/TagToIdMapper.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/TagToIdMapperImpl.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/ZGrid.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/lambda/ConsumerUtil.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/LongToLongMap.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/SortedLong2LongMap.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/DensePage.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/EmptyPage.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/Page.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/PageLoader.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/PageLocation.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/long2long/page/SparsePage.java
delete mode 100644 oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/reactive/MyLambdaSubscriber.java
delete mode 100644 oshdb-etl/src/test/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/OSHDBHandlerTest.java
delete mode 100644 oshdb-etl/src/test/java/org/heigit/ohsome/oshdb/tool/importer/tests/TestPolyFileReader.java
delete mode 100644 oshdb-etl/src/test/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/TransformOSHNodeTest.java
delete mode 100644 oshdb-etl/src/test/resources/poly/australia-open.poly
delete mode 100644 oshdb-etl/src/test/resources/poly/australia.poly
delete mode 100644 oshdb-etl/src/test/resources/poly/russia.poly
delete mode 100644 oshdb-etl/src/test/resources/poly/south-africa.poly
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 648d16f63..125608d27 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ Changelog
### breaking changes
* remove class `oshdb-util:util.time.TimestampFormatter` ([#419])
+* drop "old" etl module ([#447])
### bugfixes
@@ -23,6 +24,7 @@ Changelog
[#438]: https://github.com/GIScience/oshdb/pull/438
[#441]: https://github.com/GIScience/oshdb/pull/441
[#443]: https://github.com/GIScience/oshdb/pull/443
+[#447]: https://github.com/GIScience/oshdb/pull/447
## 0.7.2
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/OSHDBToIgnite.java b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/ignite/util/OSHDBToIgnite.java
similarity index 75%
rename from oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/OSHDBToIgnite.java
rename to oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/ignite/util/OSHDBToIgnite.java
index 6260eb6c4..2257b2fd1 100644
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/util/OSHDBToIgnite.java
+++ b/oshdb-api-ignite/src/main/java/org/heigit/ohsome/oshdb/api/ignite/util/OSHDBToIgnite.java
@@ -1,13 +1,9 @@
-package org.heigit.ohsome.oshdb.tool.importer.util;
+package org.heigit.ohsome.oshdb.api.ignite.util;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -30,7 +26,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Utility class for loading oshdb h2 file into ignite cluster.
+ *
+ */
public class OSHDBToIgnite {
+ private OSHDBToIgnite() {}
+
private static final Logger LOG = LoggerFactory.getLogger(OSHDBToIgnite.class);
@@ -122,43 +124,4 @@ private static void doGridImport(Ignite ignite, Statement stmt, TableNames c
}
}
}
-
- private static class Config {
- @Parameter(names = {"-ignite", "-igniteConfig", "-icfg"},
- description = "Path ot ignite-config.xml", required = true, order = 1)
- public File ignitexml;
-
- @Parameter(names = {"--prefix"}, description = "cache table prefix", required = false)
- public String prefix;
-
- @Parameter(names = {"-db", "-oshdb", "-outputDb"}, description = "Path to output H2",
- required = true, order = 2)
- public File oshdb;
-
- @Parameter(names = {"-help", "--help", "-h", "--h"}, help = true, order = 0)
- public boolean help = false;
- }
-
- public static void main(String[] args) throws SQLException, IgniteCheckedException {
- Config largs = new Config();
- JCommander jcom = JCommander.newBuilder().addObject(largs).build();
- try {
- jcom.parse(args);
- } catch (ParameterException e) {
- System.out.println("");
- LOG.error(e.getLocalizedMessage());
- System.out.println("");
- jcom.usage();
-
- return;
- }
-
- if (largs.help) {
- jcom.usage();
- return;
- }
- try (Connection con = DriverManager.getConnection("jdbc:h2:" + largs.oshdb, "sa", null)) {
- OSHDBToIgnite.load(largs.ignitexml, con, largs.prefix);
- }
- }
}
diff --git a/oshdb-etl/README.md b/oshdb-etl/README.md
deleted file mode 100644
index beb2163cc..000000000
--- a/oshdb-etl/README.md
+++ /dev/null
@@ -1,96 +0,0 @@
-# Setup a Local OSHDB
-
-In order to create a local database instance of the OSHDB yourself, you
-need an .osh.pbf-file of your area of interest. You can get one, e.g., from
-[Geofabrik](https://download.geofabrik.de/). The OSHDB instance may then
-be created by the steps extract, transform and load as follows:
-
-## Extract
-
-The first step is to extract data from your .osh.pbf-File. To do so,
-you have to figure out the valid time period contained in this file and
-to provide the start of this period in the ISO date-time format. The
-actual extraction is performed by running the following commands, assuming
-that you are in the base directory of the downloaded
-[OSHDB-Code](https://github.com/GIScience/oshdb).
-
-
-```bash
-cd oshdb-etl
-mvn compile
-mvn exec:java -Dexec.mainClass="org.heigit.ohsome.oshdb.tool.importer.extract.Extract" -Dexec.args="--pbf /absolute/path/to/file.osh.pbf -tmpDir ./tmpFiles --timevalidity_from YYYY-MM-DD"
-```
-
-This creates the files `extract_keys`, `extract_keyvalues`, and `extract_roles`
-containing the keys, values of tags, and roles of relations.
-
-For large files, you might have to increase the size of your JVM by executing
-`export MAVEN_OPTS="-Xmx???"` (replace ??? with a reasonable size for your machine)
-before extracting.
-
-Run `mvn exec:java -Dexec.mainClass="org.heigit.ohsome.oshdb.tool.importer.extract.Extract"`
-to get help and more options.
-
-
-## Transform
-
-After extraction, a transformation step creates the actual OSHDB using
-the H2 database engine:
-
-```bash
-mvn exec:java -Dexec.mainClass="org.heigit.ohsome.oshdb.tool.importer.transform.Transform" -Dexec.args="--pbf /absolute/path/to/file.osh.pbf -tmpDir ./tmpFiles"
-```
-The transformation step is computation-intensive, so be easy on your computer
-and do not use too large files.
-
-
-## Load
-
-### Load into an H2 Database
-
-The transformed data has to be loaded into a database to which the OSHDB will interface.
-In order to enable the OSHDB to provide a proper attribution of the imported data, you
-have to set an attribution text and an attribution url
-
-```bash
-mvn exec:java -Dexec.mainClass="org.heigit.ohsome.oshdb.tool.importer.load.handle.OSHDB2H2Handler" -Dexec.args="-tmpDir ./tmpFiles --out /absolute/path/to/your-H2-database --attribution '© OpenStreetMap contributors' --attribution-url 'https://www.openstreetmap.org/copyright'"
-```
-
-You now have a ready-to-use OSHDB named **your-H2-database.mv.db** in the specified
-output directory (the file extension .mv.db is appended automatically).
-
-
-### Deploy on Apache Ignite (optional)
-
-If you wish to deploy the OSHDB on [Apache ignite](https://ignite.apache.org),
-the previously created H2 database has to be loaded into ignite. You can
-[download ignite](https://ignite.apache.org/download.cgi#binaries) from the
-Apache website.
-
-1. In order to deploy the OSHDB dependencies to ignite,
- change back to the base directory and let maven copy all dependencies to
- your ignite installation. It assumes this is at /opt/ignite. If it is not,
- you may place a link to the actual location there.
- ```bash
- cd ../..
- mvn -Pdeployignite clean install
- ```
-
-2. Then start as many ignite servers in a pseudo-distributed system as you
- think your computer can handle using an appropriate ignite-config.xml.
- Ignite's default configuration file is located at
- `/opt/ignite/config/default-config.xml`.
- You either have to use a terminal for each ignite server or nohup:
- ```bash
- nohup /opt/ignite/bin/ignite.sh ignite-config.xml &>/opt/ignite/log.log &
- /opt/ignite/bin/ignite.sh ignite-config.xml
- ```
-
-3. Finally, you can import the data as follows using
- the same ignite-config as in the previous step (note that the file extension .mv.db
- has to be omitted):
- ```bash
- cd oshdb-etl
- mvn exec:java -Dexec.mainClass="org.heigit.ohsome.oshdb.tool.importer.util.OSHDB2Ignite" -Dexec.args="-ignite ignite-config.xml -db /absolute/path/to/your-H2-database"
- ```
-
diff --git a/oshdb-etl/pom.xml b/oshdb-etl/pom.xml
deleted file mode 100644
index 83b9e82e5..000000000
--- a/oshdb-etl/pom.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-
-
- 4.0.0
-
-
- org.heigit.ohsome
- oshdb-parent
- 0.8.0-SNAPSHOT
-
-
- oshdb-etl
- jar
- Extract, Transform and Load your data into an JDBC or Ignite Database.
-
-
- 0.9.17
- 8.1.1
- 1.72
-
-
-
-
- ${project.groupId}
- oshdb
- ${project.version}
-
-
-
- ${project.groupId}
- oshdb-util
- ${project.version}
-
-
-
- ${project.groupId}
- oshdb-oshpbf-parser
- ${project.version}
-
-
-
- io.reactivex.rxjava2
- rxjava
- ${rxjava2.version}
-
-
-
- com.google.guava
- guava
- ${guava.version}
-
-
-
- org.roaringbitmap
- RoaringBitmap
- ${roaringbitmap.version}
-
-
-
- org.wololo
- jts2geojson
- ${wololo.version}
-
-
-
- com.beust
- jcommander
- ${jcommander.version}
-
-
-
- it.unimi.dsi
- fastutil
- ${fastutil.version}
-
-
-
- com.h2database
- h2
- ${h2.version}
-
-
-
- org.apache.ignite
- ignite-core
- ${ignite.version}
-
-
-
- org.apache.ignite
- ignite-spring
- ${ignite.version}
-
-
-
- org.slf4j
- slf4j-simple
- ${slf4j.version}
-
-
-
- junit
- junit
- test
-
-
-
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/CommonArgs.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/CommonArgs.java
deleted file mode 100644
index 7d8b9891a..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/CommonArgs.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli;
-
-import com.beust.jcommander.Parameter;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import org.heigit.ohsome.oshdb.tool.importer.cli.validator.DirExistValidator;
-
-public class CommonArgs {
- @Parameter(names = {"-workDir", "--workingDir"}, description = "path to store the result files.",
- validateWith = DirExistValidator.class, required = false, order = 10)
- public Path workDir = Paths.get(".");
-
- @Parameter(names = {"-tmpDir", "--temporayDirectory"},
- description = "path to store temporary files", validateWith = DirExistValidator.class,
- required = false, order = 11)
- public Path tempDir;
-
- @Parameter(names = {"-help", "--help", "-h", "--h"}, description = "prints this help",
- help = true, order = 99)
- public boolean help = false;
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/DistributableArgs.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/DistributableArgs.java
deleted file mode 100644
index e7366c4a1..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/DistributableArgs.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.validators.PositiveInteger;
-
-public class DistributableArgs {
- @Parameter(names = {"--worker"}, description = "number of this worker (beginning with 0).",
- validateWith = PositiveInteger.class, required = false, order = 20)
- public int worker = 0;
-
- @Parameter(names = {"--totalWorker"}, description = "total number of workers.",
- validateWith = PositiveInteger.class, required = false, order = 21)
- public int totalWorkers = 1;
-
- @Parameter(names = {"--merge"}, description = "merge output of workers together",
- required = false, order = 22)
- public boolean merge;
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/DirExistValidator.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/DirExistValidator.java
deleted file mode 100644
index 940a9c8ec..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/DirExistValidator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli.validator;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.ParameterException;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-public class DirExistValidator implements IParameterValidator {
-
- @Override
- public void validate(String name, String value) throws ParameterException {
- final Path path = Paths.get(value);
- try {
- Files.createDirectories(path);
- } catch (IOException e) {
- throw new ParameterException(
- "Directory " + value + " for parameter " + name + " could not created");
- }
- if (!Files.exists(path) && Files.isDirectory(path)) {
- throw new ParameterException(
- "Directory " + value + " for parameter " + name + " does not exist!");
- }
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/FileExistValidator.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/FileExistValidator.java
deleted file mode 100644
index 6e2a91257..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/FileExistValidator.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli.validator;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.ParameterException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-public class FileExistValidator implements IParameterValidator {
-
- @Override
- public void validate(String name, String value) throws ParameterException {
- final Path path = Paths.get(value);
- if (!Files.exists(path) || !Files.isRegularFile(path)) {
- throw new ParameterException("File " + value + " for parameter " + name + " does not exist!");
- }
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/PathExistValidator.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/PathExistValidator.java
deleted file mode 100644
index c540bff81..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/PathExistValidator.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli.validator;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.ParameterException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-public class PathExistValidator implements IParameterValidator {
-
- @Override
- public void validate(String name, String value) throws ParameterException {
-
- if (!Files.exists(Paths.get(value))) {
- throw new ParameterException("File " + value + " for parameter " + name + " does not exist!");
- }
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/TimeValidity.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/TimeValidity.java
deleted file mode 100644
index 9ca41e937..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/cli/validator/TimeValidity.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.cli.validator;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.ParameterException;
-import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser;
-
-public class TimeValidity implements IParameterValidator {
- @Override
- public void validate(String name, String value) throws ParameterException {
- try {
- IsoDateTimeParser.parseIsoDateTime(value);
- } catch (Exception e) {
- throw new ParameterException(e.getMessage());
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/Extract.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/Extract.java
deleted file mode 100644
index 7b1600181..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/Extract.java
+++ /dev/null
@@ -1,440 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator.estimatedSizeOf;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.ParameterException;
-import com.google.common.base.Functions;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Streams;
-import com.google.common.io.CountingOutputStream;
-import io.reactivex.Flowable;
-import java.io.BufferedOutputStream;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.ParseException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.tool.importer.extract.cli.ExtractArgs;
-import org.heigit.ohsome.oshdb.tool.importer.extract.collector.KeyValueFrequencyCollector;
-import org.heigit.ohsome.oshdb.tool.importer.extract.collector.RoleCollector;
-import org.heigit.ohsome.oshdb.tool.importer.extract.collector.StatsCollector;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.OsmPbfMeta;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.Role;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.ValueFrequency;
-import org.heigit.ohsome.oshdb.tool.importer.util.ExternalSort;
-import org.heigit.ohsome.oshdb.tool.importer.util.PolyFileReader;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Relation;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.RelationMember;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Tag;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.TagText;
-import org.heigit.ohsome.oshpbf.parser.rx.Osh;
-import org.heigit.ohsome.oshpbf.parser.rx.RxOshPbfReader;
-import org.wololo.geojson.GeoJSON;
-
-public class Extract {
-
- public static class ExtractKeyTablesResult {
- public final KeyValueFrequencyCollector kvFrequency;
- public final RoleCollector roleFrequency;
-
- public ExtractKeyTablesResult(KeyValueFrequencyCollector kvFrequency,
- RoleCollector roleFrequency) {
- this.kvFrequency = kvFrequency;
- this.roleFrequency = roleFrequency;
- }
- }
-
- public static class KeyValuePointer {
- public final String key;
- public final int freq;
- public final int valuesNumber;
- public final long valuesOffset;
-
- private KeyValuePointer(String key, int freq, int valuesNumber, long valuesOffset) {
- this.key = key;
- this.freq = freq;
- this.valuesNumber = valuesNumber;
- this.valuesOffset = valuesOffset;
- }
-
- /**
- * Write to output.
- */
- public void write(DataOutput out) throws IOException {
- out.writeUTF(key);
- out.writeInt(freq);
- out.writeInt(valuesNumber);
- out.writeLong(valuesOffset);
- }
-
- /**
- * Read from input.
- */
- public static KeyValuePointer read(DataInput in) throws IOException {
- final String key = in.readUTF();
- final int freq = in.readInt();
- final int valuesNumber = in.readInt();
- final long valuesOffset = in.readLong();
- return new KeyValuePointer(key, freq, valuesNumber, valuesOffset);
- }
-
- /**
- * EstimateSize.
- */
- public long estimateSize() {
- final long size = estimatedSizeOf("") + estimatedSizeOf(key) // value
- + 4 // freq
- + 4 // valueNumber
- + 8 // valuesOffset
- ;
- return size;
- }
- }
-
- private final long maxMemory;
- private Path workDirectory = Paths.get(".");
- private Path tempDirectory = Paths.get(".");
-
- public static OsmPbfMeta pbfMetaData(Path pbf) throws Exception {
- return TypeStartFinder.getMetaData(pbf);
- }
-
- private Extract(long maxMemory) {
- this.maxMemory = maxMemory;
- }
-
- public static Extract withMaxMemory(long availableMemory) {
- return new Extract(availableMemory);
- }
-
- public Extract withWorkDirectory(Path workDirectory) {
- this.workDirectory = workDirectory;
- return this;
- }
-
- public Extract withTempDirectory(Path tempDirectory) {
- this.tempDirectory = tempDirectory;
- return this;
- }
-
- /**
- * Extract.
- */
- public ExtractKeyTablesResult extract(ExtractArgs config, int workerId, int workerTotal,
- boolean keepTemp) throws IOException, ParseException {
- final Path pbf = config.pbf;
- final StatsCollector stats = new StatsCollector(pbf);
-
- final KeyValueFrequencyCollector kvFrequency = new KeyValueFrequencyCollector();
- kvFrequency.setWorkerId(workerId);
- kvFrequency.setTempDir(workerTotal > 1 ? workDirectory.toFile() : tempDirectory.toFile());
- kvFrequency.setTempDeleteOneExit(workerTotal > 1 || keepTemp);
-
- final RoleCollector roleFrequency = new RoleCollector();
- roleFrequency.setWorkerId(workerId);
- roleFrequency.setTempDir(workerTotal > 1 ? workDirectory.toFile() : tempDirectory.toFile());
- roleFrequency.setTempDeleteOneExit(workerTotal > 1 || keepTemp);
-
- final long fileLength = pbf.toFile().length();
- final long workSize = (long) Math.ceil(fileLength / (double) workerTotal);
- final long start = workSize * workerId;
- final long softEnd = Math.min(start + workSize, fileLength);
-
- Flowable oshFlow = RxOshPbfReader.readOsh(pbf, start, softEnd, -1, stats::addHeader);
-
- oshFlow = oshFlow.doOnNext(osh -> {
- stats.add(osh);
- });
-
- oshFlow = oshFlow.doOnNext(osh -> {
- if (kvFrequency.getEstimatedSize() + roleFrequency.getEstimatedSize() > maxMemory) {
- kvFrequency.writeTemp();
- roleFrequency.writeTemp();
- }
-
- final Set uniqueTags = new HashSet<>();
- osh.getVersions().forEach(version -> {
- for (Tag tag : version.getTags()) {
- uniqueTags.add((TagText) tag);
- }
- });
- kvFrequency.addAll(uniqueTags);
- });
-
- oshFlow = oshFlow.doOnNext(osh -> {
- if (kvFrequency.getEstimatedSize() + roleFrequency.getEstimatedSize() > maxMemory) {
- kvFrequency.writeTemp();
- roleFrequency.writeTemp();
- }
- if (osh.getType() != OSMType.RELATION) {
- return;
- }
- final Set uniqueRoles = new HashSet<>();
-
- osh.getVersions().forEach(version -> {
- Relation r = (Relation) version;
- for (RelationMember member : r.members) {
- uniqueRoles.add(member.role);
- }
- });
- roleFrequency.addAll(uniqueRoles);
- });
-
- oshFlow.count().blockingGet();
-
- try (
- PrintStream out = new PrintStream(
- workDirectory.resolve("extract_meta").toFile(), UTF_8)) {
- stats.print(out);
-
- if (!config.md5.trim().isEmpty()) {
- out.println("file.md5=" + config.md5);
- }
-
- if (config.polyFile != null) {
- GeoJSON json = PolyFileReader.parse(config.polyFile);
- out.println("extract.region=" + json.toString());
- } else if (config.bbox != null) {
- out.println("extract.region={\"bbox\":[" + config.bbox + "]}");
- }
-
- out.print("extract.timerange=" + config.timeValidityFrom);
- if (config.timeValidityTo != null) {
- out.println("," + config.timeValidityTo);
- } else if (stats.header.hasOsmosisReplicationTimestamp()
- && stats.header.getOsmosisReplicationTimestamp() > 0) {
- out.println("," + ZonedDateTime
- .ofInstant(Instant.ofEpochSecond(stats.header.getOsmosisReplicationTimestamp()),
- ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
- } else {
- out.println(
- "," + ZonedDateTime.ofInstant(Instant.ofEpochSecond(stats.maxTs), ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
- }
- }
-
- ExtractKeyTablesResult result = new ExtractKeyTablesResult(kvFrequency, roleFrequency);
- return result;
- }
-
- public void sortByFrequency(ExtractKeyTablesResult extratKeyTablesResult)
- throws IOException {
- System.out.print("sorting tags by frequency ...");
- sortByFrequency(extratKeyTablesResult.kvFrequency);
- System.out.println(" done!");
- System.out.print("sorting roles by frequency ...");
- sortByFrequency(extratKeyTablesResult.roleFrequency);
- System.out.println(" done!");
- }
-
- public void sortByFrequency(KeyValueFrequencyCollector kvFrequency)
- throws IOException {
- final long maxSize = maxMemory;
- final ExternalSort valueSorter = ExternalSort.of((a, b) -> {
- final int c = Integer.compare(a.freq, b.freq);
- if (c != 0) {
- return c * -1; // reverse order
- }
- return a.value.compareTo(b.value);
- }, maxSize, ValueFrequency::estimateSize).with(ValueFrequency::write, ValueFrequency::read);
-
- final Function output = Functions.identity();
- final List keys = new ArrayList<>();
- try (
- FileOutputStream fout =
- new FileOutputStream(workDirectory.resolve("extract_keyvalues").toFile());
- BufferedOutputStream bout = new BufferedOutputStream(fout);
- CountingOutputStream keyValuesPositionOutput = new CountingOutputStream(output.apply(bout));
- DataOutputStream keyValuesDataOutput = new DataOutputStream(keyValuesPositionOutput)) {
- kvFrequency.forEach(kvf -> {
- try {
- long offset = keyValuesPositionOutput.getCount();
- int values = (int) Streams.stream(valueSorter.sort(kvf.vfIterator)).peek(vf -> {
- try {
- vf.write(keyValuesDataOutput);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }).count();
-
- KeyValuePointer kvp = new KeyValuePointer(kvf.key, kvf.freq, values, offset);
- keys.add(kvp);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- }
-
- try (FileOutputStream fs = new FileOutputStream(workDirectory.resolve("extract_keys").toFile());
- BufferedOutputStream bs = new BufferedOutputStream(fs);
- OutputStream os = output.apply(fs);
- DataOutputStream keyValuesDataOutput = new DataOutputStream(os)) {
- keyValuesDataOutput.writeInt(keys.size());
- var sort = ExternalSort.of((a, b) -> {
- final int c = Integer.compare(a.freq, b.freq);
- if (c != 0) {
- return c * -1; // reverse order
- }
- return a.key.compareTo(b.key);
- }, maxSize, KeyValuePointer::estimateSize)
- .with(KeyValuePointer::write, KeyValuePointer::read)
- .sort(keys.iterator());
- while (sort.hasNext()) {
- var kvp = sort.next();
- kvp.write(keyValuesDataOutput);
- }
- }
- }
-
- public void sortByFrequency(RoleCollector roleFrequency)
- throws IOException {
- final long maxSize = maxMemory;
- final Function output = Functions.identity();
- try (
- FileOutputStream fout =
- new FileOutputStream(workDirectory.resolve("extract_roles").toFile());
- BufferedOutputStream bout = new BufferedOutputStream(fout);
- OutputStream os = output.apply(bout);
- DataOutputStream rolesDataOutput = new DataOutputStream(os)) {
- var sort = ExternalSort.of((a, b) -> {
- final int c = Integer.compare(a.freq, b.freq);
- if (c != 0) {
- return c * -1; // reverse order
- }
- return a.role.compareTo(b.role);
- }, maxSize, Role::estimateSize).with(Role::write, Role::read).sort(roleFrequency.iterator());
- while (sort.hasNext()) {
- var r = sort.next();
- r.write(rolesDataOutput);
- }
- }
- }
-
- public static void execute(ExtractArgs config) throws IOException, ParseException {
- Path workDir = config.common.workDir;
- Path tempDir = config.common.tempDir;
-
- if (workDir == null) {
- workDir = Paths.get(".");
- }
-
- if (tempDir == null) {
- tempDir = workDir;
- }
-
- int worker = config.distribute.worker;
- int workerTotal = config.distribute.totalWorkers;
- if (worker >= workerTotal) {
- throw new IllegalArgumentException("worker must be lesser than totalWorker!");
- }
-
- long availableMemory = SizeEstimator.estimateAvailableMemory();
- System.out.print("extracting key tables ...");
- Extract extract = Extract.withMaxMemory(availableMemory).withWorkDirectory(workDir)
- .withTempDirectory(tempDir);
- if (config.distribute.merge) {
- try {
- List tmp;
- try (DirectoryStream stream =
- Files.newDirectoryStream(workDir, KeyValueFrequencyCollector.tempPrefix + "_*")) {
- tmp = StreamSupport.stream(stream.spliterator(), false).map(Path::toFile)
- .collect(Collectors.toList());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- if (tmp.isEmpty()) {
- throw new RuntimeException("no files to merge");
- }
- KeyValueFrequencyCollector kvFrequency = new KeyValueFrequencyCollector(tmp);
- System.out.print("sorting tags by frequency ...");
- extract.sortByFrequency(kvFrequency);
- System.out.println(" done!");
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- try {
- List tmp;
- try (DirectoryStream stream =
- Files.newDirectoryStream(workDir, RoleCollector.tempPrefix + "_*")) {
- tmp = StreamSupport.stream(stream.spliterator(), false).map(Path::toFile)
- .collect(Collectors.toList());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- if (tmp.isEmpty()) {
- throw new RuntimeException("no files to merge");
- }
- RoleCollector roleFrequency = new RoleCollector(tmp);
- System.out.print("sorting roles by frequency ...");
- extract.sortByFrequency(roleFrequency);
- System.out.println(" done!");
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- if (config.overwrite || !Files.exists(workDir.resolve("extract_keys"))
- || !Files.exists(workDir.resolve("extract_keyvalues"))
- || !Files.exists(workDir.resolve("extract_roles"))) {
- ExtractKeyTablesResult result = extract.extract(config, worker, workerTotal, false);
-
- try {
- if (workerTotal > 1) {
- result.kvFrequency.writeTemp();
- result.roleFrequency.writeTemp();
- return;
- }
-
- extract.sortByFrequency(result);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public static void main(String[] args) throws IOException, ParseException {
- ExtractArgs config = new ExtractArgs();
- JCommander jcom = JCommander.newBuilder().addObject(config).build();
-
- try {
- jcom.parse(args);
- } catch (ParameterException e) {
- System.out.println("");
- System.out.println(e.getLocalizedMessage());
- System.out.println("");
- jcom.usage();
- return;
- }
- if (config.common.help) {
- jcom.usage();
- return;
- }
- Stopwatch stopwatch = Stopwatch.createStarted();
- execute(config);
- System.out.println("extract done in " + stopwatch);
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/TypeStartFinder.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/TypeStartFinder.java
deleted file mode 100644
index 1d21a7431..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/TypeStartFinder.java
+++ /dev/null
@@ -1,221 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import crosby.binary.Osmformat;
-import crosby.binary.Osmformat.PrimitiveBlock;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Iterator;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.OsmPbfMeta;
-import org.heigit.ohsome.oshpbf.parser.pbf.PbfBlob;
-import org.heigit.ohsome.oshpbf.parser.rx.RxOshPbfReader;
-
-public class TypeStartFinder {
-
- /**
- * Reads in the metadata.
- *
- * @param pbf Path to the pbf
- * @throws IOException if pbf could not be read
- */
- public static OsmPbfMeta getMetaData(Path pbf) throws IOException {
- OsmPbfMeta meta = new OsmPbfMeta();
- meta.pbf = pbf;
- Path metaPath = pbf.getParent().resolve(pbf.getFileName().toString() + ".meta");
-
- if (Files.exists(metaPath)) {
- try (DataInputStream input = new DataInputStream(new FileInputStream(metaPath.toFile()))) {
- meta.pbf = pbf;
- meta.nodeStart = input.readLong();
- meta.nodeEnd = input.readLong();
- meta.wayStart = input.readLong();
- meta.wayEnd = input.readLong();
- meta.relationStart = input.readLong();
- meta.relationEnd = input.readLong();
- return meta;
- }
- }
-
- long fileSize = pbf.toFile().length();
-
- long nodeStort = fileSize;
- long wayStart = fileSize;
- long relStart = fileSize;
- long count = 0;
- for (PbfBlob blob : RxOshPbfReader.readBlob(pbf, 0, fileSize, -1).filter(PbfBlob::isData)
- .limit(100).blockingIterable()) {
- switch (getType(blob)) {
- case NODE:
- nodeStort = Math.min(nodeStort, blob.pos);
- break;
- case WAY:
- wayStart = Math.min(wayStart, blob.pos);
- break;
- case RELATION:
- relStart = Math.min(relStart, blob.pos);
- break;
- default:
- throw new IllegalStateException("unkown type");
- }
- count++;
- }
-
- if (count < 100) {
- meta.nodeStart = nodeStort;
- meta.nodeEnd = wayStart;
- meta.wayStart = wayStart;
- meta.wayEnd = relStart;
- meta.relationStart = relStart;
- meta.relationEnd = fileSize;
-
- } else {
- if (wayStart == fileSize) {
- PbfBlob way = findWay(pbf);
- wayStart = way != null ? way.pos : -1;
- }
- if (relStart == fileSize) {
- PbfBlob relation = findRelation(pbf, wayStart + 1);
- relStart = relation != null ? relation.pos : -1;
- }
-
- meta.nodeStart = nodeStort;
- meta.nodeEnd = wayStart;
- meta.wayStart = wayStart;
- meta.wayEnd = relStart;
- meta.relationStart = relStart;
- meta.relationEnd = fileSize;
- }
-
- try (DataOutputStream output = new DataOutputStream(new FileOutputStream(metaPath.toFile()))) {
- output.writeLong(meta.nodeStart);
- output.writeLong(meta.nodeEnd);
- output.writeLong(meta.wayStart);
- output.writeLong(meta.wayEnd);
- output.writeLong(meta.relationStart);
- output.writeLong(meta.relationEnd);
- }
- return meta;
- }
-
- /**
- * Seek for first PBF Way Blob.
- *
- * @param pbf Path to Pbf
- */
- public static PbfBlob findWay(Path pbf) throws InvalidProtocolBufferException {
- long fileSize = pbf.toFile().length();
-
- long low = 0;
- long high = fileSize;
-
- while (high >= low) {
- long middle = (low + high) / 2;
-
- Iterator blob =
- RxOshPbfReader.readBlob(pbf, middle, -1, -1).take(2).blockingIterable().iterator();
-
- if (blob.hasNext()) {
- PbfBlob b = blob.next();
- OSMType type = getType(b);
-
- if (type == OSMType.NODE) {
- if (!blob.hasNext()) {
- System.out.println("Found nothing");
- return null;
- }
- b = blob.next();
- type = getType(b);
- if (type == OSMType.WAY) {
- System.out.println("Found Way at " + b.pos);
- return b;
- } else if (type == OSMType.NODE) {
- low = middle + 1;
- }
- } else {
- high = middle + 1;
- }
- } else {
- System.out.println("Found nothing");
- return null;
- }
- }
- return null;
- }
-
- /**
- * Seeks to the first Pbf Relation Blob.
- *
- * @param pbf Path to Pbf
- * @param startPos Offset to begin.
- */
- public static PbfBlob findRelation(Path pbf, long startPos)
- throws InvalidProtocolBufferException {
- long fileSize = pbf.toFile().length();
-
- long low = startPos;
- long high = fileSize;
-
- while (high >= low) {
- long middle = (low + high) / 2;
-
- Iterator blob =
- RxOshPbfReader.readBlob(pbf, middle, -1, -1).take(2).blockingIterable().iterator();
-
- if (blob.hasNext()) {
- PbfBlob b = blob.next();
- OSMType type = getType(b);
-
- if (type == OSMType.WAY) {
- if (!blob.hasNext()) {
- System.out.println("Found nothing");
- return null;
- }
- b = blob.next();
- type = getType(b);
- if (type == OSMType.RELATION) {
- System.out.println("Found Relation at " + b.pos);
- return b;
- } else if (type == OSMType.WAY) {
- low = middle + 1;
- }
- } else {
- high = middle + 1;
- }
- } else {
- System.out.println("Found nothing");
- return null;
- }
- }
- return null;
- }
-
- /**
- * Checks the of given PbfBlob.
- *
- * @param blob Blob.
- */
- public static OSMType getType(PbfBlob blob) throws InvalidProtocolBufferException {
- PrimitiveBlock block = blob.getPrimitivBlock();
- if (block != null) {
- Osmformat.PrimitiveGroup group = block.getPrimitivegroup(0);
- if (group.hasDense() || group.getNodesCount() > 0) {
- return OSMType.NODE;
- }
- if (group.getWaysCount() > 0) {
- return OSMType.WAY;
- }
- if (group.getRelationsCount() > 0) {
- return OSMType.RELATION;
- }
- throw new IllegalArgumentException("unkown type for Pbf PrimitiveGroup!");
- }
- throw new IllegalArgumentException(
- "block is null, could be a header pbf block instead of a data block!");
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/cli/ExtractArgs.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/cli/ExtractArgs.java
deleted file mode 100644
index 89cf41703..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/cli/ExtractArgs.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.cli;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import java.nio.file.Path;
-import org.heigit.ohsome.oshdb.tool.importer.cli.CommonArgs;
-import org.heigit.ohsome.oshdb.tool.importer.cli.DistributableArgs;
-import org.heigit.ohsome.oshdb.tool.importer.cli.validator.FileExistValidator;
-import org.heigit.ohsome.oshdb.tool.importer.cli.validator.TimeValidity;
-
-public class ExtractArgs {
- @ParametersDelegate
- public CommonArgs common = new CommonArgs();
-
- @ParametersDelegate
- public DistributableArgs distribute = new DistributableArgs();
-
- @Parameter(names = {"--pbf"}, description = "path to pbf-File to import",
- validateWith = FileExistValidator.class, required = true, order = 0)
- public Path pbf;
-
- @Parameter(names = {"--md5"}, description = "MD5 checksum")
- public String md5 = "";
-
- @Parameter(names = {"--poly"}, description = "extract region",
- validateWith = FileExistValidator.class)
- public Path polyFile;
-
- @Parameter(names = {"--bbox"}, description = "extract_region")
- public String bbox = "";
-
- @Parameter(names = {"--timevalidity_from"},
- description = "first valid timestamp in isodate format", required = true,
- validateWith = TimeValidity.class)
- public String timeValidityFrom; // cc-by-sa 2007-10-07
-
- @Parameter(names = {"--timevalidity_to"},
- description = "latest valid timestamp in isodate format", validateWith = TimeValidity.class)
- public String timeValidityTo = null;
-
- @Parameter(names = {"--overwrite"}, description = "overwrite existing files", order = 1)
- public boolean overwrite = false;
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/KeyValueFrequencyCollector.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/KeyValueFrequencyCollector.java
deleted file mode 100644
index 29fa3a9a3..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/KeyValueFrequencyCollector.java
+++ /dev/null
@@ -1,434 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.collector;
-
-import static org.heigit.ohsome.oshdb.tool.importer.util.lambda.ConsumerUtil.throwingConsumer;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-import com.google.common.collect.Streams;
-import it.unimi.dsi.fastutil.objects.Object2IntAVLTreeMap;
-import it.unimi.dsi.fastutil.objects.Object2IntMap.Entry;
-import it.unimi.dsi.fastutil.objects.Object2ObjectAVLTreeMap;
-import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.PriorityQueue;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.KeyValueFrequency;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.ValueFrequency;
-import org.heigit.ohsome.oshdb.tool.importer.util.MergeIterator;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.TagText;
-
-public class KeyValueFrequencyCollector implements Iterable {
-
- private Function outputStreamFunction = (out) -> out;
- private Function inputStreamFunction = (in) -> in;
-
- private final Object2IntAVLTreeMap key2Frequency = new Object2IntAVLTreeMap<>();
- private final Object2ObjectAVLTreeMap> key2Values =
- new Object2ObjectAVLTreeMap<>();
-
- private long estimatedSize = 0;
- private final List splits;
-
- public static final String tempPrefix = "temp_keyvaluefrequency_";
- private String tempSuffix = "_00.tmp";
- private File tempDir = null;
- private boolean tempDeleteOnExit = true;
-
- public KeyValueFrequencyCollector() {
- this(new ArrayList<>());
- }
-
- public KeyValueFrequencyCollector(List files) {
- this.splits = files;
- }
-
- public void setWorkerId(int workerId) {
- tempSuffix = String.format("_%02d.tmp", workerId);
- }
-
- public void setTempDir(File dir) {
- dir.mkdirs();
- if (dir.exists()) {
- this.tempDir = dir;
- }
- }
-
- public void setTempDeleteOneExit(boolean deleteOnExit) {
- this.tempDeleteOnExit = deleteOnExit;
- }
-
- public void addAll(Collection item) {
- item.forEach(t -> {
- final String key = t.key;
- final String value = t.value;
-
- if (key2Frequency.addTo(key, 1) == 0) {
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryKey(key);
- }
-
- Object2IntAVLTreeMap value2Frequency = key2Values.get(key);
- if (value2Frequency == null) {
- value2Frequency = new Object2IntAVLTreeMap<>();
- key2Values.put(key, value2Frequency);
- }
- if (value2Frequency.addTo(value, 1) == 0) {
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue(value);
- }
- });
- }
-
- public void inputOutputStream(Function input,
- Function output) {
- this.inputStreamFunction = input;
- this.outputStreamFunction = output;
- }
-
- public void writeTemp() throws IOException {
- if (key2Frequency.isEmpty()) {
- return;
- }
- File newTempFile;
- newTempFile = File.createTempFile(tempPrefix, tempSuffix, tempDir);
- if (tempDeleteOnExit) {
- newTempFile.deleteOnExit();
- }
- try (OutputStream out = new FileOutputStream(newTempFile)) {
- writeTemp(out);
- }
- splits.add(newTempFile);
- }
-
- public void writeTemp(OutputStream outStream) throws IOException {
- try (OutputStream outStream2 = outputStreamFunction.apply(outStream);
- DataOutputStream out = new DataOutputStream(new BufferedOutputStream(outStream2))) {
- out.writeInt(key2Frequency.size());
- key2Frequency.object2IntEntrySet().forEach(throwingConsumer(keyFrequency -> {
- final String key = keyFrequency.getKey();
- final int keyFreq = keyFrequency.getIntValue();
- writeKeyValueFrequency(out, key, keyFreq, key2Values.get(key));
- }));
- key2Frequency.clear();
- key2Values.clear();
- estimatedSize = 0;
- }
- }
-
- private static void writeKeyValueFrequency(DataOutputStream out, String key, int keyFreq,
- Object2IntAVLTreeMap value) throws IOException {
- out.writeUTF(key);
- out.writeInt(keyFreq);
- out.writeInt(value.size());
- value.object2IntEntrySet().forEach(throwingConsumer(valueFrequency -> {
- out.writeUTF(valueFrequency.getKey());
- out.writeInt(valueFrequency.getIntValue());
- }));
- }
-
- public long getEstimatedSize() {
- return estimatedSize;
- }
-
- public List getSplits() {
- return splits;
- }
-
- @Override
- public Iterator iterator() {
- List> iters = new ArrayList<>(splits.size() + key2Frequency.size());
- splits.stream().map(file -> {
- DataInputStream dataInput = null;
- try {
- InputStream input = inputStreamFunction.apply(new FileInputStream(file));
- dataInput = new DataInputStream(new BufferedInputStream(input));
- return KeyValueFrequencyFileReader.of(dataInput);
- } catch (IOException e) {
- if (dataInput != null) {
- try {
- dataInput.close();
- } catch (Exception e2) {
- e.addSuppressed(e2);
- }
- }
- throw new UncheckedIOException(e);
- }
- }).forEach(iters::add);
- if (key2Frequency.size() > 0) {
- iters.add(KeyValueFrequencyMapReader.of(key2Frequency, key2Values, true));
- }
-
- return MergeIterator.of(iters, (a, b) -> a.key.compareTo(b.key), list -> {
- final List> values = new ArrayList<>(list.size());
-
- final String key = list.get(0).key;
- final int freq = list.stream().mapToInt(it -> {
- values.add(it.vfIterator);
- return it.freq;
- }).sum();
-
- Iterator valueItr =
- MergeIterator.of(values, (a, b) -> a.value.compareTo(b.value), valueList -> {
- final String value = valueList.get(0).value;
- final int valueFreq = valueList.stream().mapToInt(ValueFrequency::freq).sum();
- return new ValueFrequency(value, valueFreq);
- });
- return new KeyValueFrequency(key, freq, valueItr);
- });
- }
-
- public Stream stream() {
- return Streams.stream(this);
- }
-
- public static class ValueFrequencyIterator implements Iterator {
- private final PriorityQueue> queue;
- private final List> peekingIters;
- private final Comparator comparator = (a, b) -> a.value.compareTo(b.value);
-
- public static ValueFrequencyIterator of(List> iters) {
- final List> peekingIters =
- iters.stream().map(itr -> Iterators.peekingIterator(itr)).collect(Collectors.toList());
- return new ValueFrequencyIterator(peekingIters);
- }
-
- private ValueFrequencyIterator(List> peekingIters) {
- this.peekingIters = peekingIters;
- queue = new PriorityQueue<>(peekingIters.size(),
- (a, b) -> comparator.compare(a.peek(), b.peek()));
- }
-
- @Override
- public boolean hasNext() {
- return !queue.isEmpty() || !peekingIters.isEmpty();
- }
-
- @Override
- public ValueFrequency next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- queue.addAll(peekingIters);
- peekingIters.clear();
-
- ValueFrequency vf = poll();
-
- final String value = vf.value;
- int freq = vf.freq;
-
- while (!queue.isEmpty() && comparator.compare(vf, queue.peek().peek()) == 0) {
- vf = poll();
- freq += vf.freq;
- }
-
- return new ValueFrequency(value, freq);
- }
-
- private ValueFrequency poll() {
- final PeekingIterator iter = queue.poll();
- final ValueFrequency ret = iter.next();
- if (iter.hasNext()) {
- peekingIters.add(iter);
- }
- return ret;
- }
-
- }
-
- public static class KeyValueFrequencyMapReader implements Iterator {
-
- private final ObjectBidirectionalIterator> keyIterator;
- private final Object2ObjectAVLTreeMap> key2Values;
- private final boolean remove;
- private Map lastValues = Collections.emptyMap();
-
- public static KeyValueFrequencyMapReader of(Object2IntAVLTreeMap key2Frequency,
- Object2ObjectAVLTreeMap> key2Values, boolean remove) {
- final ObjectBidirectionalIterator> keyIterator =
- key2Frequency.object2IntEntrySet().iterator();
- return new KeyValueFrequencyMapReader(keyIterator, key2Values, remove);
- }
-
- private KeyValueFrequencyMapReader(ObjectBidirectionalIterator> keyIterator,
- Object2ObjectAVLTreeMap> key2Values, boolean remove) {
- this.keyIterator = keyIterator;
- this.key2Values = key2Values;
- this.remove = remove;
- }
-
- @Override
- public boolean hasNext() {
- return keyIterator.hasNext();
- }
-
- @Override
- public KeyValueFrequency next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- final Entry entry = keyIterator.next();
- final String key = entry.getKey();
- final int freq = entry.getIntValue();
-
- Object2IntAVLTreeMap values = key2Values.get(key);
- if (remove) {
- lastValues.clear();
- keyIterator.remove();
- key2Values.remove(key);
- lastValues = values;
- }
-
- return new KeyValueFrequency(key, freq, ValueFrequencyMapReader.of(values, remove));
- }
-
- }
-
- public static class ValueFrequencyMapReader implements Iterator {
- final ObjectBidirectionalIterator> valueIterator;
- final boolean remove;
-
- public static ValueFrequencyMapReader of(Object2IntAVLTreeMap values, boolean remove) {
- ObjectBidirectionalIterator> valueIterator =
- values.object2IntEntrySet().iterator();
- return new ValueFrequencyMapReader(valueIterator, remove);
- }
-
- private ValueFrequencyMapReader(ObjectBidirectionalIterator> valueIterator,
- boolean remove) {
- this.valueIterator = valueIterator;
- this.remove = remove;
- }
-
- @Override
- public boolean hasNext() {
- return valueIterator.hasNext();
- }
-
- @Override
- public ValueFrequency next() {
- final Entry entry = valueIterator.next();
- final String value = entry.getKey();
- final int freq = entry.getIntValue();
- if (remove) {
- valueIterator.remove();
- }
- return new ValueFrequency(value, freq);
- }
-
- }
-
- public static class KeyValueFrequencyFileReader
- implements Closeable, Iterator {
- private final DataInputStream input;
- private final int keys;
- private int index = 0;
- private Iterator values = Collections.emptyIterator();
-
- public static KeyValueFrequencyFileReader of(DataInputStream input) throws IOException {
- final int keys = input.readInt();
- return new KeyValueFrequencyFileReader(input, keys);
- }
-
- private KeyValueFrequencyFileReader(DataInputStream input, int keys) {
- this.input = input;
- this.keys = keys;
- }
-
- @Override
- public void close() throws IOException {
- input.close();
- }
-
- @Override
- public boolean hasNext() {
- return index < keys;
- }
-
- @Override
- public KeyValueFrequency next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- try {
- // skip all unread values
- while (values.hasNext()) {
- values.next();
- }
-
- final String key = input.readUTF();
- final int freq = input.readInt();
- values = ValueFrequencyFileReader.of(input);
-
- index++;
-
- return new KeyValueFrequency(key, freq, values);
-
- } catch (IOException e) {
- index = Integer.MAX_VALUE;
- throw new NoSuchElementException(e.getMessage());
- }
- }
-
- }
-
- public static class ValueFrequencyFileReader implements Iterator {
- private final DataInputStream input;
- private final int values;
- private int index = 0;
-
- public static ValueFrequencyFileReader of(DataInputStream input) throws IOException {
- final int values = input.readInt();
- return new ValueFrequencyFileReader(input, values);
- }
-
- private ValueFrequencyFileReader(DataInputStream input, int values) {
- this.input = input;
- this.values = values;
- }
-
- @Override
- public boolean hasNext() {
- return index < values;
- }
-
- @Override
- public ValueFrequency next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- try {
- final String value = input.readUTF();
- final int freq = input.readInt();
- index++;
- return new ValueFrequency(value, freq);
- } catch (IOException e) {
- index = Integer.MAX_VALUE;
- throw new NoSuchElementException(e.getMessage());
- }
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/RoleCollector.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/RoleCollector.java
deleted file mode 100644
index 419fab112..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/RoleCollector.java
+++ /dev/null
@@ -1,232 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.collector;
-
-import static org.heigit.ohsome.oshdb.tool.importer.util.lambda.ConsumerUtil.throwingConsumer;
-
-import it.unimi.dsi.fastutil.objects.Object2IntAVLTreeMap;
-import it.unimi.dsi.fastutil.objects.Object2IntMap.Entry;
-import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.function.Function;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.Role;
-import org.heigit.ohsome.oshdb.tool.importer.util.MergeIterator;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-
-public class RoleCollector implements Iterable {
-
- private final List tmpFiles;
- private Function outputStreamFunction = (out) -> out;
- private Function inputStreamFunction = (in) -> in;
-
- private final Object2IntAVLTreeMap role2Frequency = new Object2IntAVLTreeMap<>();
-
- private long estimatedSize = 0;
-
- public static final String tempPrefix = "temp_rolefrequency_";
- private String tempSuffix = "_00.tmp";
- private File tempDir = null;
- private boolean tempDeleteOnExit = true;
-
- public RoleCollector() {
- this(new ArrayList<>());
- }
-
- public void setWorkerId(int workerId) {
- tempSuffix = String.format("_%02d.tmp", workerId);
- }
-
- public RoleCollector(List tmpFiles) {
- this.tmpFiles = tmpFiles;
- }
-
- public void setTempDir(File dir) {
- dir.mkdirs();
- if (dir.exists()) {
- this.tempDir = dir;
- }
- }
-
- public void setTempDeleteOneExit(boolean deleteOnExit) {
- this.tempDeleteOnExit = deleteOnExit;
- }
-
- public long getEstimatedSize() {
- return estimatedSize;
- }
-
- public void inputOutputStream(Function input,
- Function output) {
- this.inputStreamFunction = input;
- this.outputStreamFunction = output;
- }
-
- public void addAll(Collection roles) {
- roles.forEach(role -> {
- if (role2Frequency.addTo(role, 1) == 0) {
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue(role);
- }
- });
- }
-
- public void writeTemp(OutputStream outStream) throws IOException {
- try (OutputStream outStream2 = outputStreamFunction.apply(outStream);
- DataOutputStream out = new DataOutputStream(new BufferedOutputStream(outStream2))) {
- out.writeInt(role2Frequency.size());
- role2Frequency.object2IntEntrySet().forEach(throwingConsumer(roleFrequency -> {
- out.writeUTF(roleFrequency.getKey());
- out.writeInt(roleFrequency.getIntValue());
- }));
-
- role2Frequency.clear();
- estimatedSize = 0;
- }
- }
-
- public void writeTemp() throws IOException {
- if (role2Frequency.isEmpty()) {
- return;
- }
- File newTempFile;
- newTempFile = File.createTempFile(tempPrefix, tempSuffix, tempDir);
- if (tempDeleteOnExit) {
- newTempFile.deleteOnExit();
- }
- try (OutputStream out = new FileOutputStream(newTempFile)) {
- writeTemp(out);
- }
- tmpFiles.add(newTempFile);
- }
-
- public static class RoleFileReader implements Iterator {
- private final DataInputStream input;
- private final int roles;
- private int index = 0;
-
- public static RoleFileReader of(DataInputStream input) throws IOException {
- final int roles = input.readInt();
- return new RoleFileReader(input, roles);
- }
-
- private RoleFileReader(DataInputStream input, int roles) {
- this.input = input;
- this.roles = roles;
- }
-
- @Override
- public boolean hasNext() {
- return index < roles;
- }
-
- @Override
- public Role next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- try {
- final String role = input.readUTF();
- final int freq = input.readInt();
- index++;
-
- return new Role(role, freq);
-
- } catch (IOException e) {
- index = Integer.MAX_VALUE;
- throw new NoSuchElementException(e.getMessage());
- }
- }
- }
-
- public static class RoleMapReader implements Iterator {
-
- private final ObjectBidirectionalIterator> roleIterator;
-
- public static RoleMapReader of(Object2IntAVLTreeMap role2Frequency) {
- final ObjectBidirectionalIterator> roleIterator =
- role2Frequency.object2IntEntrySet().iterator();
- return new RoleMapReader(roleIterator);
- }
-
- private RoleMapReader(ObjectBidirectionalIterator> roleIterator) {
- this.roleIterator = roleIterator;
- }
-
- @Override
- public boolean hasNext() {
- return roleIterator.hasNext();
- }
-
- @Override
- public Role next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- final Entry entry = roleIterator.next();
- final String role = entry.getKey();
- final int freq = entry.getIntValue();
-
- return new Role(role, freq);
- }
- }
-
- @Override
- public Iterator iterator() {
- List> iters = new ArrayList<>(tmpFiles.size() + role2Frequency.size());
- tmpFiles.stream().map(file -> {
- DataInputStream dataInput = null;
- try {
- InputStream input = inputStreamFunction.apply(new FileInputStream(file));
- dataInput = new DataInputStream(new BufferedInputStream(input));
- return RoleFileReader.of(dataInput);
- } catch (IOException e) {
- if (dataInput != null) {
- try {
- dataInput.close();
- } catch (Exception e2) {
- e.addSuppressed(e2);
- }
- }
- throw new UncheckedIOException(e);
- }
- }).forEach(iters::add);
- if (role2Frequency.size() > 0) {
- iters.add(RoleMapReader.of(role2Frequency));
- }
-
- return MergeIterator.of(iters, (a, b) -> a.role.compareTo(b.role), list -> {
- final String role = list.get(0).role;
- final int freq = list.stream().mapToInt(it -> {
- return it.freq;
- }).sum();
-
- return new Role(role, freq);
- });
-
- }
-
- protected Role read(DataInputStream in) throws IOException {
- final String key = in.readUTF();
- final int frequency = in.readInt();
- return new Role(key, frequency);
- }
-
- protected void write(DataOutputStream out, Role role) throws IOException {
- out.writeUTF(role.role);
- out.writeInt(role.freq);
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/StatsCollector.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/StatsCollector.java
deleted file mode 100644
index e071510ba..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/collector/StatsCollector.java
+++ /dev/null
@@ -1,171 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.collector;
-
-import static org.heigit.ohsome.oshdb.osm.OSMCoordinates.GEOM_PRECISION;
-
-import crosby.binary.Osmformat.HeaderBBox;
-import crosby.binary.Osmformat.HeaderBlock;
-import java.io.PrintStream;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Arrays;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Node;
-import org.heigit.ohsome.oshpbf.parser.rx.Osh;
-
-public class StatsCollector {
-
- private long[] nodes = new long[5];
- private long[] ways = new long[5];
- private long[] relations = new long[5];
- private long[] unknown = new long[5];
-
- private long minLon = Long.MAX_VALUE;
- private long minLat = Long.MAX_VALUE;
- private long maxLon = Long.MIN_VALUE;
- private long maxLat = Long.MIN_VALUE;
-
- private long minTs = Long.MAX_VALUE;
- public long maxTs = Long.MIN_VALUE;
-
- public HeaderBlock header = null;
- private Path pbf = null;
-
- public StatsCollector(Path pbf) {
- this.pbf = pbf;
- }
-
- /**
- * Adds Header Block.
- *
- * @param header HeaderBlock
- */
- public void addHeader(HeaderBlock header) {
- this.header = header;
-
- nodes[3] = Long.MAX_VALUE;
- nodes[4] = Long.MIN_VALUE;
- ways[3] = Long.MAX_VALUE;
- ways[4] = Long.MIN_VALUE;
- relations[3] = Long.MAX_VALUE;
- relations[4] = Long.MIN_VALUE;
- unknown[3] = Long.MAX_VALUE;
- unknown[4] = Long.MIN_VALUE;
-
- }
-
- /**
- * Add OSH Entity.
- *
- * @param osh Entity
- */
- public void add(Osh osh) {
- OSMType type = osh.getType();
-
- long[] entity;
- switch (type) {
- case NODE:
- entity = nodes;
- break;
- case WAY:
- entity = ways;
- break;
- case RELATION:
- entity = relations;
- break;
- default:
- entity = unknown;
- }
-
- int highesVersion = Integer.MIN_VALUE;
- boolean alive = false;
-
- for (Entity e : osh.getVersions()) {
-
- minTs = Math.min(minTs, e.getTimestamp());
- maxTs = Math.max(maxTs, e.getTimestamp());
-
- if (e.getVersion() > highesVersion) {
- highesVersion = e.getVersion();
- alive = e.isVisible();
- }
-
- if (!e.isVisible()) {
- continue;
- }
-
- if (type == OSMType.NODE) {
- Node n = (Node) e;
- minLon = Math.min(minLon, n.getLongitude());
- minLat = Math.min(minLat, n.getLatitude());
- maxLon = Math.max(maxLon, n.getLongitude());
- maxLat = Math.max(maxLat, n.getLatitude());
- }
-
- }
-
- entity[0]++;
- entity[1] += osh.getVersions().size();
- if (alive) {
- entity[2]++;
- }
-
- entity[3] = Math.min(entity[3], osh.getId());
- entity[4] = Math.max(entity[4], osh.getId());
- }
-
- /**
- * Prints the collected Stats to given PrintStream.
- *
- * @param out PrintStream
- */
- public void print(PrintStream out) {
- if (pbf != null) {
- out.println("file.name=" + pbf.getFileName());
- }
- if (header != null) {
- HeaderBBox bbox = header.getBbox();
- out.printf("header.bbox=%d,%d,%d,%d%n", bbox.getLeft(), bbox.getBottom(), bbox.getRight(),
- bbox.getTop());
- out.println("header.source=" + header.getSource());
- out.println("header.generator=" + header.getWritingprogram());
- out.println("header.osmosis_replication_base_url=" + header.getOsmosisReplicationBaseUrl());
- out.println("header.osmosis_replication_sequence_number="
- + header.getOsmosisReplicationSequenceNumber());
- out.println(
- "header.osmosis_replication_timestamp=" + header.getOsmosisReplicationTimestamp());
- }
-
- if (minLon != Integer.MAX_VALUE) {
- out.printf("data.bbox=%.7f,%.7f,%.7f,%.7f%n", minLon * GEOM_PRECISION,
- minLat * GEOM_PRECISION, maxLon * GEOM_PRECISION,
- maxLat * GEOM_PRECISION);
- }
-
- out.printf("data.timerange=%s,%s%n",
- ZonedDateTime.ofInstant(Instant.ofEpochSecond(minTs), ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
- ZonedDateTime.ofInstant(Instant.ofEpochSecond(maxTs), ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
-
- out.println("data.nodes="
- + Arrays.stream(nodes, 1, nodes.length).collect(() -> new StringBuilder("" + nodes[0]),
- (sb, i) -> sb.append(",").append(i), (a, b) -> a.append(b)).toString());
- out.println("data.ways="
- + Arrays.stream(ways, 1, ways.length).collect(() -> new StringBuilder("" + ways[0]),
- (sb, i) -> sb.append(",").append(i), (a, b) -> a.append(b)).toString());
- out.println("data.relations=" + Arrays.stream(relations, 1, relations.length)
- .collect(() -> new StringBuilder("" + relations[0]), (sb, i) -> sb.append(",").append(i),
- (a, b) -> a.append(b))
- .toString());
- if (unknown[0] > 0) {
- out.println("data.unknown=" + Arrays.stream(unknown, 1, unknown.length)
- .collect(() -> new StringBuilder("" + unknown[0]), (sb, i) -> sb.append(",").append(i),
- (a, b) -> a.append(b))
- .toString());
- }
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/KeyValueFrequency.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/KeyValueFrequency.java
deleted file mode 100644
index c126fce92..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/KeyValueFrequency.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.data;
-
-import java.util.Iterator;
-
-public class KeyValueFrequency {
-
- public final String key;
- public final int freq;
-
- public final Iterator vfIterator;
-
- public KeyValueFrequency(String key, int freq, Iterator vfIterator) {
- this.key = key;
- this.freq = freq;
- this.vfIterator = vfIterator;
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/OsmPbfMeta.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/OsmPbfMeta.java
deleted file mode 100644
index fa3b0cafd..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/OsmPbfMeta.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.data;
-
-import java.nio.file.Path;
-
-public class OsmPbfMeta {
-
- public Path pbf;
-
- public long nodeStart;
- public long nodeEnd;
-
- public long wayStart;
- public long wayEnd;
-
- public long relationStart;
- public long relationEnd;
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/Role.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/Role.java
deleted file mode 100644
index 4c117c582..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/Role.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.data;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Comparator;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-
-public class Role {
- public final String role;
- public final int freq;
-
- public Role(String role, int frequency) {
- this.role = role;
- this.freq = frequency;
- }
-
- public long estimateSize() {
- return SizeEstimator.estimatedSizeOf(role) + 4;
- }
-
- public static final Comparator comparatorByFrequency = (a, b) -> {
- int c = Integer.compare(a.freq, b.freq);
- if (c != 0) {
- return c * -1;
- }
- return a.role.compareTo(b.role);
- };
-
- public void write(DataOutput out) throws IOException {
- out.writeUTF(role);
- out.writeInt(freq);
- }
-
- public static Role read(DataInput in) throws IOException {
- final String role = in.readUTF();
- final int freq = in.readInt();
- return new Role(role, freq);
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/ValueFrequency.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/ValueFrequency.java
deleted file mode 100644
index 12721af7b..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/extract/data/ValueFrequency.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.extract.data;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-
-public class ValueFrequency {
- public final String value;
- public final int freq;
-
- public ValueFrequency(String value, int freq) {
- this.value = value;
- this.freq = freq;
- }
-
- public int freq() {
- return freq;
- }
-
- public void write(DataOutput out) throws IOException {
- out.writeUTF(value);
- out.writeInt(freq);
- }
-
- public static ValueFrequency read(DataInput in) throws IOException {
- final String value = in.readUTF();
- final int freq = in.readInt();
- return new ValueFrequency(value, freq);
- }
-
- @Override
- public String toString() {
- return String.format("(%s:%d)", value, freq);
- }
-
- public long estimateSize() {
- final long size = SizeEstimator.estimatedSizeOf("") + SizeEstimator.estimatedSizeOf(value) + 4;
- return size;
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/Loader.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/Loader.java
deleted file mode 100644
index 398195400..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/Loader.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load;
-
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Supplier;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-
-public abstract class Loader implements Closeable {
-
- protected final int minEntitiesPerCell;
-
- protected List loaders = new ArrayList<>(2);
-
- public Loader(int minEntitiesPerCell) {
- this.minEntitiesPerCell = minEntitiesPerCell;
- }
-
- public void load() {
- load(Long.MAX_VALUE, true);
- }
-
- public void load(long cellId) {
- load(cellId, false);
- }
-
- public abstract void load(long cellId2, boolean all);
-
- public void addLoader(Loader loader) {
- this.loaders.add(loader);
- }
-
- public void visitNode(TransformOSHNode node) {
-
- }
-
- public void visitWay(TransformOSHWay way) {
-
- }
-
- public int initZoomLevel(int maxZoom, int zoom, List zoomLevel, Supplier add) {
- if (maxZoom < zoom) {
- for (int i = maxZoom; i < zoom; i++) {
- zoomLevel.add(add.get());
- }
- maxZoom = zoom;
- }
- return maxZoom;
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderKeyTables.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderKeyTables.java
deleted file mode 100644
index 0f6401142..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderKeyTables.java
+++ /dev/null
@@ -1,86 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load;
-
-import com.google.common.base.Functions;
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.RandomAccessFile;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Function;
-import org.heigit.ohsome.oshdb.tool.importer.extract.Extract.KeyValuePointer;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.Role;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.ValueFrequency;
-
-public class LoaderKeyTables {
-
- public interface Handler {
- void loadKeyValues(int id, String key, List values);
-
- void loadRole(int id, String role);
- }
-
- Path workDirectory;
- private Handler handler;
-
- public LoaderKeyTables(Path workdirectory, Handler handler) {
- this.workDirectory = workdirectory;
- this.handler = handler;
- }
-
- public void load() throws IOException {
- loadTags();
- }
-
- public void loadTags() throws IOException {
- final Function input = Functions.identity();
-
- try (
- DataInputStream keyIn = new DataInputStream(input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_keys").toFile()))));
- RandomAccessFile raf =
- new RandomAccessFile(workDirectory.resolve("extract_keyvalues").toFile(), "r");
- FileChannel valuesChannel = raf.getChannel();) {
-
- final int length = keyIn.readInt();
- for (int i = 0; i < length; i++) {
- final KeyValuePointer kvp = KeyValuePointer.read(keyIn);
-
- final String key = kvp.key;
- List values = Collections.emptyList();
-
- values = new ArrayList<>(kvp.valuesNumber);
-
- valuesChannel.position(kvp.valuesOffset);
- try (DataInputStream valueStream =
- new DataInputStream(Channels.newInputStream(valuesChannel));) {
- for (int j = 0; j < kvp.valuesNumber; j++) {
- final ValueFrequency vf = ValueFrequency.read(valueStream);
- values.add(vf.value);
- }
- }
- handler.loadKeyValues(i, key, values);
- }
- }
- }
-
- public void loadRoles() throws IOException {
- final Function input = Functions.identity();
- try (DataInputStream roleIn = new DataInputStream(input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_roles").toFile()))))) {
- for (int id = 0; true; id++) {
- final Role role = Role.read(roleIn);
- handler.loadRole(id, role.role);
- }
- } catch (EOFException e) {
- return; // ignore end of file exception
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
deleted file mode 100644
index 659cb3126..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderNode.java
+++ /dev/null
@@ -1,151 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.reader.TransfromNodeReaders;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-
-public class LoaderNode extends Loader {
-
- public interface Handler {
- void handleNodeGrid(long cellId, Collection nodes);
- }
-
- private static class Grid {
- long cellId = -1;
- List entities;
-
- public void clear() {
- entities = null;
- }
- }
-
- final List zoomLevel = new ArrayList<>(20);
- int maxZoom = -1;
- final int maxZoomLevel;
-
- private final TransfromNodeReaders reader;
- private final Handler handler;
- private final boolean onlyNodesWithTags;
-
- public LoaderNode(Path workDirectory, Handler handler, int minEntitiesPerCell,
- boolean onlyNodesWithTags, int maxZoomLevel) throws IOException {
- super(minEntitiesPerCell);
- Path[] files;
- try (DirectoryStream stream =
- Files.newDirectoryStream(workDirectory, "transform_node_*")) {
- files = StreamSupport.stream(stream.spliterator(), false).collect(Collectors.toList())
- .toArray(new Path[0]);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
- reader = new TransfromNodeReaders(files);
- this.handler = handler;
- this.onlyNodesWithTags = onlyNodesWithTags;
- this.maxZoomLevel = Math.max(1, maxZoomLevel);
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- public final Long2ObjectMap invalidNodes = new Long2ObjectAVLTreeMap<>();
-
- @Override
- public void load(long cellId2, boolean all) {
- if (!reader.hasNext()) {
- return;
- }
-
- if (reader.getCellId() == -1) {
- Set set = reader.next();
- set.stream().forEach(node -> invalidNodes.put(node.getId(), node));
- handler.handleNodeGrid(-1, invalidNodes.values());
- }
-
- while (reader.hasNext()
- && (all || ZGrid.ORDER_DFS_TOP_DOWN.compare(reader.getCellId(), cellId2) <= 0)) {
- final long cellId = reader.getCellId();
- final int zoom = ZGrid.getZoom(cellId);
- final Set nodes = reader.next();
-
- nodes.forEach(node -> {
- for (Loader bla : loaders) {
- bla.visitNode(node);
- }
- });
-
- initZoomLevel(zoom);
-
- store(zoom);
-
- Grid grid = zoomLevel.get(zoom);
- grid.cellId = cellId;
-
- if (onlyNodesWithTags) {
- grid.entities = nodes.stream().filter(osh -> {
- return osh.stream().anyMatch(osm -> osm.getTags().size() > 0);
- }).collect(Collectors.toList());
- } else {
- grid.entities = new ArrayList<>(nodes);
- }
-
- }
- if (!reader.hasNext()) {
- store(0);
- }
- }
-
- private void store(int zoom) {
- for (int i = maxZoom; i >= zoom; i--) {
- Grid grid = zoomLevel.get(i);
- if (grid == null || grid.entities == null) {
- continue;
- }
-
- if (i > maxZoomLevel || grid.entities.size() < minEntitiesPerCell && i > 0) {
- Grid parent = zoomLevel.get(i - 1);
- if (parent.entities == null) {
- parent.cellId = ZGrid.getParent(grid.cellId);
- parent.entities = grid.entities;
- } else {
- parent.entities.addAll(grid.entities);
- }
- grid.clear();
- continue;
- }
- handler.handleNodeGrid(grid.cellId, grid.entities);
- grid.clear();
- }
- }
-
- protected Grid getParentInZoomHierachie(int zoom) {
- if (zoom > 0) {
- return zoomLevel.get(zoom - 1);
- }
- return null;
- }
-
- protected void initZoomLevel(int zoom) {
- if (maxZoom < zoom) {
- for (int i = maxZoom; i < zoom; i++) {
- zoomLevel.add(new Grid());
- }
- maxZoom = zoom;
- }
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderRelation.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderRelation.java
deleted file mode 100644
index 20ee958a0..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderRelation.java
+++ /dev/null
@@ -1,238 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load;
-
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransfomRelation;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.transform.reader.TransformRelationReaders;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-
-public class LoaderRelation extends Loader {
-
- public interface Handler {
- void handleRelationGrid(long cellId, Collection entities,
- Collection nodes, Collection ways);
- }
-
- private static class Grid {
- long cellId;
- List entities = null;
- Set nodesSet = Collections.emptySet();
- List nodeforGrid = null;
-
- Set waysSet = Collections.emptySet();
- List wayforGrid = null;
-
- public void clear() {
- entities = null;
- nodesSet = Collections.emptySet();
- nodeforGrid = null;
- waysSet = Collections.emptySet();
- wayforGrid = null;
- }
- }
-
- final List zoomLevel = new ArrayList<>(20);
- int currentZoom;
- int maxZoom = -1;
- final int maxZoomLevel;
- int lastZoom = -1;
-
- Set nodesForCellSet = new HashSet<>();
- List nodesForGrid;
- Set waysForCellSet = new HashSet<>();
- List waysForGrid;
-
- final TransformRelationReaders reader;
- final Handler handler;
- final LoaderNode nodeLoader;
- final LoaderWay wayLoader;
-
- public LoaderRelation(Path workDirectory, Handler handler, int minEntitiesPerCell,
- LoaderNode nodeLoader, LoaderWay wayLoader, int maxZoomLevel) throws IOException {
- super(minEntitiesPerCell);
- Path[] files;
- try (DirectoryStream stream =
- Files.newDirectoryStream(workDirectory, "transform_relation_*")) {
- files = StreamSupport.stream(stream.spliterator(), false).collect(Collectors.toList())
- .toArray(new Path[0]);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- this.reader = new TransformRelationReaders(files);
- this.handler = handler;
- this.nodeLoader = nodeLoader;
- nodeLoader.addLoader(this);
- this.wayLoader = wayLoader;
- wayLoader.addLoader(this);
- this.maxZoomLevel = Math.max(1, maxZoomLevel);
-
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- @Override
- public void load(long cellId2, boolean all) {
- if (!reader.hasNext()) {
- return;
- }
-
- if (reader.getCellId() == -1) {
- Set set = reader.next();
- handler.handleRelationGrid(-1, set, Collections.emptyList(), Collections.emptyList());
- }
-
- while (reader.hasNext()
- && (all || ZGrid.ORDER_DFS_TOP_DOWN.compare(reader.getCellId(), cellId2) <= 0)) {
- final long cellId = reader.getCellId();
- final int zoom = ZGrid.getZoom(cellId);
- currentZoom = zoom;
-
- final Set entities = reader.next();
-
- maxZoom = initZoomLevel(maxZoom, zoom, zoomLevel, () -> new Grid());
-
- nodesForCellSet = new HashSet<>();
- waysForCellSet = new HashSet<>();
-
- entities.forEach(e -> {
- for (Long id : e.getNodeIds()) {
- nodesForCellSet.add(id);
- }
- for (Long id : e.getWayIds()) {
- waysForCellSet.add(id);
- }
- });
-
- nodesForGrid = new ArrayList<>(nodesForCellSet.size());
- waysForGrid = new ArrayList<>(waysForCellSet.size());
-
- wayLoader.load(cellId, false);
-
- store(zoom);
-
- Grid grid = zoomLevel.get(zoom);
- grid.cellId = cellId;
- grid.entities = new ArrayList<>(entities);
- grid.nodeforGrid = nodesForGrid;
- grid.nodesSet = nodesForCellSet;
- grid.wayforGrid = waysForGrid;
- grid.waysSet = waysForCellSet;
-
- lastZoom = zoom;
- }
-
- if (!reader.hasNext()) {
- wayLoader.load(0, true);
- store(0);
- }
-
- }
-
- private void store(int zoom) {
- for (int i = lastZoom; i >= zoom; i--) {
- Grid grid = zoomLevel.get(i);
- if (grid == null || grid.entities == null) {
- continue;
- }
-
- if (i > maxZoomLevel || grid.entities.size() < minEntitiesPerCell && i > 0) {
- Grid parent = zoomLevel.get(i - 1);
-
- if (parent.entities == null) {
- parent.cellId = ZGrid.getParent(grid.cellId);
- parent.entities = grid.entities;
- parent.nodeforGrid = grid.nodeforGrid;
- parent.nodesSet = grid.nodesSet;
- parent.wayforGrid = grid.wayforGrid;
- parent.waysSet = grid.waysSet;
- } else {
- parent.entities.addAll(grid.entities);
-
- parent.nodeforGrid.addAll(grid.nodeforGrid);
- for (TransformOSHNode osh : grid.nodeforGrid) {
- parent.nodesSet.remove(osh.getId());
- }
- parent.nodesSet.addAll(grid.nodesSet);
-
- parent.wayforGrid.addAll(grid.wayforGrid);
- for (TransformOSHWay osh : grid.wayforGrid) {
- parent.waysSet.remove(osh.getId());
- }
- }
-
- grid.clear();
- continue;
- }
-
- // some nodes could still be left, maybe they are invalid!
- // TODO or should we ignore them?
- grid.nodesSet.forEach(id -> {
- TransformOSHNode e = nodeLoader.invalidNodes.get(id.longValue());
- if (e != null) {
- grid.nodeforGrid.add(e);
- }
- });
-
- grid.waysSet.forEach(id -> {
- TransformOSHWay e = wayLoader.invalids.get(id.longValue());
- if (e != null) {
- grid.wayforGrid.add(e);
- }
- });
-
- handler.handleRelationGrid(grid.cellId, grid.entities, grid.nodeforGrid, grid.wayforGrid);
- grid.clear();
- }
- }
-
- @Override
- public void visitNode(TransformOSHNode osh) {
- final Long id = osh.getId();
- if (nodesForCellSet.contains(id)) {
- nodesForGrid.add(osh);
- nodesForCellSet.remove(id);
- } else {
- for (int i = lastZoom; i >= 0; i--) {
- final Grid g = zoomLevel.get(i);
- if (g.nodesSet.contains(id)) {
- g.nodeforGrid.add(osh);
- g.nodesSet.remove(id);
- break;
- }
- }
- }
- }
-
- @Override
- public void visitWay(TransformOSHWay osh) {
- final Long id = osh.getId();
- if (waysForCellSet.contains(id)) {
- waysForGrid.add(osh);
- waysForCellSet.remove(id);
- } else {
- for (int i = lastZoom; i >= 0; i--) {
- final Grid g = zoomLevel.get(i);
- if (g.waysSet.contains(id)) {
- g.wayforGrid.add(osh);
- g.waysSet.remove(id);
- break;
- }
- }
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderWay.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderWay.java
deleted file mode 100644
index 11c157bfc..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/LoaderWay.java
+++ /dev/null
@@ -1,211 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.transform.reader.TransformWayReaders;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-
-public class LoaderWay extends Loader {
-
- public interface Handler {
- void handleWayGrid(long cellId, Collection ways,
- Collection nodes);
- }
-
- private static class Grid {
- long cellId = -2;
- List entities = null;
- Set nodesSet = Collections.emptySet();
- List forGrid = null;
-
- public void clear() {
- entities = null;
- forGrid = null;
- nodesSet = Collections.emptySet();
- }
- }
-
- final List zoomLevel = new ArrayList<>(20);
- int currentZoom;
- int maxZoom = -1;
- final int maxZoomLevel;
- int lastZoom = -1;
-
- final TransformWayReaders reader;
- final Handler handler;
-
- final LoaderNode nodeLoader;
-
- public final Long2ObjectMap invalids = new Long2ObjectAVLTreeMap<>();
-
- Set nodesForCellSet = new HashSet<>();
- List forGrid;
-
- public LoaderWay(Path workDirectory, Handler handler, int minEntitiesPerCell,
- LoaderNode nodeLoader, int maxZoomLevel) throws IOException {
- super(minEntitiesPerCell);
- Path[] files;
- try (
- DirectoryStream stream = Files.newDirectoryStream(workDirectory, "transform_way_*")) {
- files = StreamSupport.stream(stream.spliterator(), false).collect(Collectors.toList())
- .toArray(new Path[0]);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- this.reader = new TransformWayReaders(files);
- this.handler = handler;
- this.nodeLoader = nodeLoader;
- nodeLoader.addLoader(this);
- this.maxZoomLevel = Math.max(1, maxZoomLevel);
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- public int count = 0;
-
- @Override
- public void load(long cellId2, boolean all) {
- if (!reader.hasNext()) {
- return;
- }
-
- if (reader.getCellId() == -1) {
- Set set = reader.next();
- set.stream().forEach(osh -> invalids.put(osh.getId(), osh));
- handler.handleWayGrid(-1, invalids.values(), Collections.emptyList());
- }
-
- while (reader.hasNext()
- && (all || ZGrid.ORDER_DFS_TOP_DOWN.compare(reader.getCellId(), cellId2) <= 0)) {
- final long cellId = reader.getCellId();
- final int zoom = ZGrid.getZoom(cellId);
- currentZoom = zoom;
-
- final Set ways = reader.next();
- count++;
- ways.forEach(way -> {
- for (Loader loader : loaders) {
- loader.visitWay(way);
- }
- });
-
- maxZoom = initZoomLevel(maxZoom, zoom, zoomLevel, () -> new Grid());
-
- nodesForCellSet = new HashSet<>();
- ways.stream().forEach(osh -> {
- for (Long id : osh.getNodeIds()) {
- nodesForCellSet.add(id);
- }
- });
-
- forGrid = new ArrayList<>(nodesForCellSet.size());
-
- nodeLoader.load(cellId, false);
-
- store(zoom);
-
- Grid grid = zoomLevel.get(zoom);
- grid.cellId = cellId;
- grid.entities = new ArrayList<>(ways);
- grid.forGrid = forGrid;
- grid.nodesSet = nodesForCellSet;
-
- lastZoom = zoom;
- }
-
- if (!reader.hasNext()) {
- nodeLoader.load(0, true);
- store(0);
- nodesForCellSet = null;
- }
- }
-
- private void store(int zoom) {
- for (int i = lastZoom; i >= zoom; i--) {
- Grid grid = zoomLevel.get(i);
- if (grid == null || grid.entities == null) {
- continue;
- }
-
- if (i > maxZoomLevel || grid.entities.size() < minEntitiesPerCell && i > 0) {
- Grid parent = zoomLevel.get(i - 1);
-
- if (parent.entities == null) {
- parent.cellId = ZGrid.getParent(grid.cellId);
- parent.entities = grid.entities;
- parent.forGrid = grid.forGrid;
- parent.nodesSet = grid.nodesSet;
- } else {
- parent.entities.addAll(grid.entities);
- parent.forGrid.addAll(grid.forGrid);
-
- for (TransformOSHNode osh : grid.forGrid) {
- parent.nodesSet.remove(osh.getId());
- }
- parent.nodesSet.addAll(grid.nodesSet);
- }
- grid.clear();
- continue;
- }
-
- for (TransformOSHNode osh : grid.forGrid) {
- Long id = osh.getId();
- for (int j = i - 1; j >= 0; j--) {
- final Grid parent = zoomLevel.get(j);
- if (parent.nodesSet.contains(id)) {
- parent.forGrid.add(osh);
- parent.nodesSet.remove(id);
- break;
- }
- }
- }
-
- // some nodes could still be left, maybe they are invalid!
- // TODO or should we ignore them?
- grid.nodesSet.forEach(id -> {
- TransformOSHNode node = nodeLoader.invalidNodes.get(id.longValue());
- if (node != null) {
- grid.forGrid.add(node);
- }
- });
-
- handler.handleWayGrid(grid.cellId, grid.entities, grid.forGrid);
- grid.clear();
- }
- }
-
- @Override
- public void visitNode(TransformOSHNode node) {
- final Long id = node.getId();
- if (nodesForCellSet.contains(id)) {
- forGrid.add(node);
- nodesForCellSet.remove(id);
- } else {
- for (int i = lastZoom; i >= 0; i--) {
- final Grid g = zoomLevel.get(i);
- if (g.nodesSet.contains(id)) {
- g.forGrid.add(node);
- g.nodesSet.remove(id);
- break;
- }
- }
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/cli/DbH2Arg.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/cli/DbH2Arg.java
deleted file mode 100644
index 6c87d582d..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/cli/DbH2Arg.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load.cli;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.beust.jcommander.validators.PositiveInteger;
-import java.nio.file.Path;
-import org.heigit.ohsome.oshdb.tool.importer.cli.CommonArgs;
-
-public class DbH2Arg {
- @ParametersDelegate
- public CommonArgs common = new CommonArgs();
-
- @Parameter(names = {"-mn", "--min-nodes"}, description = "minimum of nodes per grid cell",
- validateWith = PositiveInteger.class)
- public int minNodesPerGrid = 1000;
-
- @Parameter(names = {"-mw", "--min-ways"}, description = "minimum of ways per grid cell",
- validateWith = PositiveInteger.class)
- public int minWaysPerGrid = 100;
-
- @Parameter(names = {"-mr", "--min-relations"}, description = "minimum of relations per grid cell",
- validateWith = PositiveInteger.class)
- public int minRelationPerGrid = 10;
-
- @Parameter(names = {"--nodesWithTagsOnly"},
- description = "only nodes with tags in the nodes grid")
- public boolean onlyNodesWithTags = true;
-
- @Parameter(names = {"--withOutKeyTables"}, description = "load also keytables in to h2 db")
- public boolean withOutKeyTables;
-
- @Parameter(names = {"--out"}, description = "output path", required = true)
- public Path h2db;
-
- @Parameter(names = {"-z", "--maxZoom"}, description = "maximal zoom level",
- validateWith = PositiveInteger.class, order = 2)
- public int maxZoom = 15;
-
- @Parameter(names = {"--attribution"}, required = true)
- public String attribution = "Copyright Right";
-
- @Parameter(names = {"--attribution-url"}, required = true)
- public String attributionUrl;
-
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/ConsoleHandler.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/ConsoleHandler.java
deleted file mode 100644
index 3a0f1a3f6..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/ConsoleHandler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load.handle;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransfomRelation;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-
-public class ConsoleHandler extends LoaderHandler {
-
- @Override
- public void loadKeyValues(int id, String key, List values) {
- final int valueLimit = 10;
- System.out.printf("%4d : k=%s, (%d:%s%s)%n", id, key, values.size(),
- values.stream().limit(valueLimit).collect(Collectors.toList()).toString(),
- values.size() > valueLimit ? "..." : "");
- }
-
- @Override
- public void loadRole(int id, String role) {
- System.out.printf("%4d : r=%s%n", id, role);
- }
-
- @Override
- public void handleNodeGrid(long cellId, Collection nodes) {
- final int zoom = ZGrid.getZoom(cellId);
- final long id = ZGrid.getIdWithoutZoom(cellId);
- System.out.printf("load node grid (%d:%d %d) nodes:%d%n", zoom, id, cellId, nodes.size());
- }
-
- @Override
- public void handleWayGrid(long cellId, Collection ways,
- Collection nodes) {
- final int zoom = ZGrid.getZoom(cellId);
- final long id = ZGrid.getIdWithoutZoom(cellId);
- System.out.printf("load way grid (%d:%d %d) ways:%d nodes:%d%n", zoom, id, cellId, ways.size(),
- nodes.size());
- }
-
- @Override
- public void handleRelationGrid(long cellId, Collection entities,
- Collection nodes, Collection ways) {
- final int zoom = ZGrid.getZoom(cellId);
- final long id = ZGrid.getIdWithoutZoom(cellId);
- System.out.printf("load relation grid (%d:%d %d) ways:%d nodes:%d ways:%d%n", zoom, id, cellId,
- entities.size(), nodes.size(), ways.size());
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/H2Handler.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/H2Handler.java
deleted file mode 100644
index 0476bd5ef..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/H2Handler.java
+++ /dev/null
@@ -1,321 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load.handle;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.ParameterException;
-import com.google.common.base.Stopwatch;
-import it.unimi.dsi.fastutil.io.FastByteArrayInputStream;
-import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream;
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
-import org.heigit.ohsome.oshdb.grid.GridOSHNodes;
-import org.heigit.ohsome.oshdb.grid.GridOSHRelations;
-import org.heigit.ohsome.oshdb.grid.GridOSHWays;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderKeyTables;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderNode;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderRelation;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderWay;
-import org.heigit.ohsome.oshdb.tool.importer.load.cli.DbH2Arg;
-import org.roaringbitmap.longlong.Roaring64NavigableMap;
-
-public class H2Handler extends OSHDBHandler {
-
- private PreparedStatement insertKey;
- private PreparedStatement insertValue;
- private PreparedStatement insertRole;
- private PreparedStatement insertNode;
- private PreparedStatement insertWay;
- private PreparedStatement insertRelation;
-
- public H2Handler(Roaring64NavigableMap bitmapNodes, Roaring64NavigableMap bitmapWays,
- PreparedStatement insertKey, PreparedStatement insertValue, PreparedStatement insertRole,
- PreparedStatement insertNode, PreparedStatement insertWay, PreparedStatement insertRelation) {
- super(bitmapNodes, bitmapWays);
- this.insertKey = insertKey;
- this.insertValue = insertValue;
- this.insertRole = insertRole;
- this.insertNode = insertNode;
- this.insertWay = insertWay;
- this.insertRelation = insertRelation;
-
- }
-
- @Override
- public void loadKeyValues(int keyId, String key, List values) {
- try {
- insertKey.setInt(1, keyId);
- insertKey.setString(2, key);
- insertKey.executeUpdate();
-
- int valueId = 0;
- for (String value : values) {
- try {
- insertValue.setInt(1, keyId);
- insertValue.setInt(2, valueId);
- insertValue.setString(3, value);
- insertValue.addBatch();
- valueId++;
- } catch (SQLException e) {
- System.err.printf("error %d:%s %d:%s%n", keyId, key, valueId, value);
- throw e;
- }
- }
- insertValue.executeBatch();
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- @Override
- public void loadRole(int id, String role) {
- try {
- insertRole.setInt(1, id);
- insertRole.setString(2, role);
- insertRole.executeUpdate();
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- FastByteArrayOutputStream out = new FastByteArrayOutputStream(1024);
-
- @Override
- public void handleNodeGrid(GridOSHNodes grid) {
- try {
- out.reset();
- try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
- oos.writeObject(grid);
- oos.flush();
- }
- System.out.print("insert " + grid.getLevel() + ":" + grid.getId());
- insertNode.setInt(1, grid.getLevel());
- insertNode.setLong(2, grid.getId());
- insertNode.setBinaryStream(3, new FastByteArrayInputStream(out.array, 0, out.length));
- insertNode.executeUpdate();
- System.out.println(" done!");
- } catch (IOException | SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void handleWayGrid(GridOSHWays grid) {
- // System.out.println("way "+grid.getLevel()+":"+grid.getId());
- try {
- out.reset();
- try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
- oos.writeObject(grid);
- oos.flush();
- }
- FastByteArrayInputStream in = new FastByteArrayInputStream(out.array, 0, out.length);
-
- insertWay.setInt(1, grid.getLevel());
- insertWay.setLong(2, grid.getId());
- insertWay.setBinaryStream(3, in);
- insertWay.executeUpdate();
-
- } catch (IOException | SQLException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- @Override
- public void handleRelationsGrid(GridOSHRelations grid) {
- try {
- out.reset();
- try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
- oos.writeObject(grid);
- oos.flush();
- }
- FastByteArrayInputStream in = new FastByteArrayInputStream(out.array, 0, out.length);
-
- insertRelation.setInt(1, grid.getLevel());
- insertRelation.setLong(2, grid.getId());
- insertRelation.setBinaryStream(3, in);
- insertRelation.executeUpdate();
-
- } catch (IOException | SQLException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- public static void load(DbH2Arg config) throws ClassNotFoundException, Exception {
- final Path workDirectory = config.common.workDir;
- Path oshdb = config.h2db;
- int maxZoomLevel = config.maxZoom;
-
- int minNodesPerGrid = config.minNodesPerGrid;
- int minWaysPerGrid = config.minWaysPerGrid;
- int minRelationPerGrid = config.minRelationPerGrid;
-
- boolean onlyNodesWithTags = config.onlyNodesWithTags;
-
- boolean withOutKeyTables = config.withOutKeyTables;
-
- Class.forName("org.h2.Driver");
- try (Connection conn =
- DriverManager.getConnection("jdbc:h2:" + oshdb.toString() + "", "sa", null)) {
- try (Statement stmt = conn.createStatement()) {
-
- try (
- BufferedReader br =
- new BufferedReader(new FileReader(
- workDirectory.resolve("extract_meta").toFile(), UTF_8));
- PreparedStatement insert = conn.prepareStatement(
- "insert into metadata (key,value) values (?,?)");) {
- stmt.executeUpdate("drop table if exists metadata"
- + "; create table if not exists metadata"
- + "(key varchar primary key, value varchar)");
-
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.trim().isEmpty()) {
- continue;
- }
-
- String[] split = line.split("=", 2);
- if (split.length != 2) {
- throw new RuntimeException("metadata file is corrupt");
- }
-
- insert.setString(1, split[0]);
- insert.setString(2, split[1]);
- insert.addBatch();
- }
-
- insert.setString(1, "attribution.short");
- insert.setString(2, config.attribution);
- insert.addBatch();
- insert.setString(1, "attribution.url");
- insert.setString(2, config.attributionUrl);
- insert.addBatch();
-
- insert.setString(1, "oshdb.maxzoom");
- insert.setString(2, Integer.toString(maxZoomLevel));
- insert.addBatch();
-
- insert.executeBatch();
- }
-
- stmt.executeUpdate("drop table if exists grid_node"
- + "; create table if not exists grid_node"
- + "(level int, id bigint, data blob, primary key(level,id))");
-
- stmt.executeUpdate("drop table if exists grid_way"
- + "; create table if not exists grid_way"
- + "(level int, id bigint, data blob, primary key(level,id))");
-
- stmt.executeUpdate("drop table if exists grid_relation"
- + "; create table if not exists grid_relation"
- + "(level int, id bigint, data blob, primary key(level,id))");
-
- Roaring64NavigableMap bitmapWays = new Roaring64NavigableMap();
- try (
- FileInputStream fileIn = new FileInputStream(
- workDirectory.resolve("transform_wayWithRelation.bitmap").toFile());
- ObjectInputStream in = new ObjectInputStream(fileIn)) {
- bitmapWays.readExternal(in);
- }
- try (
- PreparedStatement insertNode = conn.prepareStatement(
- "insert into grid_node (level,id,data) values(?,?,?)");
- PreparedStatement insertWay = conn.prepareStatement(
- "insert into grid_way (level,id,data) values(?,?,?)");
- PreparedStatement insertRelation = conn.prepareStatement(
- "insert into grid_relation (level,id,data) values(?,?,?)")) {
- LoaderHandler handler;
-
- Stopwatch loadingWatch = Stopwatch.createUnstarted();
- if (!withOutKeyTables) {
- stmt.executeUpdate("drop table if exists key"
- + "; create table if not exists key "
- + "(id int primary key, txt varchar)");
- stmt.executeUpdate("drop table if exists keyvalue "
- + "; create table if not exists keyvalue"
- + "(keyId int, valueId int, txt varchar, primary key (keyId,valueId))");
- stmt.executeUpdate("drop table if exists role"
- + "; create table if not exists role"
- + "(id int primary key, txt varchar)");
-
- try (
- PreparedStatement insertKey = conn.prepareStatement(
- "insert into key (id,txt) values (?,?)");
- PreparedStatement insertValue = conn.prepareStatement(
- "insert into keyvalue ( keyId, valueId, txt ) values(?,?,?)");
- PreparedStatement insertRole = conn.prepareStatement(
- "insert into role (id,txt) values(?,?)");) {
-
- handler = new H2Handler(Roaring64NavigableMap.bitmapOf(), bitmapWays, insertKey,
- insertValue, insertRole, insertNode, insertWay, insertRelation);
-
- LoaderKeyTables keyTables = new LoaderKeyTables(workDirectory, handler);
- System.out.print("loading tags ... ");
- loadingWatch.reset().start();
- keyTables.loadTags();
- System.out.println(" done! " + loadingWatch);
- System.out.print("loading roles ...");
- loadingWatch.reset().start();
- keyTables.loadRoles();
- System.out.println(" done! " + loadingWatch);
- }
- } else {
- handler = new H2Handler(Roaring64NavigableMap.bitmapOf(), bitmapWays, null, null, null,
- insertNode, insertWay, insertRelation);
- }
-
- try (
- LoaderNode node = new LoaderNode(workDirectory, handler, minNodesPerGrid,
- onlyNodesWithTags, maxZoomLevel);
- LoaderWay way =
- new LoaderWay(workDirectory, handler, minWaysPerGrid, node, maxZoomLevel);
- LoaderRelation rel = new LoaderRelation(workDirectory, handler, minRelationPerGrid,
- node, way, maxZoomLevel);) {
- System.out.print("loading to grid ...");
- loadingWatch.reset().start();
- rel.load();
- }
- System.out.println(" done! " + loadingWatch);
- }
- }
- }
- }
-
- public static void main(String[] args) throws Exception {
-
- DbH2Arg config = new DbH2Arg();
- JCommander jcom = JCommander.newBuilder().addObject(config).build();
-
- try {
- jcom.parse(args);
- } catch (ParameterException e) {
- System.out.println("");
- System.out.println(e.getLocalizedMessage());
- System.out.println("");
- jcom.usage();
- return;
- }
- if (config.common.help) {
- jcom.usage();
- return;
- }
-
- final Stopwatch stopWatch = Stopwatch.createStarted();
- load(config);
- System.out.println("loading done in " + stopWatch);
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/LoaderHandler.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/LoaderHandler.java
deleted file mode 100644
index e5636f2a2..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/LoaderHandler.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load.handle;
-
-import java.util.Collection;
-import java.util.List;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderKeyTables;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderNode;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderRelation;
-import org.heigit.ohsome.oshdb.tool.importer.load.LoaderWay;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransfomRelation;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-
-public class LoaderHandler implements LoaderKeyTables.Handler, LoaderNode.Handler,
- LoaderWay.Handler, LoaderRelation.Handler {
-
- @Override
- public void handleNodeGrid(long cellId, Collection nodes) {
- }
-
- @Override
- public void handleWayGrid(long cellId, Collection ways,
- Collection nodes) {
- }
-
- @Override
- public void handleRelationGrid(long cellId, Collection entities,
- Collection nodes, Collection ways) {
- }
-
- @Override
- public void loadKeyValues(int id, String key, List values) {
- }
-
- @Override
- public void loadRole(int id, String role) {
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/OSHDBHandler.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/OSHDBHandler.java
deleted file mode 100644
index 9318db3b6..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/load/handle/OSHDBHandler.java
+++ /dev/null
@@ -1,164 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.load.handle;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
-import org.heigit.ohsome.oshdb.grid.GridOSHNodes;
-import org.heigit.ohsome.oshdb.grid.GridOSHRelations;
-import org.heigit.ohsome.oshdb.grid.GridOSHWays;
-import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl;
-import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl;
-import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl;
-import org.heigit.ohsome.oshdb.index.XYGrid;
-import org.heigit.ohsome.oshdb.osh.OSHNode;
-import org.heigit.ohsome.oshdb.osh.OSHRelation;
-import org.heigit.ohsome.oshdb.osh.OSHWay;
-import org.heigit.ohsome.oshdb.osm.OSMNode;
-import org.heigit.ohsome.oshdb.osm.OSMRelation;
-import org.heigit.ohsome.oshdb.osm.OSMWay;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransfomRelation;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-import org.roaringbitmap.longlong.Roaring64NavigableMap;
-
-public abstract class OSHDBHandler extends LoaderHandler {
-
- protected final Roaring64NavigableMap bitmapNodeRelation;
- protected final Roaring64NavigableMap bitmapWayRelation;
-
- protected OSHDBHandler(Roaring64NavigableMap bitmapNodeRelation,
- Roaring64NavigableMap bitmapWayRelation) {
- this.bitmapNodeRelation = bitmapNodeRelation;
- this.bitmapWayRelation = bitmapWayRelation;
- }
-
- public abstract void handleNodeGrid(GridOSHNodes grid);
-
- @Override
- public void handleNodeGrid(long zid, Collection nodes) {
- if (zid < 0) {
- return;
- }
- List entries = nodes.stream().map(osh2 -> {
- List versions = osh2.stream().collect(Collectors.toList());
- return OSHNodeImpl.build(versions);
- }).collect(Collectors.toList());
-
- if (!entries.isEmpty()) {
- entries.sort((a, b) -> Long.compare(a.getId(), b.getId()));
- var grid = grid(zid, (xyId, zoom, lon, lat) ->
- GridOSHNodes.rebase(xyId, zoom, entries.get(0).getId(), 0, lon, lat, entries));
- handleNodeGrid(grid);
- }
- }
-
- protected Long2ObjectAVLTreeMap waysForRelation = new Long2ObjectAVLTreeMap<>();
-
- public abstract void handleWayGrid(GridOSHWays grid);
-
- @Override
- public void handleWayGrid(long zid, Collection ways,
- Collection nodes) {
- if (zid < 0) {
- return;
- }
- var idOshMap = mapNodes(nodes);
-
- List entities = ways.stream().map(tosh -> {
- List versions = tosh.stream().collect(Collectors.toList());
-
- List nodesForThisWay = new ArrayList<>(tosh.getNodeIds().length);
- for (long id : tosh.getNodeIds()) {
- if (idOshMap.get(id) == null) {
- continue;
- }
- nodesForThisWay.add(idOshMap.get(id));
- }
-
- var osh = OSHWayImpl.build(versions, nodesForThisWay);
- if (bitmapWayRelation.contains(osh.getId())) {
- waysForRelation.put(osh.getId(), osh);
- }
- return osh;
- }).collect(Collectors.toList());
-
- if (!entities.isEmpty()) {
- entities.sort((a, b) -> Long.compare(a.getId(), b.getId()));
- var grid = grid(zid, (xyId, zoom, lon, lat) ->
- GridOSHWays.compact(xyId, zoom, entities.get(0).getId(), 0, lon, lat, entities));
- handleWayGrid(grid);
- }
- }
-
- private T grid(long zid, GridInstance newInstance) {
- OSHDBBoundingBox bbox = ZGrid.getBoundingBox(zid);
- int longitude = bbox.getMinLongitude() + (bbox.getMaxLongitude() - bbox.getMinLongitude()) / 2;
- int latitude = bbox.getMinLatitude() + (bbox.getMaxLatitude() - bbox.getMinLatitude()) / 2;
- int zoom = ZGrid.getZoom(zid);
- var xyGrid = new XYGrid(zoom);
- long xyId = xyGrid.getId(longitude, latitude);
- return newInstance.newGrid(xyId, zoom, longitude, latitude);
- }
-
- @FunctionalInterface
- private interface GridInstance {
- T newGrid(long xyId, int zoom, int longitude, int latitude);
- }
-
- private Map mapNodes(Collection nodes) {
- Map idOshMap = new HashMap<>(nodes.size());
- nodes.forEach(osh2 -> {
- Long id = osh2.getId();
- List versions = osh2.stream().collect(Collectors.toList());
- OSHNode osh = OSHNodeImpl.build(versions);
- idOshMap.put(id, osh);
- });
- return idOshMap;
- }
-
- public abstract void handleRelationsGrid(GridOSHRelations grid);
-
- @Override
- public void handleRelationGrid(long zid, Collection relations,
- Collection nodes, Collection ways) {
- if (zid < 0) {
- return;
- }
-
- Map idOshMap = mapNodes(nodes);
- List entities = relations.stream().map(osh2 -> {
- List versions = osh2.stream().collect(Collectors.toList());
-
- List nodesForThisRelation = new ArrayList<>(osh2.getNodeIds().length);
- for (long id : osh2.getNodeIds()) {
- OSHNode node = idOshMap.get(id);
- if (node != null) {
- nodesForThisRelation.add(node);
- }
- }
-
- List waysForThisRelation = new ArrayList<>(osh2.getWayIds().length);
- for (long id : osh2.getWayIds()) {
- OSHWay way = waysForRelation.get(id);
- if (way != null) {
- waysForThisRelation.add(way);
- }
- }
-
- return OSHRelationImpl.build(versions, nodesForThisRelation, waysForThisRelation);
- }).collect(Collectors.toList());
-
- if (!entities.isEmpty()) {
- entities.sort((a, b) -> Long.compare(a.getId(), b.getId()));
- var grid = grid(zid, (xyId, zoom, lon, lat) ->
- GridOSHRelations.compact(xyId, zoom, entities.get(0).getId(), 0, lon, lat, entities));
- handleRelationsGrid(grid);
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transform.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transform.java
deleted file mode 100644
index 0d5d4faea..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transform.java
+++ /dev/null
@@ -1,220 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.ParameterException;
-import com.google.common.base.Stopwatch;
-import io.reactivex.Flowable;
-import io.reactivex.functions.Action;
-import io.reactivex.functions.Consumer;
-import io.reactivex.internal.functions.ObjectHelper;
-import io.reactivex.internal.operators.flowable.FlowableBlockingSubscribe;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import org.heigit.ohsome.oshdb.tool.importer.extract.Extract;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.OsmPbfMeta;
-import org.heigit.ohsome.oshdb.tool.importer.transform.cli.TransformArgs;
-import org.heigit.ohsome.oshdb.tool.importer.util.RoleToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.long2long.SortedLong2LongMap;
-import org.heigit.ohsome.oshdb.tool.importer.util.reactive.MyLambdaSubscriber;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.rx.RxOshPbfReader;
-import org.reactivestreams.Publisher;
-
-public class Transform {
- private static final long MB = 1024L * 1024L;
- private static final long GB = 1024L * MB;
-
- private final long maxMemory;
- private Path workDirectory = Paths.get(".");
-
- private Transform(long maxMemory) {
- this.maxMemory = maxMemory;
- }
-
- public static Transform withMaxMemory(long availableMemory) {
- return new Transform(availableMemory);
- }
-
- public Transform withWorkDirectory(Path workDirectory) {
- this.workDirectory = workDirectory;
- return this;
- }
-
- public static TagToIdMapper getTagToIdMapper(Path workDirectory)
- throws IOException {
- return TransformerTagRoles.getTagToIdMapper(workDirectory);
- }
-
- public static RoleToIdMapper getRoleToIdMapper(Path workDirectory)
- throws IOException {
- return TransformerTagRoles.getRoleToIdMapper(workDirectory);
- }
-
- public void transformNodes(OsmPbfMeta pbfMeta, int maxZoom, TagToIdMapper tag2Id, int workerId,
- int workerTotal) throws IOException {
- final Transformer transformer =
- new TransformerNode(maxMemory, maxZoom, workDirectory, tag2Id, workerId);
- Flowable> flow = RxOshPbfReader
- .readOsh(pbfMeta.pbf, pbfMeta.nodeStart, pbfMeta.nodeEnd, pbfMeta.nodeEnd)
- .map(osh -> osh.getVersions());
- subscribe(flow, transformer::transform, transformer::error, transformer::complete);
- }
-
- public void transformWays(OsmPbfMeta pbfMeta, int maxZoom, TagToIdMapper tag2Id,
- SortedLong2LongMap node2cell, int workerId, int workerTotal) throws IOException {
- final Transformer transformer =
- new TransformerWay(maxMemory, maxZoom, workDirectory, tag2Id, node2cell, workerId);
- Flowable> flow = RxOshPbfReader
- .readOsh(pbfMeta.pbf, pbfMeta.wayStart, pbfMeta.wayEnd, pbfMeta.wayEnd)
- .map(osh -> osh.getVersions());
- subscribe(flow, transformer::transform, transformer::error, transformer::complete);
-
- }
-
- public void transformRelations(OsmPbfMeta pbfMeta, int maxZoom, TagToIdMapper tag2Id,
- RoleToIdMapper role2Id, SortedLong2LongMap node2cell, SortedLong2LongMap way2cell,
- int workerId, int workerTotal) throws IOException {
- final Transformer transformer = new TransformerRelation(maxMemory, maxZoom, workDirectory,
- tag2Id, role2Id, node2cell, way2cell, workerId);
- Flowable> flow = RxOshPbfReader
- .readOsh(pbfMeta.pbf, pbfMeta.relationStart, pbfMeta.relationEnd, pbfMeta.relationEnd)
- .map(osh -> osh.getVersions());
- subscribe(flow, transformer::transform, transformer::error, transformer::complete);
-
- }
-
- private static void subscribe(Publisher extends T> o, final Consumer super T> onNext,
- final Consumer super Throwable> onError, final Action onComplete) {
- ObjectHelper.requireNonNull(onNext, "onNext is null");
- ObjectHelper.requireNonNull(onError, "onError is null");
- ObjectHelper.requireNonNull(onComplete, "onComplete is null");
- FlowableBlockingSubscribe.subscribe(o,
- new MyLambdaSubscriber(onNext, onError, onComplete, 1L));
- }
-
- public static void transform(TransformArgs config) throws Exception {
-
- final Path pbf = config.pbf;
- final Path workDir = config.common.workDir;
-
- final String step = config.step;
- final int maxZoom = config.maxZoom;
-
- int worker = config.distribute.worker;
- int workerTotal = config.distribute.totalWorkers;
- if (worker >= workerTotal) {
- throw new IllegalArgumentException("worker must be lesser than totalWorker!");
- }
- if (workerTotal > 1 && step.startsWith("a")) {
- throw new IllegalArgumentException(
- "step all with totalWorker > 1 is not allwod use step (node,way or relation)");
- }
-
- // reserve 1GB for parsing
- final long availableHeapMemory = SizeEstimator.estimateAvailableMemory();
- // reserve at least 1GB or 1/3 of the total memory
- final long availableMemory = availableHeapMemory - Math.max(1 * GB, availableHeapMemory / 3);
-
- System.out.println("Transform:");
- System.out.println("avaliable memory: " + availableMemory / MB + " mb");
-
- final OsmPbfMeta pbfMeta = Extract.pbfMetaData(pbf);
-
- final TagToIdMapper tag2Id = Transform.getTagToIdMapper(workDir);
-
- if (step.startsWith("a") || step.startsWith("n")) {
- long maxMemory = availableMemory - tag2Id.estimatedSize();
- if (maxMemory < 100 * MB) {
- System.out.println("warning: only 100MB memory left for transformation! "
- + "Increase heapsize -Xmx if possible");
- }
- if (maxMemory < 1 * MB) {
- throw new Exception("to few memory left for transformation. "
- + "You need to increase JVM heapsize -Xmx for transforming");
- }
-
- System.out.println("maxMemory for transformation: " + maxMemory / MB + " mb");
- System.out.print("start transforming nodes ...");
- Transform.withMaxMemory(maxMemory).withWorkDirectory(workDir).transformNodes(pbfMeta, maxZoom,
- tag2Id, worker, workerTotal);
- System.out.println(" done!");
- }
-
- if (step.startsWith("a") || step.startsWith("w")) {
- final long mapMemory = availableMemory / 2L;
- try (SortedLong2LongMap node2Cell =
- new SortedLong2LongMap(workDir.resolve("transform_idToCell_" + "node"), mapMemory);) {
- long maxMemory = availableMemory - tag2Id.estimatedSize() - mapMemory;
- if (maxMemory < 100 * MB) {
- System.out.println("warning: only 100MB memory left for transformation! "
- + "Increase heapsize -Xmx if possible");
- }
- if (maxMemory < 1 * MB) {
- throw new Exception("to few memory left for transformation. "
- + "You need to increase JVM heapsize -Xmx for transforming");
- }
-
- System.out.println("maxMemory for transformation: " + maxMemory / MB + " mb");
- System.out.print("start transforming ways ...");
- Transform.withMaxMemory(maxMemory).withWorkDirectory(workDir).transformWays(pbfMeta,
- maxZoom, tag2Id, node2Cell, worker, workerTotal);
- }
- System.out.println(" done!");
- }
-
- if (step.startsWith("a") || step.startsWith("r")) {
- final RoleToIdMapper role2Id = Transform.getRoleToIdMapper(workDir);
- final long mapMemory = availableMemory / 2L;
- final long mapMemoryNode = mapMemory / 3L;
- try (
- SortedLong2LongMap node2Cell = new SortedLong2LongMap(
- workDir.resolve("transform_idToCell_" + "node"), mapMemoryNode);
- SortedLong2LongMap way2Cell = new SortedLong2LongMap(
- workDir.resolve("transform_idToCell_" + "way"), mapMemory - mapMemoryNode);) {
- long maxMemory =
- availableMemory - tag2Id.estimatedSize() - role2Id.estimatedSize() - mapMemory;
- if (maxMemory < 100 * MB) {
- System.out.println("warning: only 100MB memory left for transformation! "
- + "Increase heapsize -Xmx if possible");
- }
- if (maxMemory < 1 * MB) {
- throw new Exception("to few memory left for transformation. "
- + "You need to increase JVM heapsize -Xmx for transforming");
- }
-
- System.out.println("maxMemory for transformation: " + maxMemory / MB + " mb");
- System.out.print("start transforming relations ...");
- Transform.withMaxMemory(maxMemory).withWorkDirectory(workDir).transformRelations(pbfMeta,
- maxZoom, tag2Id, role2Id, node2Cell, way2Cell, worker, workerTotal);
- }
- System.out.println(" done!");
- }
- }
-
- public static void main(String[] args) throws Exception {
- TransformArgs config = new TransformArgs();
- JCommander jcom = JCommander.newBuilder().addObject(config).build();
-
- try {
- jcom.parse(args);
- } catch (ParameterException e) {
- System.out.println("");
- System.out.println(e.getLocalizedMessage());
- System.out.println("");
- jcom.usage();
- return;
- }
- if (config.common.help) {
- jcom.usage();
- return;
- }
- Stopwatch stopwatch = Stopwatch.createStarted();
-
- transform(config);
- System.out.println("transform done in " + stopwatch);
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformReader.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformReader.java
deleted file mode 100644
index f7fb0ff0e..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformReader.java
+++ /dev/null
@@ -1,193 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.TreeSet;
-import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl;
-import org.heigit.ohsome.oshdb.osh.OSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-
-public class TransformReader {
- public final Path path;
- private final RandomAccessFile raf;
- private final long end;
- protected final FileChannel channel;
-
- private final ByteBuffer header = ByteBuffer.allocateDirect(8 + 4 + 4);
-
- protected long pos = 0;
-
- public long cellId;
- protected int size;
- protected int bytes;
-
- private TransformReader(Path path) throws IOException {
- this.path = path;
- this.raf = new RandomAccessFile(path.toFile(), "r");
- this.end = raf.length();
- this.channel = raf.getChannel();
-
- readHeader();
- }
-
- protected void skip() {
- pos += bytes;
- }
-
- protected void readHeader() throws IOException {
- header.clear();
- channel.read(header, pos);
- pos += header.capacity();
-
- header.flip();
- this.cellId = header.getLong();
- this.size = header.getInt();
- this.bytes = header.getInt();
- }
-
- public long getCellId() {
- return cellId;
- }
-
- public boolean hasNext() {
- return pos + bytes < end;
- }
-
- public void next() throws IOException {
- skip();
- readHeader();
- }
-
- @Override
- public String toString() {
- return String.format("%d:%d %d, (#%d %d)", pos - 16, end, cellId, size, bytes);
- }
-
- public static class NodeReader extends TransformReader {
- public NodeReader(Path path) throws IOException {
- super(path);
- }
-
- public Set get() throws IOException {
- final ByteBuffer data = ByteBuffer.allocateDirect(bytes);
- channel.read(data, pos);
- data.flip();
-
- Set ret = new TreeSet<>((a, b) -> Long.compare(a.getId(), b.getId()));
- while (data.hasRemaining()) {
- int length = data.getInt();
- byte[] content = new byte[length];
- data.get(content);
- OSHNode node = OSHNodeImpl.instance(content, 0, length);
- System.out.println(node.getId());
- ret.add(node);
- }
- ret.stream().filter(node -> node.getId() == 553542L).forEach(System.out::println);
- return ret;
- }
- }
-
- public static class NodeReaders {
- PriorityQueue queue = new PriorityQueue<>((a, b) -> {
- int c = ZGrid.ORDER_DFS_TOP_DOWN.compare(a.cellId, b.cellId);
- if (c != 0) {
- return c;
- }
- return a.path.compareTo(b.path);
- });
-
- public NodeReaders(NodeReader... readers) {
- for (NodeReader r : readers) {
- queue.add(r);
- }
- }
-
- public long cellId() {
- return queue.peek().cellId;
- }
-
- public Set get() throws IOException {
- List readers = new ArrayList<>(queue.size());
- NodeReader reader = queue.poll();
- readers.add(reader);
- Set nodes = reader.get();
- while (!queue.isEmpty() && queue.peek().cellId == reader.cellId) {
- nodes.addAll(queue.peek().get());
- readers.add(queue.poll());
- }
- readers.stream().filter(NodeReader::hasNext).forEach(r -> {
- try {
- r.next();
- queue.add(r);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
- });
- return nodes;
- }
-
- public boolean hasNext() {
- return !queue.isEmpty();
- }
- }
-
- public static class WayReader extends TransformReader {
- public WayReader(Path path) throws IOException {
- super(path);
- }
-
- public Set get() throws IOException {
- final ByteBuffer data = ByteBuffer.allocateDirect(bytes);
- channel.read(data, pos);
- data.flip();
-
- Set ret = new TreeSet<>((a, b) -> Long.compare(a.getId(), b.getId()));
- while (data.hasRemaining()) {
- int length = data.getInt();
- byte[] content = new byte[length];
- data.get(content);
- TransformOSHWay e = TransformOSHWay.instance(content, 0, length);
- ret.add(e);
-
- }
- return ret;
- }
- }
-
- public static void load(List> zoomLevel, int zoom,
- Long2ObjectMap nodes) {
- Grid grid = zoomLevel.get(zoom);
- if (grid != null) {
- System.out.println("remove " + zoom);
- zoomLevel.set(zoom, null);
- }
- }
-
- public static class Grid {
- public final long cellId;
- public final Set entities;
- // public final Roaring64NavigableMap refMap;
- public final Set refMapSet;
-
- private Grid(long cellId, Set entities, Set refMapSet) {
- this.cellId = cellId;
- this.entities = entities;
- this.refMapSet = refMapSet;
- }
-
- public static Grid of(long cellId, Set ways, Set refMapSet) {
- return new Grid<>(cellId, ways, refMapSet);
- }
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transformer.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transformer.java
deleted file mode 100644
index 7dfce69fe..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/Transformer.java
+++ /dev/null
@@ -1,281 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectAVLTreeMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
-import it.unimi.dsi.fastutil.longs.LongSet;
-import it.unimi.dsi.fastutil.objects.ObjectIterator;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.LongFunction;
-import java.util.stream.Collectors;
-import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.tool.importer.util.RoleToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagId;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.ZGrid;
-import org.heigit.ohsome.oshdb.tool.importer.util.long2long.SortedLong2LongMap;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.TagText;
-import org.roaringbitmap.longlong.Roaring64NavigableMap;
-
-public abstract class Transformer {
- private static class OSHDataContainer {
- private long sizeInBytesOfData = 0;
- private long estimatedMemoryUsage =
- SizeEstimator.objOverhead() + 2 * SizeEstimator.intField() + SizeEstimator.linkedList();
-
- private long lastId = 0;
- private List list = new LinkedList<>();
-
- public OSHDataContainer add(byte[] data) {
- sizeInBytesOfData += data.length + 4; // count of bytes + 4 bytes for the length of this array
- estimatedMemoryUsage += SizeEstimator.estimatedSizeOf(data) + SizeEstimator.linkedListEntry();
- list.add(data);
- return this;
- }
- }
-
- private static final int PAGE_POWER = 17; // ~1MB per page
-
- private final TagToIdMapper tagToIdMapper;
- private final RoleToIdMapper roleToIdMapper;
- private final Long2ObjectAVLTreeMap collector;
- private final SortedLong2LongMap.Sink idToCellSink;
- private final SortedLong2LongMap idToCell;
-
- private final Map> typeRefsMaps =
- new HashMap<>(OSMType.values().length);
- private long estimatedMemoryUsage;
- private final long maxMemoryUsage;
-
- protected final Path workDirectory;
- private final int workerId;
- private int fileNumber = 0;
-
- private final ZGrid grid;
-
- public Transformer(long maxMemoryUsage, int maxZoom, Path workDirectory,
- TagToIdMapper tagToIdMapper, int workerId) throws IOException {
- this(maxMemoryUsage, maxZoom, workDirectory, tagToIdMapper, null, workerId);
- }
-
- public Transformer(long maxMemoryUsage, int maxZoom, Path workDirectory,
- TagToIdMapper tagToIdMapper, RoleToIdMapper roleToIdMapper, int workerId) throws IOException {
- this.maxMemoryUsage = maxMemoryUsage;
- this.workDirectory = workDirectory;
- this.tagToIdMapper = tagToIdMapper;
- this.roleToIdMapper = roleToIdMapper;
- this.workerId = workerId;
- this.collector = new Long2ObjectAVLTreeMap<>(ZGrid.ORDER_DFS_TOP_DOWN);
- this.grid = new ZGrid(maxZoom);
-
- this.idToCellSink = new SortedLong2LongMap.Sink(
- workDirectory.resolve("transform_idToCell_" + type().toString().toLowerCase()), PAGE_POWER);
- this.idToCell = null;
-
- }
-
- public abstract void transform(long id, List versions);
-
- public void transform(List versions) {
- final Entity e = versions.get(0);
- if (type() == e.getType()) {
- transform(e.getId(), versions);
- }
- }
-
- public void error(Throwable t) {
- System.err.println(t);
- }
-
- public void complete() {
- System.out.println("COMPLETE");
- saveToDisk();
- idToCellSink.close();
- }
-
- public int modifiedVersion(Entity entity) {
- return entity.getVersion() * (entity.isVisible() ? 1 : -1);
- }
-
- public int[] getKeyValue(TagText[] tags) {
- if (tags.length == 0) {
- return new int[0];
- }
-
- final List ids = new ArrayList<>(tags.length);
-
- for (TagText tag : tags) {
- final int key = tagToIdMapper.getKey(tag.key);
- final int value = tagToIdMapper.getValue(key, tag.value);
- ids.add(TagId.of(key, value));
- }
-
- ids.sort((a, b) -> {
- final int c = Integer.compare(a.key, b.key);
- return c != 0 ? c : Integer.compare(a.value, b.value);
- });
- final int[] ret = new int[tags.length * 2];
- int i = 0;
- for (TagId tag : ids) {
- ret[i++] = tag.key;
- ret[i++] = tag.value;
- }
-
- return ret;
- }
-
- public int getRole(String role) {
- return roleToIdMapper != null ? roleToIdMapper.getRole(role) : 0;
- }
-
- protected long getCell(long longitude, long latitude) {
- return grid.getIdSingleZidWithZoom(longitude, latitude);
- }
-
- protected OSHDBBoundingBox getCellBounce(long cellId) {
- return ZGrid.getBoundingBox(cellId);
- }
-
- protected static long findBestFittingCellId(Set cellIds) {
- if (cellIds.isEmpty()) {
- return -1;
- }
-
- if (cellIds.size() == 1) {
- return cellIds.iterator().next();
- }
-
- int minZoom = Integer.MAX_VALUE;
- for (Long cellId : cellIds) {
- minZoom = Math.min(minZoom, ZGrid.getZoom(cellId));
- }
- final int zoom = minZoom;
- // bring all to the same zoom level
- Set bestCellId = cellIds.stream().filter(id -> id >= 0)
- .map(id -> ZGrid.getParent(id, zoom)).collect(Collectors.toSet());
-
- while (bestCellId.size() > 1) {
- cellIds = bestCellId;
- bestCellId = cellIds.stream().map(id -> ZGrid.getParent(id)).collect(Collectors.toSet());
- }
- final long cellId = bestCellId.iterator().next();
- return cellId;
- }
-
- protected void addIdToCell(long id, long cellId) throws IOException {
- if (idToCell == null) {
- idToCellSink.put(id, cellId);
- } else {
- final long cell = idToCell.get(id);
- if (cell != cellId) {
- System.err.println("id" + id + " did not match" + cellId + " stored " + cell);
- }
- }
- }
-
- private void saveToDisk() {
- if (collector.isEmpty()) {
- return;
- }
- final Path filePath = workDirectory.resolve(String.format("transform_%s_%02d_%02d",
- type().toString().toLowerCase(), workerId, fileNumber));
- System.out.print("transformer saveToDisk " + filePath.toString() + " ... ");
- long bytesWritten = 0;
- try (RandomAccessFile out = new RandomAccessFile(filePath.toFile(), "rw");
- FileChannel channel = out.getChannel()) {
- final ByteBuffer header = ByteBuffer.allocateDirect(8 + 4 + 4);
- ByteBuffer byteBuffer = ByteBuffer.allocateDirect(0);
-
- ObjectIterator> iter = collector.long2ObjectEntrySet().iterator();
- while (iter.hasNext()) {
- Entry entry = iter.next();
-
- final long cellId = entry.getLongKey();
- final OSHDataContainer container = entry.getValue();
- final int rawSize = (int) container.sizeInBytesOfData;
-
- if (byteBuffer.capacity() < rawSize) {
- byteBuffer = ByteBuffer.allocate(rawSize);
- } else {
- byteBuffer.clear();
- }
-
- for (byte[] data : container.list) {
- byteBuffer.putInt(data.length);
- byteBuffer.put(data);
- }
- byteBuffer.flip();
-
- header.clear();
- header.putLong(cellId);
- header.putInt(container.list.size());
- if (rawSize < 0) {
- System.err.println("saveToDisk rawSize negative " + cellId + " " + container.list.size());
- }
- header.putInt(rawSize);
- header.flip();
-
- bytesWritten += header.remaining();
- channel.write(header);
- bytesWritten += byteBuffer.remaining();
- channel.write(byteBuffer);
- }
- System.out.println("done! " + bytesWritten + " bytes");
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
- fileNumber++;
- collector.clear();
- typeRefsMaps.clear();
- estimatedMemoryUsage = 0L;
- }
-
- protected void store(long cellId, long id, LongFunction data) {
- if (estimatedMemoryUsage > maxMemoryUsage) {
- saveToDisk();
- }
-
- OSHDataContainer dataContainer = collector.get(cellId);
- if (dataContainer == null) {
- dataContainer = new OSHDataContainer();
- collector.put(cellId, dataContainer);
- estimatedMemoryUsage += SizeEstimator.avlTreeEntry();
- }
- estimatedMemoryUsage -= dataContainer.estimatedMemoryUsage;
-
- dataContainer.add(data.apply(dataContainer.lastId));
- dataContainer.lastId = id;
- estimatedMemoryUsage += dataContainer.estimatedMemoryUsage;
- }
-
- protected void store(long cellId, long id, LongFunction data, LongSet nodes) {
- store(cellId, id, data);
- }
-
- protected void store(long cellId, long id, LongFunction data, LongSet nodes,
- LongSet ways) {
- store(cellId, id, data, nodes);
- }
-
- protected void store(long cellId, long id, LongFunction data, LongSet nodes, LongSet ways,
- LongSet relation) {
- store(cellId, id, data, nodes, ways);
- }
-
- public abstract OSMType type();
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerNode.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerNode.java
deleted file mode 100644
index 40bc321bc..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerNode.java
+++ /dev/null
@@ -1,90 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.function.LongFunction;
-import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
-import org.heigit.ohsome.oshdb.osm.OSMNode;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHNode;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.util.bytearray.ByteArrayOutputWrapper;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Node;
-
-public class TransformerNode extends Transformer {
-
- private final ByteArrayOutputWrapper baData = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper baRecord = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper baAux = new ByteArrayOutputWrapper(1024);
-
- public TransformerNode(long maxMemory, int maxZoom, Path workDirectory,
- TagToIdMapper tagToIdMapper, int workerId) throws IOException {
- super(maxMemory, maxZoom, workDirectory, tagToIdMapper, workerId);
- }
-
- @Override
- public OSMType type() {
- return OSMType.NODE;
- }
-
- @Override
- public void transform(long id, List versions) {
-
- final List nodes = new ArrayList<>(versions.size());
- final Set cellIds = new TreeSet<>();
- for (Entity version : versions) {
- final Node node = (Node) version;
- if (version.isVisible()) {
- final long zId = getCell(node.getLongitude(), node.getLatitude());
- if (zId >= 0) {
- cellIds.add(zId);
- }
- }
- nodes.add(getNode(node));
- }
- final long cellId = !cellIds.isEmpty() ? findBestFittingCellId(cellIds) : -1;
-
- try {
-
- final OSHDBBoundingBox bbox = getCellBounce(cellId);
-
- final long baseLongitude = bbox.getMinLongitude();
- final long baseLatitude = bbox.getMinLatitude();
-
- final LongFunction toByteArray = baseId -> {
- try {
- TransformOSHNode.build(baData, baRecord, baAux, nodes,
- baseId, 0L, baseLongitude, baseLatitude);
-
- final byte[] bytes = new byte[baRecord.length()];
- System.arraycopy(baRecord.array(), 0, bytes, 0, bytes.length);
-
- return bytes;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- };
-
- store(cellId, id, toByteArray);
- addIdToCell(id, cellId);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private OSMNode getNode(Node entity) {
- return new OSMNode(entity.getId(),
- modifiedVersion(entity),
- entity.getTimestamp(),
- entity.getChangeset(),
- entity.getUserId(),
- getKeyValue(entity.getTags()),
- entity.getLongitude(), entity.getLatitude());
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerRelation.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerRelation.java
deleted file mode 100644
index 1521393d6..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerRelation.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import it.unimi.dsi.fastutil.longs.LongAVLTreeSet;
-import it.unimi.dsi.fastutil.longs.LongSet;
-import it.unimi.dsi.fastutil.longs.LongSortedSet;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.UncheckedIOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.LongFunction;
-import org.heigit.ohsome.oshdb.osm.OSMMember;
-import org.heigit.ohsome.oshdb.osm.OSMRelation;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransfomRelation;
-import org.heigit.ohsome.oshdb.tool.importer.util.RoleToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.long2long.SortedLong2LongMap;
-import org.heigit.ohsome.oshdb.util.bytearray.ByteArrayOutputWrapper;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Relation;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.RelationMember;
-import org.roaringbitmap.longlong.Roaring64NavigableMap;
-
-public class TransformerRelation extends Transformer {
- private final ByteArrayOutputWrapper wrapperData = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper wrapperRecord = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper wrapperNodeData = new ByteArrayOutputWrapper(1024);
-
- final SortedLong2LongMap nodeToCell;
- final SortedLong2LongMap wayToCell;
-
- /**
- * Creates a new {@code TransfomerRelation}.
- */
- public TransformerRelation(long maxMemory, int maxZoom, Path workDirectory,
- TagToIdMapper tagToIdMapper, RoleToIdMapper role2Id, SortedLong2LongMap nodeToCell,
- SortedLong2LongMap wayToCell, int workerId) throws IOException {
- super(maxMemory, maxZoom, workDirectory, tagToIdMapper, role2Id, workerId);
- this.nodeToCell = nodeToCell;
- this.wayToCell = wayToCell;
-
- }
-
- @Override
- public OSMType type() {
- return OSMType.RELATION;
- }
-
- private Roaring64NavigableMap bitmapRefNode = new Roaring64NavigableMap();
- private Roaring64NavigableMap bitmapRefWay = new Roaring64NavigableMap();
-
- @Override
- public void transform(long id, List versions) {
- List entities = new ArrayList<>(versions.size());
- LongSortedSet nodeIds = new LongAVLTreeSet();
- LongSortedSet wayIds = new LongAVLTreeSet();
- for (Entity version : versions) {
- final Relation entity = (Relation) version;
- final OSMRelation osm = getOSM(entity);
- entities.add(osm);
-
- for (OSMMember member : osm.getMembers()) {
- final OSMType type = member.getType();
- if (type == OSMType.NODE) {
- nodeIds.add(member.getId());
- bitmapRefNode.add(member.getId());
- } else if (type == OSMType.WAY) {
- wayIds.add(member.getId());
- bitmapRefWay.add(member.getId());
- }
- }
- }
-
- LongSet cellIds;
- if (!nodeIds.isEmpty()) {
- cellIds = nodeToCell.get(nodeIds);
- cellIds.addAll(wayToCell.get(wayIds));
- } else {
- cellIds = wayToCell.get(wayIds);
- }
-
- final long cellId = findBestFittingCellId(cellIds);
- try {
- final LongFunction toByteArray = baseId -> {
- try {
- TransfomRelation.build(wrapperData, wrapperRecord, wrapperNodeData,
- entities, nodeIds, wayIds, baseId, 0, 0, 0);
-
- final byte[] record = new byte[wrapperRecord.length()];
- System.arraycopy(wrapperRecord.array(), 0, record, 0, record.length);
-
- return record;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- };
-
- store(cellId, id, toByteArray, nodeIds, wayIds);
- addIdToCell(id, cellId);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- public void complete() {
- super.complete();
-
- bitmapRefNode.runOptimize();
- try (
- FileOutputStream fileOut = new FileOutputStream(
- workDirectory.resolve("transform_nodeWithRelation.bitmap").toFile());
- ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
- bitmapRefNode.writeExternal(out);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- bitmapRefWay.runOptimize();
- try (
- FileOutputStream fileOut = new FileOutputStream(
- workDirectory.resolve("transform_wayWithRelation.bitmap").toFile());
- ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
- bitmapRefWay.writeExternal(out);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private OSMRelation getOSM(Relation entity) {
- return new OSMRelation(entity.getId(),
- modifiedVersion(entity),
- entity.getTimestamp(),
- entity.getChangeset(),
- entity.getUserId(),
- getKeyValue(entity.getTags()),
- convertToOSMMembers(entity.getMembers()));
- }
-
- private OSMMember[] convertToOSMMembers(RelationMember[] members) {
- OSMMember[] ret = new OSMMember[members.length];
- int i = 0;
- for (RelationMember member : members) {
- ret[i++] = new OSMMember(member.memId, OSMType.fromInt(member.type), getRole(member.role));
- }
- return ret;
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerTagRoles.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerTagRoles.java
deleted file mode 100644
index 139f75784..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerTagRoles.java
+++ /dev/null
@@ -1,196 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import com.google.common.base.Functions;
-import it.unimi.dsi.fastutil.ints.Int2IntAVLTreeMap;
-import it.unimi.dsi.fastutil.ints.Int2IntMap;
-import it.unimi.dsi.fastutil.objects.Object2IntAVLTreeMap;
-import it.unimi.dsi.fastutil.objects.Object2IntMap;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.RandomAccessFile;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.function.Function;
-import java.util.function.ToIntFunction;
-import org.heigit.ohsome.oshdb.tool.importer.extract.Extract.KeyValuePointer;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.Role;
-import org.heigit.ohsome.oshdb.tool.importer.extract.data.ValueFrequency;
-import org.heigit.ohsome.oshdb.tool.importer.util.RoleToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.RoleToIdMapperImpl;
-import org.heigit.ohsome.oshdb.tool.importer.util.SizeEstimator;
-import org.heigit.ohsome.oshdb.tool.importer.util.StringToIdMappingImpl;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapperImpl;
-
-public class TransformerTagRoles {
-
- public static TagToIdMapper getTagToIdMapper(Path workDirectory)
- throws IOException {
- final ToIntFunction hashFunction = s -> s.hashCode();
-
- final Path tag2IdPath = workDirectory.resolve("transform_tag2Id");
- if (Files.exists(tag2IdPath)) {
- return TagToIdMapperImpl.load(tag2IdPath.toString(), hashFunction);
- }
-
- final Int2IntAVLTreeMap keyHash2Cnt = new Int2IntAVLTreeMap();
- final StringToIdMappingImpl[] value2IdMappings;
-
- final Function input = Functions.identity();
- try (RandomAccessFile raf =
- new RandomAccessFile(workDirectory.resolve("extract_keyvalues").toFile(), "r")) {
- final FileChannel channel = raf.getChannel();
- try (
- InputStream in = input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_keys").toFile())));
- DataInputStream dataIn = new DataInputStream(in)) {
- final int length = dataIn.readInt();
-
- value2IdMappings = new StringToIdMappingImpl[length];
-
- for (int i = 0; i < length; i++) {
- final KeyValuePointer kvp = KeyValuePointer.read(dataIn);
- value2IdMappings[i] = value2IdMapping(kvp, channel, hashFunction);
- final int hash = hashFunction.applyAsInt(kvp.key);
- keyHash2Cnt.addTo(hash, 1);
- }
- }
- }
-
- final Int2IntMap uniqueKeys = new Int2IntAVLTreeMap();
- final Object2IntMap notUniqueKeys = new Object2IntAVLTreeMap<>();
- long estimatedSize = 0;
- try (
- InputStream in = input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_keys").toFile())));
- DataInputStream dataIn = new DataInputStream(in)) {
- final int length = dataIn.readInt();
-
- for (int i = 0; i < length; i++) {
- final KeyValuePointer kvp = KeyValuePointer.read(dataIn);
- final int hash = hashFunction.applyAsInt(kvp.key);
- if (keyHash2Cnt.get(hash) > 1) {
- notUniqueKeys.put(kvp.key, i);
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue(kvp.key) + 4;
- } else {
- uniqueKeys.put(hash, i);
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue("") + 8;
- }
- }
- }
- final StringToIdMappingImpl key2IdMapping =
- new StringToIdMappingImpl(uniqueKeys, notUniqueKeys, hashFunction, estimatedSize);
- final TagToIdMapperImpl tag2IdMapper = new TagToIdMapperImpl(key2IdMapping, value2IdMappings);
- try (
- FileOutputStream fout =
- new FileOutputStream(workDirectory.resolve("transform_tag2Id").toFile());
- BufferedOutputStream bout = new BufferedOutputStream(fout);
- DataOutputStream out = new DataOutputStream(bout);) {
- tag2IdMapper.write(out);
- }
- return tag2IdMapper;
- }
-
- private static StringToIdMappingImpl value2IdMapping(KeyValuePointer kvp, FileChannel channel,
- ToIntFunction hashFunction) throws IOException {
- final Int2IntAVLTreeMap valueHash2Cnt = new Int2IntAVLTreeMap();
-
- channel.position(kvp.valuesOffset);
- try (DataInputStream valueStream = new DataInputStream(
- new BufferedInputStream(Channels.newInputStream(channel), 1024 * 1024));) {
- for (int j = 0; j < kvp.valuesNumber; j++) {
- final ValueFrequency vf = ValueFrequency.read(valueStream);
- final int hash = hashFunction.applyAsInt(vf.value);
- valueHash2Cnt.addTo(hash, 1);
- }
- }
-
- final Int2IntMap uniqueValues = new Int2IntAVLTreeMap();
- final Object2IntMap notUniqueValues = new Object2IntAVLTreeMap<>();
- long estimatedSize = 0;
- channel.position(kvp.valuesOffset);
- try (DataInputStream valueStream = new DataInputStream(Channels.newInputStream(channel));) {
- for (int j = 0; j < kvp.valuesNumber; j++) {
- final ValueFrequency vf = ValueFrequency.read(valueStream);
- final int hash = hashFunction.applyAsInt(vf.value);
- if (valueHash2Cnt.get(hash) > 1) {
- notUniqueValues.put(vf.value, j);
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue(kvp.key) + 4;
- } else {
- uniqueValues.put(hash, j);
- estimatedSize += SizeEstimator.avlTreeEntry() + 8;
- }
- }
- }
-
- return new StringToIdMappingImpl(uniqueValues, notUniqueValues, hashFunction, estimatedSize);
- }
-
- public static RoleToIdMapper getRoleToIdMapper(Path workDirectory)
- throws IOException {
- final ToIntFunction hashFunction = s -> s.hashCode();
-
- final Path role2IdPath = workDirectory.resolve("transform_role2Id");
- if (Files.exists(role2IdPath)) {
- return RoleToIdMapperImpl.load(role2IdPath.toString(), hashFunction);
- }
-
- final Int2IntAVLTreeMap roleHash2Cnt = new Int2IntAVLTreeMap();
-
- final Int2IntMap uniqueRoles = new Int2IntAVLTreeMap();
- final Object2IntMap notUniqueRoles = new Object2IntAVLTreeMap<>();
-
- final Function input = Functions.identity();
- int roleCount = 0;
- try (
- InputStream in = input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_roles").toFile())));
- DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in))) {
- while (dataIn.available() > 0) {
- final Role role = Role.read(dataIn);
- final int hash = hashFunction.applyAsInt(role.role);
- roleHash2Cnt.addTo(hash, 1);
- roleCount++;
- }
- }
-
- long estimatedSize = 0;
- try (
- InputStream in = input.apply(new BufferedInputStream(
- new FileInputStream(workDirectory.resolve("extract_roles").toFile())));
- DataInputStream dataIn = new DataInputStream(in)) {
-
- for (int i = 0; i < roleCount; i++) {
- final Role role = Role.read(dataIn);
- final int hash = hashFunction.applyAsInt(role.role);
- if (roleHash2Cnt.get(hash) > 1) {
- notUniqueRoles.put(role.role, i);
- estimatedSize += SizeEstimator.estimatedSizeOfAvlEntryValue(role.role) + 4;
- } else {
- uniqueRoles.put(hash, i);
- estimatedSize += SizeEstimator.avlTreeEntry() + 8;
- }
- }
- }
-
- RoleToIdMapperImpl idMapper = new RoleToIdMapperImpl(
- new StringToIdMappingImpl(uniqueRoles, notUniqueRoles, hashFunction, estimatedSize));
-
- try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
- new FileOutputStream(workDirectory.resolve("transform_role2Id").toFile())))) {
- idMapper.write(out);
- }
-
- return idMapper;
-
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerWay.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerWay.java
deleted file mode 100644
index 5fb5c48ef..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/TransformerWay.java
+++ /dev/null
@@ -1,96 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform;
-
-import it.unimi.dsi.fastutil.longs.LongAVLTreeSet;
-import it.unimi.dsi.fastutil.longs.LongSet;
-import it.unimi.dsi.fastutil.longs.LongSortedSet;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.LongFunction;
-import org.heigit.ohsome.oshdb.osm.OSMMember;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-import org.heigit.ohsome.oshdb.osm.OSMWay;
-import org.heigit.ohsome.oshdb.tool.importer.transform.oshdb.TransformOSHWay;
-import org.heigit.ohsome.oshdb.tool.importer.util.TagToIdMapper;
-import org.heigit.ohsome.oshdb.tool.importer.util.long2long.SortedLong2LongMap;
-import org.heigit.ohsome.oshdb.util.bytearray.ByteArrayOutputWrapper;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Entity;
-import org.heigit.ohsome.oshpbf.parser.osm.v06.Way;
-
-public class TransformerWay extends Transformer {
- private final ByteArrayOutputWrapper baData = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper baRecord = new ByteArrayOutputWrapper(1024);
- private final ByteArrayOutputWrapper wrapperNodeData = new ByteArrayOutputWrapper(1024);
-
- final SortedLong2LongMap nodeToCell;
-
- public TransformerWay(long maxMemory, int maxZoom, Path workDirectory,
- TagToIdMapper tagToIdMapper, SortedLong2LongMap nodeToCell, int workerId) throws IOException {
- super(maxMemory, maxZoom, workDirectory, tagToIdMapper, workerId);
- this.nodeToCell = nodeToCell;
- }
-
- @Override
- public OSMType type() {
- return OSMType.WAY;
- }
-
- @Override
- public void transform(long id, List versions) {
- List ways = new ArrayList<>(versions.size());
- LongSortedSet nodeIds = new LongAVLTreeSet();
- for (Entity version : versions) {
- Way way = (Way) version;
- ways.add(getOSM(way));
- for (long ref : way.refs) {
- nodeIds.add(ref);
- }
- }
-
- LongSet cellIds = nodeToCell.get(nodeIds);
-
- final long cellId = findBestFittingCellId(cellIds);
-
- try {
- final LongFunction toByteArray = baseId -> {
- try {
- TransformOSHWay.build(baData, baRecord, wrapperNodeData, ways,
- nodeIds, baseId, 0, 0, 0);
-
- final byte[] record = new byte[baRecord.length()];
- System.arraycopy(baRecord.array(), 0, record, 0, record.length);
-
- return record;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- };
-
- store(cellId, id, toByteArray, nodeIds);
- addIdToCell(id, cellId);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- private OSMWay getOSM(Way entity) {
- return new OSMWay(entity.getId(),
- modifiedVersion(entity),
- entity.getTimestamp(),
- entity.getChangeset(),
- entity.getUserId(),
- getKeyValue(entity.getTags()),
- convertNodeIdsToOSMMembers(entity.getRefs()));
- }
-
- private OSMMember[] convertNodeIdsToOSMMembers(long[] refs) {
- OSMMember[] ret = new OSMMember[refs.length];
- int i = 0;
- for (long ref : refs) {
- ret[i++] = new OSMMember(ref, OSMType.NODE, -1);
- }
- return ret;
- }
-}
\ No newline at end of file
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/TransformArgs.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/TransformArgs.java
deleted file mode 100644
index e25bf7edd..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/TransformArgs.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.cli;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.beust.jcommander.validators.PositiveInteger;
-import java.nio.file.Path;
-import org.heigit.ohsome.oshdb.tool.importer.cli.CommonArgs;
-import org.heigit.ohsome.oshdb.tool.importer.cli.DistributableArgs;
-import org.heigit.ohsome.oshdb.tool.importer.cli.validator.FileExistValidator;
-import org.heigit.ohsome.oshdb.tool.importer.transform.cli.validator.TransformStepValidator;
-
-public class TransformArgs {
- @ParametersDelegate
- public CommonArgs common = new CommonArgs();
-
- @ParametersDelegate
- public DistributableArgs distribute = new DistributableArgs();
-
- @Parameter(names = {"--pbf"}, description = "path to pbf-File to import",
- validateWith = FileExistValidator.class, required = true, order = 0)
- public Path pbf;
-
- @Parameter(names = {"-s", "--step"},
- description = "step for transformation (all|node|way|relation)",
- validateWith = TransformStepValidator.class, order = 1)
- public String step = "all";
-
- @Parameter(names = {"-z", "--maxZoom"}, description = "maximal zoom level",
- validateWith = PositiveInteger.class, order = 2)
- public int maxZoom = 15;
-
- @Parameter(names = {"--overwrite"}, description = "overwrite existing files", order = 3)
- public boolean overwrite = false;
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/validator/TransformStepValidator.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/validator/TransformStepValidator.java
deleted file mode 100644
index 2919b31b0..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/cli/validator/TransformStepValidator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.cli.validator;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.ParameterException;
-import com.google.common.collect.Sets;
-import java.util.Set;
-
-public class TransformStepValidator implements IParameterValidator {
- private static final Set stepSet;
-
- static {
- stepSet = Sets.newHashSet("a", "all", "n", "node", "w", "way", "r", "relation");
- }
-
- @Override
- public void validate(String name, String value) throws ParameterException {
- final String step = value.trim().toLowerCase();
-
- if (!stepSet.contains(step)) {
- throw new ParameterException(value + " for parameter " + name
- + " is not a valid value. Allowed values are (a,all,n,node,w,way,r,relation)");
- }
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMEntity.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMEntity.java
deleted file mode 100644
index 26ac64c7b..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMEntity.java
+++ /dev/null
@@ -1,99 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
-
-import org.heigit.ohsome.oshdb.OSHDBTimestamp;
-
-public class MutableOSMEntity implements OSMEntity {
-
- private long id;
-
- private int version;
- private boolean visible;
- private long timestamp;
- private long changeset;
- private int userId;
- private int[] tags;
-
- public void setEntity(long id, int version, boolean visible, long timestamp, long changeset,
- int userId, int[] tags) {
- this.id = id;
- this.version = version;
- this.visible = visible;
- this.timestamp = timestamp;
- this.changeset = changeset;
- this.userId = userId;
- this.tags = tags;
- }
-
- @Override
- public long getId() {
- return id;
- }
-
- public void setId(long id) {
- this.id = id;
- }
-
- @Override
- public int getVersion() {
- return version;
- }
-
- @Override
- public boolean isVisible() {
- return visible;
- }
-
- public void isVisible(boolean visible) {
- this.visible = visible;
- }
-
- public void setVersion(int version) {
- this.version = version;
- }
-
- @Override
- public long getEpochSecond() {
- return timestamp;
- }
-
- public void setTimestamp(OSHDBTimestamp timestamp) {
- this.timestamp = timestamp.getEpochSecond();
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
- @Override
- public long getChangeset() {
- return changeset;
- }
-
- public void setChangeset(long changeset) {
- this.changeset = changeset;
- }
-
- @Override
- public int getUserId() {
- return userId;
- }
-
- public void setUserId(int userId) {
- this.userId = userId;
- }
-
- @Override
- public int[] getTags() {
- return tags;
- }
-
- public void setTags(int[] tags) {
- this.tags = tags;
- }
-
- @Override
- public String toString() {
- return asString();
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMNode.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMNode.java
deleted file mode 100644
index 4c8ed8ed1..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMNode.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
-
-public class MutableOSMNode extends MutableOSMEntity implements OSMNode {
- private long longitude;
- private long latitude;
-
- @Override
- public long getLon() {
- return longitude;
- }
-
- public void setLon(long longitude) {
- this.longitude = longitude;
- }
-
- @Override
- public long getLat() {
- return latitude;
- }
-
- public void setLat(long latitude) {
- this.latitude = latitude;
- }
-
- public void setExtension(long longitude, long latitude) {
- this.longitude = longitude;
- this.latitude = latitude;
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMRelation.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMRelation.java
deleted file mode 100644
index d0d6999ce..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMRelation.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
-
-public class MutableOSMRelation extends MutableOSMEntity implements OSMRelation {
-
- private OSMMember[] members;
-
- @Override
- public OSMMember[] getMembers() {
- return members;
- }
-
- public void setExtension(OSMMember[] members) {
- this.members = members;
- }
-
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMWay.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMWay.java
deleted file mode 100644
index 878004cd9..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/MutableOSMWay.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
-
-public class MutableOSMWay extends MutableOSMEntity implements OSMWay {
-
- private OSMMember[] members;
-
- @Override
- public OSMMember[] getMembers() {
- return members;
- }
-
- public void setExtension(OSMMember[] members) {
- this.members = members;
- }
-}
diff --git a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSH.java b/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSH.java
deleted file mode 100644
index d1bb2793a..000000000
--- a/oshdb-etl/src/main/java/org/heigit/ohsome/oshdb/tool/importer/transform/oshdb/OSH.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.heigit.ohsome.oshdb.tool.importer.transform.oshdb;
-
-import java.util.Iterator;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.heigit.ohsome.oshdb.osm.OSMEntity;
-import org.heigit.ohsome.oshdb.osm.OSMType;
-
-/**
- * The root interface in the osh hierarchy.
- */
-public interface OSH extends Iterable {
-
- /**
- * Return the id of the osh object.
- *
- * @return the id of osh object
- */
- long getId();
-
- /**
- * Returns the type of this osh object.
- *
- * @return type
- */
- OSMType type();
-
- @Override
- Iterator