Merge pull request #301 from tilezen/zerebubuth/add-extra-data-tables

Add extra data tables

config.yaml.sample

@@ -326,5 +326,39 @@ rawr:
       planet_osm_polygon: *osm
       planet_osm_ways: *osm
       planet_osm_rels: *osm
+      wof_neighbourhood: { name: wof, value: whosonfirst.mapzen.com }
+      water_polygons: &osmdata { name: shp, value: openstreetmapdata.com }
+      land_polygons: *osmdata
+  # when a feature's shape is of the type given in the key and the feature
+  # appears in the listed layers, then generate a label centroid. multi*
+  # geometries are considered the same as single ones for the purposes of key
+  # lookup.
+  label-placement-layers:
+    point: ['earth', 'water']
+    polygon: ['buildings', 'earth', 'landuse', 'water']
+    linestring: ['earth', 'landuse', 'water']
+  # indexes to generate on the RAWR tile. features are looked up only if they
+  # appear in an index, so this list must be exhaustive. all OSM-based features
+  # are handled by a single index of {type: osm}. other indexes are provided on
+  # single table/layer combinations with the {type: simple}, providing the
+  # names of both the table and layer. it's also possible to provide start_zoom
+  # and end_zoom parameters which control when the data is visible. note that
+  # the start_zoom is inclusive (and optional; defaults to 0) and the end_zoom
+  # is exclusive (also optional; defaults to infinity).
+  indexes:
+    - type: osm
+    - type: simple
+      table: wof_neighbourhood
+      layer: places
+    - type: simple
+      table: water_polygons
+      layer: water
+    - type: simple
+      table: land_polygons
+      layer: earth
+    - type: simple
+      table: ne_10m_urban_areas
+      layer: landuse
+      end_zoom: 10
 # uncomment this to use RAWR tiles rather than go direct to the database.
 #use-rawr-tiles: true

tests/test_query_rawr.py

@@ -44,8 +44,9 @@ def _make(self, min_zoom_fn, props_fn, tables, tile_pyramid,
 
         layers = {layer_name: LayerInfo(min_zoom_fn, props_fn)}
         storage = ConstantStorage(tables)
+        indexes_cfg = [dict(type="osm")]
         return make_rawr_data_fetcher(
-            min_z, max_z, storage, layers,
+            min_z, max_z, storage, layers, indexes_cfg,
             label_placement_layers=label_placement_layers)
 
 
@@ -482,7 +483,7 @@ def props_fn(shape, props, fid, meta):
             layers[name] = LayerInfo(min_zoom_fn, props_fn)
         storage = ConstantStorage(tables)
         fetch = make_rawr_data_fetcher(
-            top_zoom, max_zoom, storage, layers)
+            top_zoom, max_zoom, storage, layers, [dict(type="osm")])
 
         for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
             read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile))

tilequeue/command.py

@@ -1725,7 +1725,8 @@ def tilequeue_rawr_process(cfg, peripherals):
     from raw_tiles.formatter.msgpack import Msgpack
     from raw_tiles.gen import RawrGenerator
     from raw_tiles.source.conn import ConnectionContextManager
-    from raw_tiles.source.osm import OsmSource
+    from raw_tiles.source import parse_sources
+    from raw_tiles.source import DEFAULT_SOURCES as DEFAULT_RAWR_SOURCES
     from tilequeue.log import JsonRawrProcessingLogger
     from tilequeue.rawr import RawrS3Sink
     from tilequeue.rawr import RawrStoreSink
@@ -1737,6 +1738,12 @@ def tilequeue_rawr_process(cfg, peripherals):
     # pass through the postgresql yaml config directly
     conn_ctx = ConnectionContextManager(rawr_postgresql_yaml)
 
+    rawr_source_list = rawr_yaml.get('sources', DEFAULT_RAWR_SOURCES)
+    assert isinstance(rawr_source_list, list), \
+        'RAWR source list should be a list'
+    assert len(rawr_source_list) > 0, \
+        'RAWR source list should be non-empty'
+
     rawr_store = rawr_yaml.get('store')
     if rawr_store:
         store = make_store(rawr_store,
@@ -1779,15 +1786,15 @@ def tilequeue_rawr_process(cfg, peripherals):
             % (toi_type,)
 
     logger = make_logger(cfg, 'rawr_process')
-    rawr_osm_source = OsmSource(conn_ctx)
+    rawr_source = parse_sources(rawr_source_list)
     rawr_formatter = Msgpack()
-    rawr_gen = RawrGenerator(rawr_osm_source, rawr_formatter, rawr_sink)
+    rawr_gen = RawrGenerator(rawr_source, rawr_formatter, rawr_sink)
     stats_handler = RawrTilePipelineStatsHandler(peripherals.stats)
     rawr_proc_logger = JsonRawrProcessingLogger(logger)
     rawr_pipeline = RawrTileGenerationPipeline(
             rawr_queue, msg_marshaller, group_by_zoom, rawr_gen,
             peripherals.queue_writer, rawr_toi_intersector, stats_handler,
-            rawr_proc_logger)
+            rawr_proc_logger, conn_ctx)
     rawr_pipeline()
 
 

tilequeue/query/__init__.py

@@ -72,6 +72,16 @@ def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool):
         source_value = data['value']
         table_sources[tbl] = Source(source_name, source_value)
 
+    label_placement_layers = rawr_yaml.get('label-placement-layers', {})
+    for geom_type, layers in label_placement_layers.items():
+        assert geom_type in ('point', 'polygon', 'linestring'), \
+            'Geom type %r not understood, expecting point, polygon or ' \
+            'linestring.' % (geom_type,)
+        label_placement_layers[geom_type] = set(layers)
+
+    indexes_cfg = rawr_yaml.get('indexes')
+    assert indexes_cfg, 'Missing definitions of table indexes.'
+
     # source types are:
     #   s3       - to fetch RAWR tiles from S3
     #   store    - to fetch RAWR tiles from any tilequeue tile source
@@ -124,20 +134,15 @@ def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool):
         assert False, 'Source type %r not understood. ' \
             'Options are s3, generate and store.' % (source_type,)
 
-    # TODO: this needs to be configurable, everywhere!
+    # TODO: this needs to be configurable, everywhere! this is a long term
+    # refactor - it's hard-coded in a bunch of places :-(
     max_z = 16
 
-    # TODO: put this in the config!
-    label_placement_layers = {
-        'point': set(['earth', 'water']),
-        'polygon': set(['buildings', 'earth', 'landuse', 'water']),
-        'linestring': set(['earth', 'landuse', 'water']),
-    }
-
     layers = _make_layer_info(layer_data, cfg.process_yaml_cfg)
 
     return make_rawr_data_fetcher(
-        group_by_zoom, max_z, storage, layers, label_placement_layers)
+        group_by_zoom, max_z, storage, layers, indexes_cfg,
+        label_placement_layers)
 
 
 def _make_layer_info(layer_data, process_yaml_cfg):