[core] Add KlioWriteToText missing fields including file_name_suffix,…

… num_shards, append_trailing_newlines etc
spotify · Aug 4, 2021 · e34273a · e34273a
1 parent c411d25
commit e34273a
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 15 deletions.
diff --git a/core/setup.py b/core/setup.py
@@ -151,6 +151,7 @@ def get_long_description(package_dir):
 }
 META_FILE = read(META_PATH)
 INSTALL_REQUIRES = [
+    "apache-beam[gcp]",
     "click",
     "glom",
     "google-api-python-client",

diff --git a/core/tests/config/test_config.py b/core/tests/config/test_config.py
@@ -248,9 +248,7 @@ def no_gcp_config_dict(job_config_dict, empty_pipeline_config_dict):
 
 @pytest.mark.parametrize("blocking", (True, False, None))
 def test_klio_job_config(
-    job_config_dict,
-    blocking,
-    final_job_config_dict,
+    job_config_dict, blocking, final_job_config_dict,
 ):
     if blocking is None:
         job_config_dict.pop("blocking")
@@ -338,8 +336,7 @@ def test_bare_klio_pipeline_config(bare_pipeline_config_dict):
 
 
 def test_klio_pipeline_config(
-    pipeline_config_dict,
-    final_pipeline_config_dict,
+    pipeline_config_dict, final_pipeline_config_dict,
 ):
 
     config_obj = config.KlioPipelineConfig(
@@ -439,10 +436,7 @@ def test_klio_read_file_config():
     "config_dict",
     (
         # The minimum inputs
-        {
-            "type": "GCS",
-            "location": "gs://sigint-output/test-parent-job-out",
-        },
+        {"type": "GCS", "location": "gs://sigint-output/test-parent-job-out",},
         # Other fields configured
         {
             "type": "GCS",
@@ -525,11 +519,7 @@ def test_klio_write_bigquery_config_raises(schema):
 
 @pytest.mark.parametrize(
     "include_topic,include_subscription",
-    (
-        (False, True),
-        (True, False),
-        (True, True),
-    ),
+    ((False, True), (True, False), (True, True),),
 )
 def test_pubsub_event_input_kwargs(include_topic, include_subscription):
     config_dict = {

diff --git a/core/tox.ini b/core/tox.ini
@@ -1,9 +1,15 @@
 [tox]
-envlist = py36,py37,py38,docs,manifest,check-formatting,lint
+envlist = {py36,py37,py38}-beam{26,27,28},docs,manifest,check-formatting,lint
 
 [testenv]
 install_command = python -m pip install {opts} {packages}
 extras = tests
+deps =
+    {toxinidir}/../core
+    beam26: apache-beam[gcp]>=2.26.0,<2.27.0
+    beam27: apache-beam[gcp]>=2.27.0,<2.28.0
+    beam28: apache-beam[gcp]>=2.28.0,<2.29.0
+
 commands =
     coverage run -m pytest {posargs}
     ; assert that protos are compiled
@@ -85,6 +91,8 @@ filterwarnings =
     ignore:invalid escape sequence:DeprecationWarning
     ; 3rd party libraries haven't updated their use of collections.abc (py37+)
     ignore:Using or importing the ABCs from:DeprecationWarning
+    ; Apache Beam-related warnings
+    ignore:Running the Apache Beam SDK on Python 3:UserWarning
 
 ; required for mapping envs -> github runtimes
 [gh-actions]