Merge branch 'master' into develop

.github/workflows/audio.yml

@@ -7,12 +7,14 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'audio/**'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'audio/**'
 

.github/workflows/cli.yml

@@ -7,13 +7,15 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since CLI depends on core, we should run tests when core changes
       - 'cli/**'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since CLI depends on core, we should run tests when core changes
       - 'cli/**'
@@ -81,6 +83,9 @@ jobs:
       - uses: "actions/setup-python@v1"
         with:
           python-version: "3.7"
+      - name: "Install dependency: klio_core"
+        run: "python -m pip install -e ."
+        working-directory: core
       - name: "Install in dev mode"
         run: "python -m pip install -e .[dev]"
       - name: "Import package"

.github/workflows/core.yml

@@ -7,12 +7,14 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'
 

.github/workflows/dependencies.yml

@@ -8,6 +8,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'devtools/**'
@@ -17,6 +18,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'devtools/**'

.github/workflows/devtools.yml

@@ -7,12 +7,14 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'devtools/**'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'devtools/**'
 

.github/workflows/docs.yml

@@ -7,13 +7,15 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'docs/**'
       - '**.rst'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'docs/**'
       - '**.rst'

.github/workflows/exec.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since exec depends on core, we should run tests when core changes
       - 'lib/**'  # since exec depends on lib, we should run tests when lib changes
@@ -15,6 +16,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since exec depends on core, we should run tests when core changes
       - 'lib/**'  # since exec depends on lib, we should run tests when lib changes
@@ -83,6 +85,12 @@ jobs:
       - uses: "actions/setup-python@v1"
         with:
           python-version: "3.7"
+      - name: "Install dependency: klio_core"
+        run: "python -m pip install -e ."
+        working-directory: core
+      - name: "Install dependency: klio"
+        run: "python -m pip install -e ."
+        working-directory: lib
       - name: "Install in dev mode"
         run: "python -m pip install -e .[dev]"
       - name: "Import package"

.github/workflows/integration.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'devtools/**'
@@ -14,6 +15,7 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths-ignore:
       - 'docs/**'
       - 'devtools/**'

.github/workflows/lib.yml

@@ -7,13 +7,15 @@ on:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since lib depends on core, we should run tests when core changes
       - 'lib/**'
   pull_request:
     branches:
       - master
       - develop
+      - release-*
     paths:
       - 'core/**'  # since lib depends on core, we should run tests when core changes
       - 'lib/**'
@@ -81,6 +83,9 @@ jobs:
       - uses: "actions/setup-python@v1"
         with:
           python-version: "3.7"
+      - name: "Install dependency: klio_core"
+        run: "python -m pip install -e ."
+        working-directory: core
       - name: "Install in dev mode"
         run: "python -m pip install -e .[dev]"
       - name: "Import package"

cli/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.5
+current_version = 21.2.0
 tag_name = cli-{new_version}
 commit = True
 tag = True

cli/setup.py

@@ -151,7 +151,7 @@ def get_long_description(package_dir):
 }
 META_FILE = read(META_PATH)
 INSTALL_REQUIRES = [
-    "klio-core>=0.2.0",
+    "klio-core>=21.2.0",
     "click",
     "dateparser",
     "docker",

cli/src/klio_cli/__init__.py

@@ -14,7 +14,7 @@
 #
 
 __author__ = "The klio developers"
-__version__ = "1.0.5"
+__version__ = "21.2.0"
 __email__ = "[email protected]"
 __description__ = "Main entrypoint for Klio jobs"
 __uri__ = "https://github.com/spotify/klio"

cli/src/klio_cli/commands/base.py

@@ -15,12 +15,11 @@
 
 import logging
 import os
-import tempfile
 
 import docker
-import yaml
 
-from klio_cli.commands.job import configuration
+from klio_core.config import core as config_core
+
 from klio_cli.utils import docker_utils
 
 
@@ -31,8 +30,6 @@ class BaseDockerizedPipeline(object):
     CONTAINER_GCP_CRED_PATH = os.path.join("/usr", GCP_CRED_FILE)
     CONTAINER_JOB_DIR = "/usr/src/app"
     DOCKER_LOGGER_NAME = "klio.base_docker_pipeline"
-    # path where the temp config-file is mounted into klio-exec's container
-    MATERIALIZED_CONFIG_PATH = "/usr/src/config/materialized_config.yaml"
 
     def __init__(self, job_dir, klio_config, docker_runtime_config):
         self.job_dir = job_dir
@@ -44,7 +41,6 @@ def __init__(self, job_dir, klio_config, docker_runtime_config):
         # if this is set to true, running the command will generate a temp file
         # and mount it to the container
         self.requires_config_file = True
-        self.materialized_config_file = None
 
     @property
     def _full_image_name(self):
@@ -102,12 +98,6 @@ def _get_volumes(self):
             },
         }
 
-        if self.materialized_config_file is not None:
-            volumes[self.materialized_config_file.name] = {
-                "bind": BaseDockerizedPipeline.MATERIALIZED_CONFIG_PATH,
-                "mode": "rw",
-            }
-
         return volumes
 
     def _get_command(self, *args, **kwargs):
@@ -118,7 +108,10 @@ def _add_base_args(self, command):
             command.extend(
                 [
                     "--config-file",
-                    BaseDockerizedPipeline.MATERIALIZED_CONFIG_PATH,
+                    os.path.join(
+                        self.CONTAINER_JOB_DIR,
+                        config_core.RUN_EFFECTIVE_CONFIG_FILE,
+                    ),
                 ]
             )
         return command
@@ -178,16 +171,10 @@ def _check_gcp_credentials_exist(self):
 
     def _write_effective_config(self):
         if self.requires_config_file:
-            self.materialized_config_file = tempfile.NamedTemporaryFile(
-                prefix="/tmp/", mode="w", delete=False
-            )
-            yaml.dump(
-                self.klio_config.as_dict(),
-                stream=self.materialized_config_file,
-                Dumper=configuration.IndentListDumper,
-                default_flow_style=False,
-                sort_keys=False,
+            path = os.path.join(
+                self.job_dir, config_core.RUN_EFFECTIVE_CONFIG_FILE
             )
+            self.klio_config.write_to_file(path)
 
     def run(self, *args, **kwargs):
         # bail early

cli/src/klio_cli/commands/job/configuration.py

@@ -21,29 +21,7 @@
 
 from klio_core import config as kconfig
 from klio_core import utils as core_utils
-
-
-class IndentListDumper(yaml.Dumper):
-    """Force indentation for lists for better visual understanding.
-
-    Instead of this:
-
-        foo:
-          bar:
-          - one
-          - two
-          - three
-
-    Format list indentations like this:
-        foo:
-          bar:
-            - one
-            - two
-            - three
-    """
-
-    def increase_indent(self, flow=False, indentless=False):
-        return super(IndentListDumper, self).increase_indent(flow, False)
+from klio_core.config import _utils as config_utils
 
 
 class EffectiveJobConfig(object):
@@ -124,7 +102,7 @@ def show(self):
         yaml.dump(
             effective_config,
             stream=sys.stdout,
-            Dumper=IndentListDumper,
+            Dumper=config_utils.IndentListDumper,
             sort_keys=False,
         )
 
@@ -142,7 +120,7 @@ def set(self, target_to_value):
             yaml.dump(
                 effective_config,
                 stream=f,
-                Dumper=IndentListDumper,
+                Dumper=config_utils.IndentListDumper,
                 default_flow_style=False,
                 sort_keys=False,
             )
@@ -156,7 +134,7 @@ def unset(self, target):
             yaml.dump(
                 self.config_data,
                 stream=f,
-                Dumper=IndentListDumper,
+                Dumper=config_utils.IndentListDumper,
                 default_flow_style=False,
                 sort_keys=False,
             )

cli/src/klio_cli/commands/job/utils/templates/dockerfile.tpl

@@ -45,8 +45,11 @@ COPY __init__.py \
      {%- endif %}
      /usr/src/app/
 
+ARG KLIO_CONFIG=klio-job.yaml
 {% if not klio.use_fnapi -%}
+COPY $KLIO_CONFIG klio-job-run-effective.yaml
+
 RUN pip install .
-{% endif -%}
-ARG KLIO_CONFIG=klio-job.yaml
-COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
+{%- else -%}
+COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
+{% endif -%}

cli/src/klio_cli/commands/job/utils/templates/setup.py.tpl

@@ -126,7 +126,7 @@ setuptools.setup(
         #    str(dir where to install files, relative to Python modules),
         #    list(str(non-Python filenames))
         # )
-        (".", ["klio-job.yaml"]),
+        (".", ["klio-job-run-effective.yaml"]),
     ],
     include_package_data=True,  # required
     # NOTE: Explicitly include Python modules names (all relevant Python files

cli/tests/commands/job/utils/fixtures/expected/fnapi/Dockerfile

@@ -33,4 +33,4 @@ COPY __init__.py \
      /usr/src/app/
 
 ARG KLIO_CONFIG=klio-job.yaml
-COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
+COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml

cli/tests/commands/job/utils/fixtures/expected/no_fnapi/Dockerfile

@@ -18,6 +18,7 @@ COPY __init__.py \
      # Include any other non-Python files your job needs
      /usr/src/app/
 
-RUN pip install .
 ARG KLIO_CONFIG=klio-job.yaml
-COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
+COPY $KLIO_CONFIG klio-job-run-effective.yaml
+
+RUN pip install .

cli/tests/commands/job/utils/fixtures/expected/no_fnapi/setup.py

@@ -126,7 +126,7 @@ def run(self):
         #    str(dir where to install files, relative to Python modules),
         #    list(str(non-Python filenames))
         # )
-        (".", ["klio-job.yaml"]),
+        (".", ["klio-job-run-effective.yaml"]),
     ],
     include_package_data=True,  # required
     # NOTE: Explicitly include Python modules names (all relevant Python files