Convert Makefile to use Python's Invoke project

.circle/Makefile

@@ -1,249 +1,33 @@
 ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-ifndef FORCE_CHECK_ALL_FILES
-CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files)
-CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py)
-CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml)
-CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json)
-endif
-CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
 VIRTUALENV_DIR ?= virtualenv
 ST2_REPO_PATH ?= /tmp/st2
 ST2_REPO_BRANCH ?= master
-FORCE_CHECK_ALL_FILES ?= false
-FORCE_CHECK_PACK ?= false
 
-export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK
+export ST2_REPO_PATH ST2_REPO_BRANCH ROOT_DIR
 
-# All components are prefixed by st2
-COMPONENTS := $(wildcard /tmp/st2/st2*)
-COMPONENTS_RUNNERS := $(wildcard /tmp/st2/contrib/runners/*)
-
-.PHONY: all
-all: requirements lint packs-resource-register packs-tests
-
-.PHONY: all-ci
-all-ci: compile .flake8 .pylint .copy-pack-to-subdirectory .configs-check .metadata-check .packs-resource-register .packs-tests
-
-.PHONY: lint
-lint: requirements flake8 pylint configs-check metadata-check
-
-.PHONY: flake8
-flake8: requirements .flake8
-
-.PHONY: pylint
-pylint: requirements .clone_st2_repo .pylint
-
-.PHONY: configs-check
-configs-check: requirements .clone_st2_repo .copy-pack-to-subdirectory .configs-check
-
-.PHONY: metadata-check
-metadata-check: requirements .metadata-check
-
-# Task which copies pack to temporary sub-directory so we can use old-style check scripts which
-# # require pack to be in a sub-directory
-.PHONY: .copy-pack-to-subdirectory
-.copy-pack-to-subdirectory:
-	rm -rf /tmp/packs/$(PACK_NAME)
-	mkdir -p /tmp/packs/$(PACK_NAME)
-	cp -r ./* /tmp/packs/$(PACK_NAME)
-
-.PHONY: packs-resource-register
-packs-resource-register: requirements .clone_st2_repo .copy-pack-to-subdirectory .packs-resource-register
-
-.PHONY: packs-missing-tests
-packs-missing-tests: requirements .packs-missing-tests
-
-.PHONY: packs-tests
-packs-tests: requirements .clone_st2_repo .packs-tests
-
-.PHONY: compile
-compile:
-	@echo "======================= compile ========================"
-	@echo "------- Compile all .py files (syntax check test) ------"
-	if python -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|virtualenv-osx|virtualenv-py3|.tox|.git|.venv-st2devbox"), quiet=True)' | grep .; then exit 1; else exit 0; fi
-
-.PHONY: .flake8
-.flake8:
-	@echo
-	@echo "==================== flake8 ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
-		find ./* -name "*.py" | while read py_file; do \
-			flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
-		done; \
-	elif [ -n "${CHANGED_PY}" ]; then \
-		for file in ${CHANGED_PY}; do \
-			if [ -n "$$file" ]; then \
-				flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
-			fi; \
-		done; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi;
-
-.PHONY: .pylint
-.pylint:
-	@echo
-	@echo "==================== pylint ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_PY}" ]; then \
-		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
-		CONFIG_DIR=$(CI_DIR)/lint-configs/ \
-		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi;
-
-.PHONY: .configs-check
-.configs-check:
-	@echo
-	@echo "==================== configs-check ===================="
-	@echo
-	@# The number of changed files in the AWS pack exceeds the limits of Bash,
-	@# leading to CI failures like this:
-	@# https://circleci.com/gh/StackStorm-Exchange/stackstorm-aws/320
-	@# Instead of passing the entire list into a Bash for loop, we convert the
-	@# make variable to a Bash string, convert that to a Bash array, and then
-	@# iterate through each element of the array
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
-		find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \
-			st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \
-		done; \
-	elif [ -n "${CHANGED_YAML}" ]; then \
-		for file in $(CHANGED_YAML); do \
-			if [ -n "$$file" ]; then \
-				st2-check-validate-yaml-file $$file || exit 1 ; \
-			fi; \
-		done; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi
-	@#
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
-		find $(CI_DIR)/* -name "*.json" | while read json_file; do \
-			st2-check-validate-json-file "$$json_file" || exit 1 ; \
-		done; \
-	elif [ -n "${CHANGED_JSON}" ]; then \
-		for file in $(CHANGED_JSON); do \
-			if [ -n "$$file" ]; then \
-				echo "file: $$file"; \
-				st2-check-validate-json-file $$file || exit 1 ; \
-			fi; \
-		done; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi
-	@#
-	@echo
-	@echo "==================== example config check ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_FILES}" ]; then \
-		st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi;
-
-.PHONY: .metadata-check
-.metadata-check:
-	@echo
-	@echo "==================== metadata-check ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \
-		st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \
-	else \
-		echo "No files have changed, skipping run..."; \
-	fi;
-
-.PHONY: .packs-resource-register
-.packs-resource-register:
-	@echo
-	@echo "==================== packs-resource-register ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ -z "${CHANGED_FILES}" ]; then \
-		echo "No files have changed, skipping run..."; \
-	else \
-		st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \
-	fi;
-
-.PHONY: .packs-tests
-.packs-tests:
-	@echo
-	@echo "==================== packs-tests ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ -z "${CHANGED_FILES}" ]; then \
-		echo "No files have changed, skipping run..."; \
-	else \
-		$(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \
-	fi;
-
-.PHONY: .packs-missing-tests
-.packs-missing-tests:
-	@echo
-	@echo "==================== pack-missing-tests ===================="
-	@echo
-	if [ -z "${CHANGED_FILES}" ]; then \
-		echo "No files have changed, skipping run..."; \
-	else \
-		st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \
-	fi;
-
-.PHONY: .clone_st2_repo
-.clone_st2_repo: /tmp/st2
-/tmp/st2:
-	@echo
-	@echo "==================== cloning st2 repo ===================="
-	@echo
-	@rm -rf /tmp/st2
-	@git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) /tmp/st2
-
-.PHONY: .install-runners
-.install-runners:
-	@echo ""
-	@echo "================== install runners ===================="
-	@echo ""
-	@for component in $(COMPONENTS_RUNNERS); do \
-		echo "==========================================================="; \
-		echo "Installing runner:" $$component; \
-		echo "==========================================================="; \
-        (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop); \
-	done
-	@echo ""
-	@echo "================== register metrics drivers ======================"
-	@echo ""
-
-	# Install st2common to register metrics drivers
-	(. $(VIRTUALENV_DIR)/bin/activate; cd $(ST2_REPO_PATH)/st2common; python setup.py develop)
-
-.PHONY: requirements
-requirements: virtualenv .clone_st2_repo .install-runners
-	@echo
-	@echo "==================== requirements ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
-
-.PHONY: requirements-ci
-requirements-ci:
-	@echo
-	@echo "==================== requirements-ci ===================="
-	@echo
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
-
-.PHONY: virtualenv
 virtualenv: $(VIRTUALENV_DIR)/bin/activate
 $(VIRTUALENV_DIR)/bin/activate:
 	@echo
 	@echo "==================== virtualenv ===================="
 	@echo
 	test -d $(VIRTUALENV_DIR) || virtualenv --no-site-packages $(VIRTUALENV_DIR)
+
+$(VIRTUALENV_DIR)/bin/invoke: $(VIRTUALENV_DIR)
+	. $(VIRTUALENV_DIR)/bin/activate && pip install invoke
+
+.PHONY: invoke
+invoke: virtualenv $(VIRTUALENV_DIR)/bin/invoke
+	$(VIRTUALENV_DIR)/bin/pip install invoke
+
+# https://stackoverflow.com/a/33018558
+# Workaround to support all previous make targets
+# This default target simply passes all targets on to invoke
+# We can't add invoke as a make dependency for the .DEFAULT target since the
+# dependency will get overridden by whatever target is passed in
+.DEFAULT:
+	@# Manually make virtualenv target
+	if [ ! -d $(VIRTUALENV_DIR) ]; then make virtualenv; fi
+	@# Manually make invoke target
+	if [ ! -e $(VIRTUALENV_DIR)/bin/invoke ]; then $(VIRTUALENV_DIR)/bin/pip install invoke; fi
+	. $(VIRTUALENV_DIR)/bin/activate && invoke --search-root=$(CI_DIR) $@
+	@#. $(VIRTUALENV_DIR)/bin/activate && echo $$PYTHONPATH

.circle/dependencies

@@ -49,14 +49,14 @@ if [ ! -z "${ROOT_DIR}" ]; then
 
     echo "Copying Makefile to ${ROOT_DIR}"
     cp ~/ci/.circle/Makefile ${ROOT_DIR}
-    make -C requirements-ci .install-runners
+    make -C requirements install-runners
 else
     PACK_REQUIREMENTS_FILE="$(pwd)/requirements.txt"
     PACK_TESTS_REQUIREMENTS_FILE="$(pwd)/requirements-tests.txt"
 
     echo "Copying Makefile to $(pwd)"
     cp ~/ci/.circle/Makefile .
-    make requirements-ci .install-runners
+    make requirements install-runners
 fi
 
 # Install pack requirements

tasks/__init__.py

@@ -0,0 +1,96 @@
+from invoke import Collection, task, run
+
+from . import check
+from . import copy
+from . import git_tasks
+from . import lint
+from . import requirements
+from . import tests
+
+
+# All tasks are implemented in submodules of this package
+# All tasks in this module are only for reverse compatibility with the original
+# Makefile
+
+# This task aliases a Python built-in
+@task(requirements.install, lint.lint, tests.packs_resource_register, tests.packs_tests)
+def all_(ctx):
+    pass
+
+
+@task(check.compile_, lint.flake8, lint.pylint, copy.copy_pack_to_subdirectory,
+      check.configs, check.metadata, tests.packs_resource_register, tests.packs_tests)
+def all_ci(ctx):
+    pass
+
+
+@task(lint.lint)
+def lint_(ctx):
+    pass
+
+
+@task(lint.flake8)
+def flake8(ctx):
+    pass
+
+
+@task(lint.pylint)
+def pylint(ctx):
+    pass
+
+
+@task(check.configs)
+def configs_check(ctx):
+    pass
+
+
+@task(check.metadata)
+def metadata_check(ctx):
+    pass
+
+
+@task(tests.packs_resource_register)
+def packs_resource_register(ctx):
+    pass
+
+
+@task(tests.packs_missing_tests)
+def packs_missing_tests(ctx):
+    pass
+
+
+@task(tests.packs_tests)
+def packs_tests(ctx):
+    pass
+
+
+@task(check.compile_)
+def compile_(ctx):
+    pass
+
+
+@task(requirements.runners)
+def install_runners(ctx):
+    pass
+
+
+@task(requirements.install)
+def requirements(ctx):
+    pass
+
+
+namespace = Collection()
+
+namespace.add_task(all_, name='all')
+namespace.add_task(all_ci, name='all-ci')
+namespace.add_task(lint_, name='lint')
+namespace.add_task(flake8, name='flake8')
+namespace.add_task(pylint, name='pylint')
+namespace.add_task(configs_check, name='configs-check')
+namespace.add_task(metadata_check, name='metadata-check')
+namespace.add_task(packs_resource_register, name='packs-resource-register')
+namespace.add_task(packs_missing_tests, name='packs-missing-tests')
+namespace.add_task(packs_tests, name='packs-tests')
+namespace.add_task(compile_, name='compile')
+namespace.add_task(install_runners, name='install-runners')
+namespace.add_task(requirements, name='requirements')