forked from dlt-hub/dlt
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
129 lines (104 loc) · 5 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
.PHONY: install-poetry build-library-prerelease has-poetry dev lint test test-common reset-test-storage recreate-compiled-deps build-library-prerelease publish-library
PYV=$(shell python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)")
.SILENT:has-poetry
# read version from package
# AUTV=$(shell cd dlt && python3 -c "from __version__ import __version__;print(__version__)")
# NAME := dlthub/dlt
# TAG := $(shell git log -1 --pretty=%h)
# IMG := ${NAME}:${TAG}
# LATEST := ${NAME}:latest${VERSION_SUFFIX}
# VERSION := ${AUTV}${VERSION_SUFFIX}
# VERSION_MM := ${AUTVMINMAJ}${VERSION_SUFFIX}
help:
@echo "make"
@echo " install-poetry"
@echo " installs newest poetry version"
@echo " dev"
@echo " prepares development env"
@echo " lint"
@echo " runs flake and mypy"
@echo " test"
@echo " tests all the components including destinations"
@echo " test-load-local"
@echo " tests all components using local destinations: duckdb and postgres"
@echo " test-common"
@echo " tests common components"
@echo " lint-and-test-snippets"
@echo " tests and lints snippets and examples in docs"
@echo " build-library"
@echo " makes dev and then builds dlt package for distribution"
@echo " publish-library"
@echo " builds library and then publishes it to pypi"
install-poetry:
ifneq ($(VIRTUAL_ENV),)
$(error you cannot be under virtual environment $(VIRTUAL_ENV))
endif
curl -sSL https://install.python-poetry.org | python3 -
has-poetry:
poetry --version
dev: has-poetry
poetry install --all-extras --with docs,providers,pipeline,sources,sentry-sdk
lint:
./tools/check-package.sh
poetry run python ./tools/check-lockfile.py
poetry run mypy --config-file mypy.ini dlt tests
poetry run flake8 --max-line-length=200 dlt
poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases
poetry run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
# poetry run isort ./ --diff
# $(MAKE) lint-security
format:
poetry run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
# poetry run isort ./
lint-snippets:
cd docs/tools && poetry run python check_embedded_snippets.py full
lint-and-test-snippets: lint-snippets
poetry run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed
poetry run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo
cd docs/website/docs && poetry run pytest --ignore=node_modules
lint-and-test-examples:
cd docs/tools && poetry run python prepare_examples_tests.py
poetry run flake8 --max-line-length=200 docs/examples
poetry run mypy --config-file mypy.ini docs/examples
cd docs/examples && poetry run pytest
test-examples:
cd docs/examples && poetry run pytest
lint-security:
poetry run bandit -r dlt/ -n 3 -l
test:
poetry run pytest tests
test-load-local:
DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data DESTINATION__DUCKDB__CREDENTIALS=duckdb:///_storage/test_quack.duckdb poetry run pytest tests -k '(postgres or duckdb)'
test-common:
poetry run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
reset-test-storage:
-rm -r _storage
mkdir _storage
python3 tests/tools/create_storages.py
build-library: dev
poetry version
poetry build
publish-library: build-library
poetry publish
test-build-images: build-library
# NOTE: poetry export does not work with our many different deps, we install a subset and freeze
# poetry export -f requirements.txt --output _gen_requirements.txt --without-hashes --extras gcp --extras redshift
poetry install --no-interaction -E gcp -E redshift -E duckdb
poetry run pip freeze > _gen_requirements.txt
# filter out libs that need native compilation
grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
preprocess-docs:
# run docs preprocessing to run a few checks and ensure examples can be parsed
cd docs/website && npm i && npm run preprocess-docs
start-test-containers:
docker compose -f "tests/load/dremio/docker-compose.yml" up -d
docker compose -f "tests/load/postgres/docker-compose.yml" up -d
docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d
docker compose -f "tests/load/sqlalchemy/docker-compose.yml" up -d
update-cli-docs:
poetry run dlt --debug render-docs docs/website/docs/reference/command-line-interface.md
check-cli-docs:
poetry run dlt --debug render-docs docs/website/docs/reference/command-line-interface.md --compare