-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
executable file
·82 lines (57 loc) · 3.66 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
####################################################################################################################
# Setup containers to run Airflow
get-data:
rm -rf ./data && rm -rf data.zip* && wget https://start-data-engg.s3.amazonaws.com/data.zip && unzip -o data.zip && chmod -R u=rwx,g=rwx,o=rwx data
docker-spin-up:
docker compose --env-file env up airflow-init && docker compose --env-file env up --build -d
perms:
mkdir -p logs plugins temp && sudo chmod -R u=rwx,g=rwx,o=rwx logs plugins temp dags tests migrations spectrum_tables
up: get-data perms docker-spin-up
down:
docker compose down
sh:
docker exec -ti webserver bash
####################################################################################################################
# Testing, auto formatting, type checks, & Lint checks
pytest:
docker exec webserver pytest -p no:warnings -v /opt/airflow/tests
format:
docker exec webserver python -m black -S --line-length 79 .
isort:
docker exec webserver isort .
type:
docker exec webserver mypy --ignore-missing-imports /opt/airflow
lint:
docker exec webserver flake8 /opt/airflow/dags
ci: isort format type lint pytest
####################################################################################################################
# Set up cloud infrastructure
tf-init:
terraform -chdir=./terraform init
infra-up:
terraform -chdir=./terraform apply
infra-down:
terraform -chdir=./terraform destroy
infra-config:
terraform -chdir=./terraform output
####################################################################################################################
# Create tables in Warehouse
spectrum-migration:
./spectrum_migrate.sh
db-migration:
@read -p "Enter migration name:" migration_name; docker exec webserver yoyo new ./migrations -m "$$migration_name"
redshift-migration:
docker exec -ti webserver yoyo apply --no-config-file --database redshift://$$(terraform -chdir=./terraform output -raw redshift_user):$$(terraform -chdir=./terraform output -raw redshift_password)@$$(terraform -chdir=./terraform output -raw redshift_dns_name):5439/dev ./migrations
redshift-rollback:
docker exec -ti webserver yoyo rollback --no-config-file --database redshift://$$(terraform -chdir=./terraform output -raw redshift_user):$$(terraform -chdir=./terraform output -raw redshift_password)@$$(terraform -chdir=./terraform output -raw redshift_dns_name):5439/dev ./migrations
warehouse-data-migration: spectrum-migration redshift-migration
####################################################################################################################
# Port forwarding to local machine
cloud-metabase:
terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o "IdentitiesOnly yes" -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) -N -v -f -L 3000:$$(terraform -chdir=./terraform output -raw ec2_public_dns):3000 && rm private_key.pem
cloud-airflow:
terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o "IdentitiesOnly yes" -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) -N -v -f -L 8080:$$(terraform -chdir=./terraform output -raw ec2_public_dns):8080 && rm private_key.pem
####################################################################################################################
# Helpers
ssh-ec2:
terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) && rm private_key.pem