adding staging and intermediate models with test #81
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate and Run dbt Workflow | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - '**' | |
| env: | |
| HOST_PREPROD: ${{ secrets.HOST_PREPROD }} | |
| PASSWORD_PREPROD: ${{ secrets.PASSWORD_PREPROD }} | |
| PORT_PREPROD: ${{ secrets.PORT_PREPROD }} | |
| TARGET_SCHEMA: ${{ secrets.SCHEMA_PROD }} | |
| USER_PREPROD: ${{ secrets.USER_PREPROD }} | |
| jobs: | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Generate matrix | |
| id: set-matrix | |
| run: | | |
| project_dirs=$(find projects -maxdepth 1 -mindepth 1 -type d -not -name 'demo' -not -name 'packages') | |
| matrix=$(echo "$project_dirs" | jq -R -s -c 'split("\n") | map(select(length > 0)) | {project_dir: .}') | |
| echo "matrix=$matrix" >> $GITHUB_ENV | |
| echo "::set-output name=matrix::$matrix" | |
| run-dbt-tests: | |
| needs: generate-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@master | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.x" | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| - name: Read db_secret from config.yml | |
| id: read-config | |
| run: | | |
| config_file="${{ matrix.project_dir }}/config.yml" | |
| db_secret=$(grep 'db_name_prod:' "$config_file" | awk '{print $2}') | |
| participation_host_name=$(grep 'participation_host_name:' "$config_file" | awk '{print $2}') | |
| echo "DB_SECRET_NAME=$db_secret" >> $GITHUB_ENV | |
| echo "PARTICIPATION_HOST_NAME=$participation_host_name" >> $GITHUB_ENV | |
| - name: Configure db name and participation host name | |
| run: | | |
| echo "DBNAME=${{ secrets[env.DB_SECRET_NAME] }}" >> $GITHUB_ENV | |
| echo "PARTICIPATION_HOST_NAME=${{ secrets[env.PARTICIPATION_HOST_NAME] }}" >> $GITHUB_ENV | |
| - name: Run dbt test | |
| run: | | |
| echo "Switching to project directory: ${{ matrix.project_dir }}" | |
| cd ${{ matrix.project_dir }} | |
| echo "Running dbt deps" | |
| dbt deps --target preprod | |
| echo "Running dbt run" | |
| dbt run --target preprod | |
| echo "Running dbt test" | |
| dbt test --target preprod |