Adding private fields

.gitignore

@@ -1,6 +1,7 @@
 .vscode/
 .user.yml
 .env
+.venv
 .idea
 .DS_Store
 package-lock.yml

projects/lyon/models/intermediate/decidim_awesome/schema.yml

@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: int_decidim_awesome_private_proposal_fields
+    description: "Model for parsing the information in the private proposals fields and unnest the XML data into readable questions and answers columns."
+    columns:
+      - name: id
+        description: "Primary key."
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+      - name: private_field_description
+        description: "Description of the private field to be answered."
+      - name: private_field_content
+        description: "The content filled by the user for a specific private field. There can be multiple content for the same private field description."
+
+
+unit_tests:
+
+   - name: testing_that_the_correct_xml_information_are_parsed
+     description: "Checks that age matching fetches the correct infos "
+     model: int_decidim_awesome_proposal_extra_fields
+     given:
+      - input: ref('stg_decidim_awesome_proposal_extra_fields')
+        format: dict
+        rows:
+          - {id: 1, proposal_id: 101, private_body_clear: "<xml><dl class=\"decidim_awesome-custom_fields\" data-generator=\"decidim_awesome\" data-version=\"0.10.3\"><dt name=\"radio-group-1725003392539-0\">Cette idée est déposée à titre :</dt><dd id=\"radio-group-1725003392539-0\" name=\"radio-group\"><div alt=\"individuel\">Individuel</div></dd><dt name=\"radio-group-1725003468065-0\">Votre tranche d'âge :</dt><dd id=\"radio-group-1725003468065-0\" name=\"radio-group\"><div alt=\"1625\">16-25 ans</div></dd></dl></xml>"}
+     expect:
+      format: dict
+      rows:
+        - {id: 1, proposal_id: 101, private_field_description: "Cette idée est déposée à titre :", private_field_content: "Individuel"}
+        - {id: 1, proposal_id: 101, private_field_description: "Votre tranche d'âge :", private_field_content: "16-25 ans"}
+

projects/lyon/models/intermediate/int_decidim_awesome_proposal_extra_fields.sql

@@ -0,0 +1,19 @@
+WITH parsed_data AS (
+    SELECT
+        decidim_awesome_proposal_extra_fields.id,
+        decidim_awesome_proposal_extra_fields.proposal_id,
+        xpath('//text()', unnest(xpath('//dt', xml_data)))::text AS private_field_description,
+        unnest(xpath('//dd/div/text()', unnest(xpath('//dd', xml_data))))::text AS private_field_content
+    FROM
+        {{ ref ("stg_decidim_awesome_proposal_extra_fields")}} AS decidim_awesome_proposal_extra_fields,
+        LATERAL xmlparse(document private_body_clear) AS xml_data
+)
+SELECT
+    parsed_data.id,
+    parsed_data.proposal_id,
+    replace(replace(private_field_description, '{"', ''), '"}', '') AS private_field_description,
+    parsed_data.private_field_content
+FROM
+    parsed_data
+WHERE
+    private_field_content IS NOT NULL

projects/lyon/models/marts/decidim_awesome/decidim_awesome_proposal_extra_fields.sql

@@ -0,0 +1,11 @@
+SELECT
+    decidim_awesome_proposal_extra_fields.id,
+    decidim_awesome_proposal_extra_fields.proposal_id,
+    decidim_awesome_proposal_extra_fields.private_field_description,
+    decidim_awesome_proposal_extra_fields.private_field_content,
+    decidim_proposals_proposals.decidim_component_id,
+    components.ps_title
+FROM
+    {{ ref ("int_decidim_awesome_proposal_extra_fields")}} AS decidim_awesome_proposal_extra_fields
+JOIN {{ ref ("stg_decidim_proposals")}} AS decidim_proposals_proposals ON decidim_awesome_proposal_extra_fields.proposal_id = decidim_proposals_proposals.id
+JOIN {{ ref ("components")}} AS components ON decidim_proposals_proposals.decidim_component_id = components.id

projects/lyon/models/staging/decidim_awesome/schema.yml

@@ -0,0 +1,25 @@
+version: 2
+
+
+sources:
+  - name: decidim
+    database: "{{ env_var('DBNAME') }}"
+    schema: public
+    tables:
+      - name: decidim_awesome_proposal_extra_fields_test
+
+models:
+  - name: stg_decidim_awesome_proposal_extra_fields
+    description: "Table for private proposal fields in the module decidim awesome."
+    columns:
+      - name: id
+        description: "Primary key."
+        data_tests:
+          - not_null
+          - unique
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+        data_tests:
+          - not_null
+      - name: private_body
+        description: "Body of the answers of the private proposals fields answered by the user. The format is XML."

projects/lyon/models/staging/decidim_awesome/stg_decidim_awesome_proposal_extra_fields.sql

@@ -0,0 +1,12 @@
+WITH source AS (
+      SELECT * FROM {{ source('decidim', 'decidim_awesome_proposal_extra_fields_test') }}
+)
+
+
+SELECT
+    id,
+    decidim_proposal_id AS proposal_id,
+    updated_at,
+    created_at,
+    replace(private_body_clear, ' ', '') AS private_body_clear
+FROM source

projects/test_lyon/dbt_packages/dbt_utils/.circleci/config.yml

@@ -0,0 +1,102 @@
+
+version: 2.1
+
+jobs:
+
+  integration-postgres:
+    docker:
+      - image: cimg/python:3.9
+      - image: cimg/postgres:9.6
+        environment:
+          POSTGRES_USER: root
+    environment:
+      POSTGRES_TEST_HOST: localhost
+      POSTGRES_TEST_USER: root
+      POSTGRES_TEST_PASS: ''
+      POSTGRES_TEST_PORT: 5432
+      POSTGRES_TEST_DBNAME: circle_test
+
+    steps:
+      - checkout
+      - run: pip install --pre dbt-postgres -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Postgres"
+          command: ./run_test.sh postgres
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-redshift:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-redshift -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Redshift"
+          command: ./run_test.sh redshift
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-snowflake:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-snowflake -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Snowflake"
+          command: ./run_test.sh snowflake
+      - store_artifacts:
+          path: integration_tests/logs          
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-bigquery:
+    environment:
+      BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-bigquery -r dev-requirements.txt
+      - run:
+          name: "Set up credentials"
+          command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
+      - run:
+          name: "Run OG Tests - BigQuery"
+          command: ./run_test.sh bigquery
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - integration-postgres
+      - integration-redshift:
+          context: profile-redshift
+          requires:
+            - integration-postgres
+      - integration-snowflake:
+          context: profile-snowflake
+          requires:
+            - integration-postgres
+      - integration-bigquery:
+          context: profile-bigquery
+          requires:
+            - integration-postgres

projects/test_lyon/dbt_packages/dbt_utils/.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @clrcrl

projects/test_lyon/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,60 @@
+---
+name: Bug report
+about: Report a bug or an issue you've found with this package
+title: ''
+labels: bug, triage
+assignees: ''
+
+---
+
+### Describe the bug
+<!---
+A clear and concise description of what the bug is. You can also use the issue title to do this
+--->
+
+### Steps to reproduce
+<!---
+In as much detail as possible, please provide steps to reproduce the issue. Sample data that triggers the issue, example model code, etc is all very helpful here.
+--->
+
+### Expected results
+<!---
+A clear and concise description of what you expected to happen.
+--->
+
+### Actual results
+<!---
+A clear and concise description of what you expected to happen.
+--->
+
+### Screenshots and log output
+<!---
+If applicable, add screenshots or log output to help explain your problem.
+--->
+
+### System information
+**The contents of your `packages.yml` file:**
+
+**Which database are you using dbt with?**
+- [ ] postgres
+- [ ] redshift
+- [ ] bigquery
+- [ ] snowflake
+- [ ] other (specify: ____________)
+
+
+**The output of `dbt --version`:**
+```
+<output goes here>
+```
+
+
+### Additional context
+<!---
+Add any other context about the problem here. For example, if you think you know which line of code is causing the issue.
+--->
+
+### Are you interested in contributing the fix?
+<!---
+Let us know if you want to contribute the fix, and whether would need a hand getting started
+--->

projects/test_lyon/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/dbt_minor_release.md

@@ -0,0 +1,29 @@
+---
+name: dbt Minor Release Follow-Up
+about: A checklist of tasks to complete after a minor release is made to dbt
+title: 'dbt Minor Release Follow up for dbt v0.x.0'
+labels:
+assignees: ''
+---
+
+<!---
+This template is to be used once a new dbt minor release is available on pypi.
+In the future, we will consider doing pre-releases.
+-->
+
+First, check if this is a breaking change
+- [ ] Increase the upper bound of the `require-dbt-version` config in the `dbt_project.yml`
+- [ ] Increase the upper bound of the dbt version in `run_test.sh`
+- [ ] Create a PR against the `main` branch to see if tests pass
+
+If test pass, this is _not_ a breaking change. You should:
+- [ ] Merge into `main`
+- [ ] Create a patch release
+
+If tests fail, this _is_ a breaking change. You'll need to create a minor release:
+- [ ] Change the PR base to be against the next `dev` branch.
+- [ ] Increase the lower bound to the current dbt minor version in both the `dbt_project.yml` and `run_test.sh` files
+- [ ] Fix any errors
+- [ ] Merge `dev` into `main`
+- [ ] Create a minor release
+- [ ] Once the release is available on hub, [create a new issue](https://github.com/dbt-labs/dbt-utils/issues/new/choose) using the "dbt-utils Minor Release Checklist" template

projects/test_lyon/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,25 @@
+---
+name: Feature request
+about: Suggest an idea for this package
+title: ''
+labels: enhancement, triage
+assignees: ''
+
+---
+
+### Describe the feature
+A clear and concise description of what you want to happen.
+
+### Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+### Additional context
+Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here.
+
+### Who will this benefit?
+What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly.
+
+### Are you interested in contributing this feature?
+<!---
+Let us know if you want to contribute the feature, and whether would need a hand getting started
+--->

projects/test_lyon/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/utils_minor_release.md

@@ -0,0 +1,38 @@
+---
+name: dbt-utils Minor Release Follow-up
+about: A checklist of tasks to complete after making a dbt-utils minor release
+title: 'dbt Minor Release Follow up for dbt-utils v0.x.0'
+labels:
+assignees: ''
+---
+
+<!---
+This template is to be used once a new dbt-utils release is available on hub.
+In the future, we will consider automating this.
+-->
+
+## Process for each dependent package
+First, check if this is a breaking change
+- [ ] Increase the upper bound of the `dbt-utils` `version:` config in the `packages.yml` of the dependent package.
+- [ ] Push to a new branch to see if tests pass, or test locally.
+
+If this is _not_ a breaking change:
+- [ ] Create a patch release
+
+If this _is_ a breaking change:
+- [ ] Fix any breaking changes
+- [ ] Increase the lower bound to the current dbt-utils minor version
+- [ ] Create a minor release for the package
+
+## Checklist of dependent packages
+| Package                                                                      | PR     | Release     |
+|------------------------------------------------------------------------------|--------|-------------|
+| [audit-helper](https://github.com/dbt-labs/dbt-audit-helper)                 | [PR]() | [Release]() |
+| [codegen](https://github.com/dbt-labs/dbt-codegen)                           | [PR]() | [Release]() |
+| [redshift](https://github.com/dbt-labs/redshift)                             | [PR]() | [Release]() |
+| [event-logging](https://github.com/dbt-labs/dbt-event-logging)               | [PR]() | [Release]() |
+| [snowplow](https://github.com/dbt-labs/snowplow)                             | [PR]() | [Release]() |
+| [external-tables](https://github.com/dbt-labs/dbt-external-tables)           | [PR]() | [Release]() |
+| [segment](https://github.com/dbt-labs/segment)                               | [PR]() | [Release]() |
+| [facebook-ads](https://github.com/dbt-labs/facebook-ads)                     | [PR]() | [Release]() |
+| [stitch-utils](https://github.com/dbt-labs/stitch-utils)                     | [PR]() | [Release]() |