Merge pull request #3 from OpenSourcePolitics/add_private_proposals_f…

…ields_cdc

feat(cdc): create decidim_awesome_private_proposal_fields models

projects/cour_des_comptes/models/intermediate/decidim_awesome/int_decidim_awesome_private_proposal_fields.sql

@@ -0,0 +1,19 @@
+WITH parsed_data AS (
+    SELECT
+        decidim_awesome_private_proposal_fields.id,
+        decidim_awesome_private_proposal_fields.proposal_id,
+        xpath('//text()', unnest(xpath('//dt', xml_data)))::text AS private_field_description,
+        unnest(xpath('//dd/div/text()', unnest(xpath('//dd', xml_data))))::text AS private_field_content
+    FROM
+        {{ ref ("stg_decidim_awesome_private_proposal_fields")}} as decidim_awesome_private_proposal_fields,
+        LATERAL xmlparse(document private_body) AS xml_data
+)
+SELECT
+    parsed_data.id,
+    parsed_data.proposal_id,
+    replace(replace(private_field_description, '{"', ''), '"}', '') AS private_field_description,
+    parsed_data.private_field_content
+FROM
+    parsed_data
+WHERE
+    private_field_content IS NOT NULL

projects/cour_des_comptes/models/intermediate/decidim_awesome/schema.yml

@@ -0,0 +1,33 @@
+version: 2
+
+models:
+  - name: int_decidim_awesome_private_proposal_fields
+    description: "Model for parsing the information in the private proposals fields and unnest the XML data into readable questions and answers columns."
+    columns:
+      - name: id
+        description: "Primary key."
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+      - name: private_field_description
+        description: "Description of the private field to be answered."
+      - name: private_field_content
+        description: "The content filled by the user for a specific private field. There can be multiple content for the same private field description."
+
+
+unit_tests:
+
+   - name: testing_that_the_correct_xml_information_are_parsed
+     description: "Checks that age matching fetches the correct infos "
+     model: int_decidim_awesome_private_proposal_fields
+     given:
+      - input: ref('stg_decidim_awesome_private_proposal_fields')
+        format: dict
+        rows:
+          - {id: 1, proposal_id: 101, private_body: '<xml><dl class="decidim_awesome-custom_fields" data-generator="decidim_awesome" data-version="0.9.2"><dt name="radio-group-1724336804706-0">Vous êtes :</dt><dd id="radio-group-1724336804706-0" name="radio-group"><div alt="sans_emploi">Sans emploi</div></dd><dt name="number-1724336900935-0">Code postal</dt><dd id="number-1724336900935-0" name="number"><div>75003</div></dd><dt name="checkbox-group-1724336967989-0">Comment avez-vous eu connaissance de la plateforme ?</dt><dd id="checkbox-group-1724336967989-0" name="checkbox-group"><div alt="option2">Par la Cour des comptes et les CRTC (site web, réseaux sociaux…)</div><div alt="option5">Par mon entourage personnel ou professionnel</div></dd></dl></xml>'}
+     expect:
+      format: dict
+      rows:
+        - {id: 1, proposal_id: 101, private_field_description: "Vous êtes :", private_field_content: "Sans emploi"}
+        - {id: 1, proposal_id: 101, private_field_description: "Code postal", private_field_content: "75003"}
+        - {id: 1, proposal_id: 101, private_field_description: "Comment avez-vous eu connaissance de la plateforme ?", private_field_content: "Par la Cour des comptes et les CRTC (site web, réseaux sociaux…)"}
+        - {id: 1, proposal_id: 101, private_field_description: "Comment avez-vous eu connaissance de la plateforme ?", private_field_content: "Par mon entourage personnel ou professionnel"}

projects/cour_des_comptes/models/intermediate/users/schema.yml

@@ -1,5 +1,5 @@
 unit_tests:
- 
+
    - name: testing_that_extended_age_data_matches_age_categories
      description: "Checks that age matching fetches the correct infos "
      model: int_users
@@ -19,6 +19,4 @@ unit_tests:
         - {id: 2, date_of_birth: "1970-06-23", age: 54, age_category: "[50-54 ans]"}
         - {id: 3, date_of_birth: "1932-05-23", age: 92, age_category: "[75 ans ou plus]"}
         - {id: 4, date_of_birth: "2009-12-01", age: 14, age_category: "[0-15 ans]"}
-        - {id: 5, date_of_birth: null, age: null, age_category: 'Âge non défini'}
-
-
+        - {id: 5, date_of_birth: null, age: null, age_category: 'Âge non défini'}

projects/cour_des_comptes/models/staging/decidim/schema.yml

@@ -0,0 +1,25 @@
+version: 2
+
+
+sources:
+  - name: decidim
+    database: "{{ env_var('DBNAME') }}"
+    schema: public
+    tables:
+      - name: decidim_awesome_private_proposal_fields
+
+models:
+  - name: stg_decidim_awesome_private_proposal_fields
+    description: "Table for private proposal fields in the module decidim awesome."
+    columns:
+      - name: id
+        description: "Primary key."
+        data_tests:
+          - not_null
+          - unique
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+        data_tests:
+          - not_null
+      - name: private_body
+        description: "Body of the answers of the private proposals fields answered by the user. The format is XML."

projects/cour_des_comptes/models/staging/decidim/stg_decidim_awesome_private_proposal_fields.sql

@@ -0,0 +1,11 @@
+WITH source AS (
+      SELECT * FROM {{ source('decidim', 'decidim_awesome_private_proposal_fields') }}
+)
+
+SELECT
+    id,
+    proposal_id,
+    updated_at,
+    created_at,
+    private_body
+FROM source