From e990f2bda75fd4f50331342d907d5df303ff018e Mon Sep 17 00:00:00 2001
From: fujistone <fujistone@hotmail.fr>
Date: Mon, 30 Sep 2024 00:53:15 +0200
Subject: [PATCH] adding private fields models to lyon project

---
 .../intermediate/decidim_awesome/schema.yml   | 32 +++++++++++++++++++
 ..._decidim_awesome_proposal_extra_fields.sql | 19 +++++++++++
 .../decidim_awesome_proposal_extra_fields.sql | 11 +++++++
 .../models/staging/decidim_awesome/schema.yml | 25 +++++++++++++++
 ..._decidim_awesome_proposal_extra_fields.sql | 12 +++++++
 5 files changed, 99 insertions(+)
 create mode 100644 projects/lyon/models/intermediate/decidim_awesome/schema.yml
 create mode 100644 projects/lyon/models/intermediate/int_decidim_awesome_proposal_extra_fields.sql
 create mode 100644 projects/lyon/models/marts/decidim_awesome/decidim_awesome_proposal_extra_fields.sql
 create mode 100644 projects/lyon/models/staging/decidim_awesome/schema.yml
 create mode 100644 projects/lyon/models/staging/decidim_awesome/stg_decidim_awesome_proposal_extra_fields.sql

diff --git a/projects/lyon/models/intermediate/decidim_awesome/schema.yml b/projects/lyon/models/intermediate/decidim_awesome/schema.yml
new file mode 100644
index 0000000..1552b69
--- /dev/null
+++ b/projects/lyon/models/intermediate/decidim_awesome/schema.yml
@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: int_decidim_awesome_private_proposal_fields
+    description: "Model for parsing the information in the private proposals fields and unnest the XML data into readable questions and answers columns."
+    columns:
+      - name: id
+        description: "Primary key."
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+      - name: private_field_description
+        description: "Description of the private field to be answered."
+      - name: private_field_content
+        description: "The content filled by the user for a specific private field. There can be multiple content for the same private field description."
+
+
+unit_tests:
+
+   - name: testing_that_the_correct_xml_information_are_parsed
+     description: "Checks that age matching fetches the correct infos "
+     model: int_decidim_awesome_proposal_extra_fields
+     given:
+      - input: ref('stg_decidim_awesome_proposal_extra_fields')
+        format: dict
+        rows:
+          - {id: 1, proposal_id: 101, private_body_clear: "<xml><dl class=\"decidim_awesome-custom_fields\" data-generator=\"decidim_awesome\" data-version=\"0.10.3\"><dt name=\"radio-group-1725003392539-0\">Cette idée est déposée à titre :</dt><dd id=\"radio-group-1725003392539-0\" name=\"radio-group\"><div alt=\"individuel\">Individuel</div></dd><dt name=\"radio-group-1725003468065-0\">Votre tranche d'âge :</dt><dd id=\"radio-group-1725003468065-0\" name=\"radio-group\"><div alt=\"1625\">16-25 ans</div></dd></dl></xml>"}
+     expect:
+      format: dict
+      rows:
+        - {id: 1, proposal_id: 101, private_field_description: "Cette idée est déposée à titre :", private_field_content: "Individuel"}
+        - {id: 1, proposal_id: 101, private_field_description: "Votre tranche d'âge :", private_field_content: "16-25 ans"}
+       
\ No newline at end of file
diff --git a/projects/lyon/models/intermediate/int_decidim_awesome_proposal_extra_fields.sql b/projects/lyon/models/intermediate/int_decidim_awesome_proposal_extra_fields.sql
new file mode 100644
index 0000000..968e239
--- /dev/null
+++ b/projects/lyon/models/intermediate/int_decidim_awesome_proposal_extra_fields.sql
@@ -0,0 +1,19 @@
+WITH parsed_data AS (
+    SELECT
+        decidim_awesome_proposal_extra_fields.id,
+        decidim_awesome_proposal_extra_fields.proposal_id,
+        xpath('//text()', unnest(xpath('//dt', xml_data)))::text AS private_field_description,
+        unnest(xpath('//dd/div/text()', unnest(xpath('//dd', xml_data))))::text AS private_field_content
+    FROM
+        {{ ref ("stg_decidim_awesome_proposal_extra_fields")}} AS decidim_awesome_proposal_extra_fields,
+        LATERAL xmlparse(document private_body_clear) AS xml_data
+)
+SELECT
+    parsed_data.id,
+    parsed_data.proposal_id,
+    replace(replace(private_field_description, '{"', ''), '"}', '') AS private_field_description,
+    parsed_data.private_field_content
+FROM
+    parsed_data
+WHERE
+    private_field_content IS NOT NULL
\ No newline at end of file
diff --git a/projects/lyon/models/marts/decidim_awesome/decidim_awesome_proposal_extra_fields.sql b/projects/lyon/models/marts/decidim_awesome/decidim_awesome_proposal_extra_fields.sql
new file mode 100644
index 0000000..116eedc
--- /dev/null
+++ b/projects/lyon/models/marts/decidim_awesome/decidim_awesome_proposal_extra_fields.sql
@@ -0,0 +1,11 @@
+SELECT
+    decidim_awesome_proposal_extra_fields.id,
+    decidim_awesome_proposal_extra_fields.proposal_id,
+    decidim_awesome_proposal_extra_fields.private_field_description,
+    decidim_awesome_proposal_extra_fields.private_field_content,
+    decidim_proposals_proposals.decidim_component_id,
+    components.ps_title
+FROM
+    {{ ref ("int_decidim_awesome_proposal_extra_fields")}} AS decidim_awesome_proposal_extra_fields
+JOIN {{ ref ("stg_decidim_proposals")}} AS decidim_proposals_proposals ON decidim_awesome_proposal_extra_fields.proposal_id = decidim_proposals_proposals.id
+JOIN {{ ref ("components")}} AS components ON decidim_proposals_proposals.decidim_component_id = components.id
\ No newline at end of file
diff --git a/projects/lyon/models/staging/decidim_awesome/schema.yml b/projects/lyon/models/staging/decidim_awesome/schema.yml
new file mode 100644
index 0000000..826f034
--- /dev/null
+++ b/projects/lyon/models/staging/decidim_awesome/schema.yml
@@ -0,0 +1,25 @@
+version: 2
+
+
+sources:
+  - name: decidim
+    database: "{{ env_var('DBNAME') }}"
+    schema: public
+    tables:
+      - name: decidim_awesome_proposal_extra_fields_test
+
+models:
+  - name: stg_decidim_awesome_proposal_extra_fields
+    description: "Table for private proposal fields in the module decidim awesome."
+    columns:
+      - name: id
+        description: "Primary key."
+        data_tests:
+          - not_null
+          - unique
+      - name: proposal_id
+        description: "Id of the proposal the private field is linked to."
+        data_tests:
+          - not_null
+      - name: private_body
+        description: "Body of the answers of the private proposals fields answered by the user. The format is XML."
\ No newline at end of file
diff --git a/projects/lyon/models/staging/decidim_awesome/stg_decidim_awesome_proposal_extra_fields.sql b/projects/lyon/models/staging/decidim_awesome/stg_decidim_awesome_proposal_extra_fields.sql
new file mode 100644
index 0000000..af85012
--- /dev/null
+++ b/projects/lyon/models/staging/decidim_awesome/stg_decidim_awesome_proposal_extra_fields.sql
@@ -0,0 +1,12 @@
+WITH source AS (
+      SELECT * FROM {{ source('decidim', 'decidim_awesome_proposal_extra_fields_test') }}
+)
+
+
+SELECT
+    id,
+    decidim_proposal_id AS proposal_id,
+    updated_at,
+    created_at,
+    replace(private_body_clear, '&nbsp;', '') AS private_body_clear
+FROM source
\ No newline at end of file