[ADD] sheet_to_dataframe module: convert file to polars dataframe and…

… process it
OCA · Oct 16, 2024 · dc9b4cb · dc9b4cb
1 parent c9d37ce
commit dc9b4cb
Show file tree

Hide file tree

Showing 18 changed files with 408 additions and 0 deletions.
diff --git a/sheet_dataframe_process/__init__.py b/sheet_dataframe_process/__init__.py
@@ -0,0 +1,2 @@
+from . import models
+from . import wizards
diff --git a/sheet_dataframe_process/__manifest__.py b/sheet_dataframe_process/__manifest__.py
@@ -0,0 +1,28 @@
+{
+    "name": "Spreadsheet Import Configurable",
+    "version": "18.0.1.0.0",
+    "author": "Akretion",
+    "license": "GPL-3",
+    "depends": [
+        "contacts",
+    ],
+    "external_dependencies": {
+        "python": [
+            "polars",
+            "fastexcel",
+        ]
+    },
+    "data": [
+        "security/ir.model.access.xml",
+        # "wizards/sheet_dataframe.xml",
+        "views/file_config.xml",
+        "views/file_field.xml",
+        "views/file_partner_field.xml",
+        # "views/file_config_field.xml",
+        # "views/test_file.xml",
+        # "data/action.xml",
+        # "data/misc.xml",
+        "data/demo.xml",
+    ],
+    "installable": True,
+}
diff --git a/sheet_dataframe_process/data/action.xml b/sheet_dataframe_process/data/action.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+
+<record id="refresh_conf" model="ir.actions.server">
+    <field name="name">📙 Mise à jour config</field>
+    <field name="model_id" ref="model_file_config" />
+    <field name="binding_model_id" ref="model_file_config" />
+    <field name="state">code</field>
+    <field name="code">env["file.config"]._refresh_conf()</field>
+</record>
+
+<record id="refresh_test" model="ir.actions.server">
+    <field name="name">📙 Mise à jour tests</field>
+    <field name="model_id" ref="model_test_file" />
+    <field name="binding_model_id" ref="model_test_file" />
+    <field name="state">code</field>
+    <field name="code">env["test.file"]._populate()</field>
+</record>
+
+</odoo>
diff --git a/sheet_dataframe_process/data/demo.xml b/sheet_dataframe_process/data/demo.xml
@@ -0,0 +1,29 @@
+<odoo>
+
+    <record id="file_config_contact" model="file.config">
+        <field name="model_id" ref="base.model_res_partner" />
+    </record>
+
+    <record id="file_config_contact_country" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__country_code" />
+    </record>
+    <record id="file_config_contact_name" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__name" />
+        <field name="required" eval="1" />
+        <field name="on_fail">skip</field>
+    </record>
+
+    <record id="file_config_contact_country_partner18" model="file.partner.field">
+        <field name="line_id" ref="file_config_contact_country" />
+        <field name="partner_id" ref="base.res_partner_18" />
+        <field name="matching_column">pays</field>
+    </record>
+    <record id="file_config_contact_country_partner5" model="file.partner.field">
+        <field name="line_id" ref="file_config_contact_country" />
+        <field name="partner_id" ref="base.res_partner_5" />
+        <field name="matching_column">Blabla</field>
+    </record>
+
+</odoo>
diff --git a/sheet_dataframe_process/models/__init__.py b/sheet_dataframe_process/models/__init__.py
@@ -0,0 +1,5 @@
+# from . import file_field_partner
+from . import file_config
+from . import file_field
+from . import file_partner_field
+# from . import test_file
diff --git a/sheet_dataframe_process/models/file_config.py b/sheet_dataframe_process/models/file_config.py
@@ -0,0 +1,23 @@
+from odoo import fields, models
+
+
+class FileConfig(models.Model):
+    _name = "file.config"
+    _inherit = "mail.thread"
+    _description = "Config import fournisseur"
+    _rec_name = "model_id"
+
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        required=True,
+        ondelete="cascade"
+    )
+    code = fields.Char(help="Allow to browse between several identical models")
+    action = fields.Selection(selection=[("display", "Display"), ("dataframe", "Dataframe"), ("import", "Import (Not yet implemented)")], default="display", help="Some other behaviors can be implemented")
+    partner_ids = fields.Many2many(comodel_name="res.partner")
+    field_ids = fields.One2many(comodel_name="file.field", inverse_name="config_id")
+    field_match_ids = fields.One2many(comodel_name="file.partner.field", inverse_name="config_id")
+
+    def populate_match_lines(self):
+        self.ensure_one()
+        return NotImplemented
diff --git a/sheet_dataframe_process/models/file_field.py b/sheet_dataframe_process/models/file_field.py
@@ -0,0 +1,30 @@
+from odoo import fields, models
+
+
+class FileField(models.Model):
+    _name = "file.field"
+    _description = "Configuration de l'import de champ"
+
+    config_id = fields.Many2one(
+        comodel_name="file.config", required=True, ondelete="cascade"
+    )
+    field_id = fields.Many2one(
+        comodel_name="ir.model.fields",
+        ondelete="cascade",
+        required=True,
+        domain="[('model_id', '=', model_id)]"
+        # [('model_id', '=', model_id)]
+    )
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        readonly=True,
+    )
+    required = fields.Boolean(
+        tracking=True,
+        help="Prevent to import missing data if field is missing in some records",
+    )
+    on_fail = fields.Selection(
+        selection=[("skip", "Skip record"), ("stop", "Stop Process")],
+        help="What should be the behavior in case of failure regarding constraint "
+        "fields (required, format, etc)",
+    )
diff --git a/sheet_dataframe_process/models/file_partner_field.py b/sheet_dataframe_process/models/file_partner_field.py
@@ -0,0 +1,16 @@
+from odoo import fields, models
+
+
+class FilePartnerField(models.Model):
+    _name = "file.partner.field"
+    _inherits = {"file.field": "line_id"}
+    _description = "Configuration de l'import de champ"
+
+    line_id = fields.Many2one(
+        comodel_name="file.field", required=True, ondelete="cascade"
+    )
+    partner_id = fields.Many2one(
+        comodel_name="res.partner",
+        readonly=True,
+    )
+    matching_column = fields.Char(help="Field name in spreadsheet")
diff --git a/sheet_dataframe_process/pyproject.toml b/sheet_dataframe_process/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"
diff --git a/sheet_dataframe_process/security/ir.model.access.xml b/sheet_dataframe_process/security/ir.model.access.xml
@@ -0,0 +1,43 @@
+<odoo>
+
+    <record id="file_config_all" model="ir.model.access">
+        <field name="name">File Config</field>
+        <field name="model_id" ref="model_file_config" />
+        <field name="group_id" ref="base.group_user" />
+        <field name="perm_read" eval="1" />
+        <field name="perm_create" eval="1" />
+        <field name="perm_write" eval="1" />
+        <field name="perm_unlink" eval="1" />
+    </record>
+
+    <record id="file_field_all" model="ir.model.access">
+        <field name="name">File Field</field>
+        <field name="model_id" ref="model_file_field" />
+        <field name="group_id" ref="base.group_user" />
+        <field name="perm_read" eval="1" />
+        <field name="perm_create" eval="1" />
+        <field name="perm_write" eval="1" />
+        <field name="perm_unlink" eval="1" />
+    </record>
+
+    <record id="file_partner_field_all" model="ir.model.access">
+        <field name="name">File Partner Field</field>
+        <field name="model_id" ref="model_file_partner_field" />
+        <field name="group_id" ref="base.group_user" />
+        <field name="perm_read" eval="1" />
+        <field name="perm_create" eval="1" />
+        <field name="perm_write" eval="1" />
+        <field name="perm_unlink" eval="1" />
+    </record>
+
+<!--     <record id="sheet_dataframe_transient_all" model="ir.model.access">
+        <field name="name">sheet.dataframe.transient</field>
+        <field name="model_id" ref="model_sheet_dataframe_transient" />
+        <field name="group_id" ref="base.group_user" />
+        <field name="perm_read" eval="1" />
+        <field name="perm_create" eval="1" />
+        <field name="perm_write" eval="1" />
+        <field name="perm_unlink" eval="1" />
+    </record>
+ -->
+</odoo>
diff --git a/sheet_dataframe_process/tests/__init__.py b/sheet_dataframe_process/tests/__init__.py
@@ -0,0 +1 @@
+from . import test_module
diff --git a/sheet_dataframe_process/tests/test_module.py b/sheet_dataframe_process/tests/test_module.py
diff --git a/sheet_dataframe_process/views/file_config.xml b/sheet_dataframe_process/views/file_config.xml
@@ -0,0 +1,76 @@
+<odoo>
+
+<record id="file_config_form" model="ir.ui.view">
+    <field name="model">file.config</field>
+    <field name="arch" type="xml">
+        <form>
+            <header>
+<!--                 <button name="populate_match_lines" string="Populate" type="object" help="Populate matching lines with partners" />
+                <button
+                        name="% (sheet_dataframe_transient_action) d"
+                        type="action"
+                        string="Import"
+                        context="{'default_partner_id': partner_id}"
+                    /> -->
+            </header>
+            <sheet>
+                <group>
+                    <group>
+                        <field name="model_id" />
+                        <field name="action" />
+                    </group>
+                    <group>
+                        <field name="code" />
+                        <field name="partner_ids" widget="many2many_tags" />
+                    </group>
+                   <field name="field_ids" />
+                   <field name="field_match_ids" />
+                </group>
+            </sheet>
+            <chatter />
+        </form>
+    </field>
+</record>
+
+<record id="file_config_tree" model="ir.ui.view">
+    <field name="model">file.config</field>
+    <field name="arch" type="xml">
+        <list>
+            <field name="model_id" />
+        </list>
+    </field>
+</record>
+
+<record id="file_config_search" model="ir.ui.view">
+    <field name="model">file.config</field>
+    <field name="arch" type="xml">
+        <search>
+            <field name="model_id" />
+        </search>
+    </field>
+</record>
+
+<record id="action_file_config" model="ir.actions.act_window">
+    <field name="name">Configuration</field>
+    <field name="res_model">file.config</field>
+    <field name="view_mode">list,form</field>
+    <field name="path">file-config</field>
+</record>
+
+<menuitem id="edi_menu" name="EDI" parent="contacts.menu_contacts" sequence="3" />
+
+<menuitem
+        id="file_config_menu"
+        action="action_file_config"
+        sequence="7"
+        parent="edi_menu"
+    />
+
+<!-- <menuitem
+        id="import_spreadsheet_wiz_menu"
+        action="import_spreadsheet_wiz_action"
+        parent="edi_menu"
+        sequence="9"
+    />
+ -->
+</odoo>
diff --git a/sheet_dataframe_process/views/file_field.xml b/sheet_dataframe_process/views/file_field.xml
@@ -0,0 +1,26 @@
+<odoo>
+
+<record id="file_field_tree" model="ir.ui.view">
+    <field name="model">file.field</field>
+    <field name="arch" type="xml">
+        <list editable="top">
+            <field
+                    name="config_id"
+                    context="{'default_config_id': parent}"
+                    optional="hide"
+                />
+            <field name="field_id" />
+            <field name="required" />
+            <field name="on_fail" />
+        </list>
+    </field>
+</record>
+
+<record id="action_file_field" model="ir.actions.act_window">
+    <field name="name">File Field</field>
+    <field name="res_model">file.field</field>
+    <field name="view_mode">list</field>
+    <field name="path">file-field</field>
+</record>
+
+</odoo>
diff --git a/sheet_dataframe_process/views/file_partner_field.xml b/sheet_dataframe_process/views/file_partner_field.xml
@@ -0,0 +1,26 @@
+<odoo>
+
+<record id="file_partner_field_tree" model="ir.ui.view">
+    <field name="model">file.partner.field</field>
+    <field name="arch" type="xml">
+        <list editable="top">
+            <field
+                    name="config_id"
+                    context="{'default_config_id': parent}"
+                    optional="hide"
+                />
+            <field name="field_id" />
+            <field name="partner_id" />
+            <field name="matching_column" />
+        </list>
+    </field>
+</record>
+
+<record id="action_file_field" model="ir.actions.act_window">
+    <field name="name">File Partner Field</field>
+    <field name="res_model">file.partner.field</field>
+    <field name="view_mode">list</field>
+    <field name="path">file-partner-field</field>
+</record>
+
+</odoo>
diff --git a/sheet_dataframe_process/wizards/__init__.py b/sheet_dataframe_process/wizards/__init__.py
@@ -0,0 +1 @@
+# from . import sheet_dataframe
diff --git a/sheet_dataframe_process/wizards/sheet_dataframe.py b/sheet_dataframe_process/wizards/sheet_dataframe.py
@@ -0,0 +1,35 @@
+import base64
+
+import polars as pl
+
+from odoo import fields, models
+
+MODULE = __name__[12 : __name__.index(".", 13)]
+
+
+class SheetDataframeTransient(models.TransientModel):
+    _name = "sheet.dataframe.transient"
+    _description = "Create polars dataframe from spreadsheet like file"
+
+    config_id = fields.Many2one(comodel_name="file.config", required=True)
+    sample = fields.Html(readonly=True)
+    filename = fields.Char()
+    file = fields.Binary(attachment=False)
+
+    def create(self, vals):
+        res = super().create(vals)
+        if res.config_id and res.file:
+            res.apply()
+        return res
+
+    def apply(self):
+        self.ensure_one()
+        file = base64.b64decode(self.file)
+        df = pl.read_excel(source=file)
+        sample = str(df.__repr__).replace("\n", "<br />").replace("┘&gt;", "")
+        self.sample = f"<html><pre>\n{sample}\n</pre></html>"
+        action = self.env.ref(
+            f"{MODULE}.sheet_dataframe_transient_action"
+        )._get_action_dict()
+        action["res_id"] = self.id
+        return action