[18.0][ADD] polars_db_process: get db query in polars dataframe

polars_db_process/README.rst

@@ -0,0 +1,100 @@
+=======================
+Polars Database Process
+=======================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github
+    :target: https://github.com/OCA/reporting-engine/tree/18.0/polars_db_process
+    :alt: OCA/reporting-engine
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-polars_db_process
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+From a db query, this module allows to transform data in Polars
+dataframe and process them according to rules in order to:
+
+-  filter data and display
+-  obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to
+create/update or import
+
+For that you need to transform/arrange data to the same way
+
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/reporting-engine/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/reporting-engine/issues/new?body=module:%20polars_db_process%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* Akretion
+
+Contributors
+------------
+
+-  Akretion
+
+   -  David BEAL [email protected]
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px
+    :target: https://github.com/bealdav
+    :alt: bealdav
+
+Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-bealdav| 
+
+This module is part of the `OCA/reporting-engine <https://github.com/OCA/reporting-engine/tree/18.0/polars_db_process>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

polars_db_process/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import wizards

polars_db_process/__manifest__.py

@@ -0,0 +1,35 @@
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Polars Database Process",
+    "version": "18.0.1.0.0",
+    "summary": "Allow to create a Polars dataframe from db.query and "
+    "check it and process it according to rules",
+    "category": "Reporting",
+    "license": "AGPL-3",
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "development_status": "Alpha",
+    "website": "https://github.com/OCA/reporting-engine",
+    "maintainers": ["bealdav"],
+    "depends": [
+        "polars_process",
+    ],
+    "external_dependencies": {
+        "python": [
+            "connectorx",
+        ]
+    },
+    "data": [
+        "security/ir.model.access.xml",
+        "wizards/df_process.xml",
+        "views/dataframe.xml",
+        "views/df_field.xml",
+        "views/df_source.xml",
+        "views/db_config.xml",
+        "data/demo.xml",
+    ],
+    "demo": [
+        "data/demo.xml",
+    ],
+    "installable": True,
+}

polars_db_process/data/demo.xml

@@ -0,0 +1,13 @@
+<odoo noupdate="1">
+    <record id="sqlite_chinook" model="db.config">
+        <field name="name">Chinook</field>
+        <field
+            name="string_connexion"
+        >sqlite://.../polars_db_schema/tests/files/chinook.sqlite</field>
+    </record>
+
+    <record id="contact_chinook" model="dataframe">
+        <field name="model_id" ref="base.model_res_partner" />
+        <field name="code">Chinook Customers</field>
+    </record>
+</odoo>

polars_db_process/data/files/customers.sql

@@ -0,0 +1,4 @@
+--{"model": "res.partner"}
+SELECT LastName AS name, State AS state, PostalCode AS zip, Email AS mail
+-- , Company, Address, City, Country, Phone, Fax, SupportRepId
+FROM customers;

polars_db_process/models/__init__.py

@@ -0,0 +1,2 @@
+from . import df_source
+from . import db_config

polars_db_process/models/db_config.py

@@ -0,0 +1,44 @@
+import connectorx as cx
+
+from odoo import _, exceptions, fields, models
+
+HELP = """
+String connexion samples:
+
+postgres://user:PASSWORD@server:port/database
+mssql://user:PASSWORD@server:port/db.encrypt=true&trusted_connection=false
+sqlite:///home/user/path/test.db
+mysql://user:PASSWORD@server:port/database
+oracle://user:PASSWORD@server:port/database
+"""
+
+
+class DbConfig(models.Model):
+    _name = "db.config"
+    _description = "External db.configuration"
+    _rec_name = "name"
+    _order = "name"
+    _rec_names_search = ["name"]
+
+    name = fields.Char(required=True)
+    string_connexion = fields.Char(required=True, help=HELP)
+    password = fields.Char(help="Not required for Sqlite")
+
+    def _get_connexion(self):
+        return self.string_connexion.replace("PASSWORD", self.password or "")
+
+    def test_connexion(self):
+        res = self._read_sql(self._get_connexion(), "SELECT 1")
+        if len(res):
+            # Not invalid in reality
+            raise exceptions.ValidationError(_("Connexion OK !"))
+
+    def _read_sql(self, connexion, query):
+        try:
+            return cx.read_sql(connexion, query, return_type="polars")
+        except RuntimeError as err:
+            raise exceptions.ValidationError(err) from err
+        except TimeoutError as err:
+            raise exceptions.ValidationError(err) from err
+        except Exception as err:
+            raise exceptions.ValidationError(err) from err

polars_db_process/models/df_source.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+from odoo import fields, models
+from odoo.modules.module import get_module_path
+from odoo.tools.safe_eval import safe_eval
+
+MODULE = __name__[12 : __name__.index(".", 13)]
+
+HELP = """Supported files: .xlsx and .sql
+Sql files may contains a comment on first line
+to be mapped automatically with dataframe, i.e:\n
+-- {'model_id': 'product.product', 'db_id': mydb}
+-- {'code': 'my_delivery_address', 'db_id': mydb}
+"""
+
+
+class DfSource(models.Model):
+    _inherit = "df.source"
+
+    name = fields.Char(help=HELP)
+    query = fields.Char()
+    # TODO : -> db_config_id
+    db_id = fields.Many2one(comodel_name="db.config", help="Database")
+
+    def _file_hook(self, file):
+        "Map sql file with the right Odoo model via dataframe and the right db.config"
+        vals = super()._file_hook(file)
+        if ".sql" in file:
+            # TODO: improve
+            content = self._get_file(file).decode("utf-8")
+            contents = content.split("\n")
+            if contents:
+                # we only detect first line
+                metadata = safe_eval(contents[0].replace("--", ""))
+                model_name = metadata.get("model")
+                model = self.env["ir.model"].search([("model", "=", model_name)])
+                if model_name:
+                    # we don't want to use these dataframes
+                    dataframes = (
+                        self.env["df.source"]
+                        .search([])
+                        .filtered(lambda s: not s.db_id)
+                        .mapped("dataframe_id")
+                    )
+                    dataframe = self.env["dataframe"].search(
+                        [
+                            ("id", "not in", dataframes.ids),
+                            ("model_id", "=", model_name),
+                        ]
+                    )
+                    if dataframe:
+                        # TODO use first
+                        vals["dataframe_id"] = dataframe[0].id
+                        db_config = self.env["db.config"].search(
+                            [("name", "ilike", metadata.get("db_id"))]
+                        )
+                        vals["db_id"] = db_config and db_config[0].id or False
+                    else:
+                        df = self.env["dataframe"].create(
+                            {"code": model.name, "model_id": model and model[0].id}
+                        )
+                        vals["dataframe_id"] = df.id
+            vals["query"] = content
+        return vals
+
+    def _populate(self):
+        chinook = self.env.ref(f"{MODULE}.sqlite_chinook")
+        if chinook:
+            # Demo behavior only
+            path = Path(get_module_path(MODULE)) / "data/chinook.sqlite"
+            chinook.string_connexion = f"sqlite://{str(path)}"
+        return super()._populate()
+
+    def _get_test_file_paths(self):
+        res = super()._get_test_file_paths()
+        res.update(
+            {
+                "polars_db_process": {
+                    "relative_path": "data/files",
+                    "xmlid": "polars_db_process.contact_chinook",
+                }
+            }
+        )
+        return res

polars_db_process/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"

polars_db_process/readme/CONTRIBUTORS.md

@@ -0,0 +1,3 @@
+- Akretion
+
+  - David BEAL <[email protected]>

polars_db_process/readme/DESCRIPTION.md

@@ -0,0 +1,8 @@
+From a db query, this module allows to transform data in Polars dataframe and process them according to rules in order to:
+
+- filter data and display 
+- obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to create/update or import
+
+For that you need to transform/arrange data to the same way

polars_db_process/security/ir.model.access.xml

@@ -0,0 +1,11 @@
+<odoo>
+    <record id="db_config_all" model="ir.model.access">
+        <field name="name">Database Config</field>
+        <field name="model_id" ref="model_db_config" />
+        <field name="group_id" ref="base.group_user" />
+        <field name="perm_read" eval="1" />
+        <field name="perm_create" eval="1" />
+        <field name="perm_write" eval="1" />
+        <field name="perm_unlink" eval="1" />
+    </record>
+</odoo>