[18.0][ADD] polars_db_process: get db query in polars dataframe

polars_db_process/README.rst

@@ -0,0 +1,100 @@
+=======================
+Polars Database Process
+=======================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github
+    :target: https://github.com/OCA/server-backend/tree/18.0/polars_db_process
+    :alt: OCA/server-backend
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_process
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/server-backend&target_branch=18.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+From a db query, this module allows to transform data in Polars
+dataframe and process them according to rules in order to:
+
+-  filter data and display
+-  obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to
+create/update or import
+
+For that you need to transform/arrange data to the same way
+
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/server-backend/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/server-backend/issues/new?body=module:%20polars_db_process%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* Akretion
+
+Contributors
+------------
+
+-  Akretion
+
+   -  David BEAL [email protected]
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px
+    :target: https://github.com/bealdav
+    :alt: bealdav
+
+Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-bealdav| 
+
+This module is part of the `OCA/server-backend <https://github.com/OCA/server-backend/tree/18.0/polars_db_process>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

polars_db_process/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import wizards

polars_db_process/__manifest__.py

@@ -0,0 +1,36 @@
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Polars Database Process",
+    "version": "18.0.1.0.0",
+    "summary": "Allow to create a Polars dataframe from db.query and "
+    "check it and process it according to rules",
+    "category": "Reporting",
+    "license": "AGPL-3",
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "development_status": "Alpha",
+    "website": "https://github.com/OCA/server-backend",
+    "maintainers": ["bealdav"],
+    "depends": [
+        "polars_process",
+    ],
+    "external_dependencies": {
+        "python": [
+            "connectorx",
+        ]
+    },
+    "data": [
+        "security/ir.model.access.xml",
+        "wizards/df_process_wiz.xml",
+        "views/model_map.xml",
+        "views/df_field.xml",
+        "views/df_source.xml",
+        "views/db_config.xml",
+        "data/action.xml",
+        "data/demo.xml",  # TODO remove
+    ],
+    "demo": [
+        "data/demo.xml",
+    ],
+    "installable": True,
+}

polars_db_process/data/action.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+    <record id="remove_related_uidstring_record" model="ir.actions.server">
+        <field name="name">Remove ir_model_data created from there</field>
+        <field name="model_id" ref="polars_db_process.model_model_map" />
+        <field name="binding_model_id" ref="polars_db_process.model_model_map" />
+        <field name="state">code</field>
+        <field name="code">env['model.map']._remove_uidstring_related_records()</field>
+    </record>
+</odoo>

polars_db_process/data/demo.xml

@@ -0,0 +1,13 @@
+<odoo noupdate="1">
+    <record id="sqlite_chinook" model="db.config">
+        <field name="name">Chinook</field>
+        <field
+            name="string_connexion"
+        >sqlite://.../polars_db_schema/tests/files/chinook.sqlite</field>
+    </record>
+
+    <record id="contact_chinook" model="model.map">
+        <field name="model_id" ref="base.model_res_partner" />
+        <field name="code">Chinook Customers</field>
+    </record>
+</odoo>

polars_db_process/data/files/customers.sql

@@ -0,0 +1,4 @@
+--{"model": "res.partner", "db_conf": "Chinook"}
+SELECT LastName AS name, State AS state, PostalCode AS zip, Email AS mail
+-- , Company, Address, City, Country, Phone, Fax, SupportRepId
+FROM customers;

polars_db_process/models/__init__.py

@@ -0,0 +1,3 @@
+from . import df_source
+from . import db_config
+from . import model_map

polars_db_process/models/db_config.py

@@ -0,0 +1,47 @@
+import connectorx as cx
+
+from odoo import _, exceptions, fields, models
+
+HELP = """
+String connexion samples:
+
+postgres://user:PASSWORD@server:port/database
+mssql://user:PASSWORD@server:port/db.encrypt=true&trusted_connection=false
+sqlite:///home/user/path/test.db
+mysql://user:PASSWORD@server:port/database
+oracle://user:PASSWORD@server:port/database
+"""
+
+
+class DbConfig(models.Model):
+    _name = "db.config"
+    _description = "External db.configuration"
+    _rec_name = "name"
+    _order = "name"
+    _rec_names_search = ["name"]
+
+    name = fields.Char(required=True)
+    string_connexion = fields.Char(required=True, help=HELP)
+    password = fields.Char(help="Not required for Sqlite")
+
+    def _get_connexion(self):
+        return self.string_connexion.replace("PASSWORD", self.password or "")
+
+    def test_connexion(self):
+        res = self._read_sql("SELECT 1")
+        if len(res):
+            # Not invalid in reality
+            raise exceptions.ValidationError(_("Connexion OK !"))
+
+    def _read_sql(self, query):
+        try:
+            return cx.read_sql(self._get_connexion(), query, return_type="polars")
+        except RuntimeError as err:
+            raise exceptions.ValidationError(err) from err
+        except TimeoutError as err:
+            raise exceptions.ValidationError(err) from err
+        except Exception as err:
+            raise exceptions.ValidationError(err) from err
+
+    def _set_uidstring_module_name(self):
+        return "polars"

polars_db_process/models/df_source.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+
+from odoo import fields, models
+from odoo.modules.module import get_module_path
+from odoo.tools.safe_eval import safe_eval
+
+MODULE = __name__[12 : __name__.index(".", 13)]
+
+HELP = """Supported files: .xlsx and .sql
+Sql files may contains a comment on first line
+to be mapped automatically with model_map, i.e:\n
+-- {'model_id': 'product.product', 'db_conf_id': mydb}
+-- {'code': 'my_delivery_address', 'db_conf_id': mydb}
+"""
+
+PARAMS = """{'model': False, 'code': False, 'db_conf': False}
+# 'model/code' to guess model.map', db_conf' name to guess db.config
+"""
+
+
+class DfSource(models.Model):
+    _inherit = "df.source"
+
+    name = fields.Char(help=HELP)
+    query = fields.Char()
+    params = fields.Char(
+        string="File Parameters",
+        default=PARAMS,
+        readonly=True,
+        help="Coming from sql files",
+    )
+    db_conf_id = fields.Many2one(
+        comodel_name="db.config", help="Database Configuration"
+    )
+
+    def _reset_process(self):
+        res = super()._reset_process()
+        mapp = self.model_map_id
+        if mapp and mapp.action == "import":
+            mapp._remove_uidstring_related_records()
+        return res
+
+    # def tmp(self, file, vals=None):
+    #     def guess_model_and_db():
+    #         domain=[]
+    #         if meta.get("model"):
+    #             domain.append(("model_id.name", '=', meta.get("model")))
+    #         if meta.get("code"):
+    #             domain.append(("code", '=', meta.get("code")))
+    #         if domain:
+    #             res = self.env["model.map"].search(domain)
+    #             vals["model_map_id"] = res and res[0].id
+    #         vals["db_conf_id"] = db_confs.get(meta.get("db_conf"))
+    #     guess_model_and_db()
+
+    def _file_hook(self, file):
+        "Map sql file with the right Odoo model via model_map and the right db.config"
+        vals = super()._file_hook(file)
+        if ".sql" in file:
+            # TODO: improve
+            db_confs = {x.name: x.id for x in self.env['db.config'].search([])}
+            content = self._get_file(file).decode("utf-8")
+            contents = content.split("\n")
+            if contents:
+                # we only detect first line
+                meta = safe_eval(contents[0].replace("--", ""))
+                if meta:
+                    vals['params'] = meta
+                model_name = meta.get("model")
+                model = self.env["ir.model"].search([("model", "=", model_name)])
+                if model_name:
+                    # we don't want to use these model_maps
+                    model_maps = (
+                        self.env["df.source"]
+                        .search([])
+                        .filtered(lambda s: not s.db_conf_id)
+                        .mapped("model_map_id")
+                    )
+                    model_map = self.env["model.map"].search(
+                        [
+                            ("id", "not in", model_maps.ids),
+                            ("model_id", "=", model_name),
+                        ]
+                    )
+                    if model_map:
+                        # TODO use first
+                        vals["model_map_id"] = model_map[0].id
+                        db_config = self.env["db.config"].search(
+                            [("name", "ilike", meta.get("db_conf_id"))]
+                        )
+                        vals["db_conf_id"] = db_config and db_config[0].id or False
+                    else:
+                        df = self.env["model.map"].create(
+                            {"code": model.name, "model_id": model and model[0].id}
+                        )
+                        vals["model_map_id"] = df.id
+            vals["query"] = content
+        return vals
+
+    def _populate(self):
+        chinook = self.env.ref(f"{MODULE}.sqlite_chinook")
+        if chinook:
+            # Demo behavior only
+            path = Path(get_module_path(MODULE)) / "data/chinook.sqlite"
+            chinook.string_connexion = f"sqlite://{str(path)}"
+        return super()._populate()
+
+    def _get_test_file_paths(self):
+        res = super()._get_test_file_paths()
+        res.update(
+            {
+                "polars_db_process": {
+                    "relative_path": "data/files",
+                    "xmlid": "polars_db_process.contact_chinook",
+                }
+            }
+        )
+        return res

polars_db_process/models/model_map.py

@@ -0,0 +1,34 @@
+from odoo import models
+
+
+class ModelMap(models.Model):
+    _inherit = "model.map"
+
+    def _remove_uidstring_related_records(self):
+        active_ids = self.env.context.get("active_ids")
+        if active_ids:
+            self = self.browse(self.env.context.get("active_ids"))
+        for rec in self:
+            im_data = self.env["ir.model.data"].search(
+                [
+                    ("model", "=", rec.model_id.model),
+                    ("module", "=", self._get_uidstring_module_name()),
+                ]
+            )
+            if im_data:
+                record_ids = [int(x.split(",")[1]) for x in im_data.mapped("reference")]
+                self.env[rec.model_id.model].browse(record_ids).unlink()
+        return True
+
+    def _get_uidstring_module_name(self):
+        # TODO move to static method
+        return "polars"
+
+    def _get_touchy_fields_to_import(self):
+        """inherit me
+         Some fields may break your process and could be benefit
+         of a specific process. We have to know them
+        i.e. {"res.partner": ["vat"]}
+        """
+        # TODO move to static method
+        return {}

polars_db_process/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"

polars_db_process/readme/CONTRIBUTORS.md

@@ -0,0 +1,3 @@
+- Akretion
+
+  - David BEAL <[email protected]>

polars_db_process/readme/DESCRIPTION.md

@@ -0,0 +1,8 @@
+From a db query, this module allows to transform data in Polars dataframe and process them according to rules in order to:
+
+- filter data and display 
+- obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to create/update or import
+
+For that you need to transform/arrange data to the same way