diff --git a/polars_db_process/README.rst b/polars_db_process/README.rst new file mode 100644 index 0000000000..51ffe43380 --- /dev/null +++ b/polars_db_process/README.rst @@ -0,0 +1,100 @@ +======================= +Polars Database Process +======================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github + :target: https://github.com/OCA/reporting-engine/tree/18.0/polars_db_process + :alt: OCA/reporting-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-polars_db_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +From a db query, this module allows to transform data in Polars +dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update or import + +For that you need to transform/arrange data to the same way + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Contributors +------------ + +- Akretion + + - David BEAL david.beal@akretion.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/polars_db_process/__init__.py b/polars_db_process/__init__.py new file mode 100644 index 0000000000..aee8895e7a --- /dev/null +++ b/polars_db_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/polars_db_process/__manifest__.py b/polars_db_process/__manifest__.py new file mode 100644 index 0000000000..ba7c441d0c --- /dev/null +++ b/polars_db_process/__manifest__.py @@ -0,0 +1,35 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Polars Database Process", + "version": "18.0.1.0.0", + "summary": "Allow to create a Polars dataframe from db.query and " + "check it and process it according to rules", + "category": "Reporting", + "license": "AGPL-3", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/reporting-engine", + "maintainers": ["bealdav"], + "depends": [ + "polars_process", + ], + "external_dependencies": { + "python": [ + "connectorx", + ] + }, + "data": [ + "security/ir.model.access.xml", + "wizards/df_process.xml", + "views/dataframe.xml", + "views/df_field.xml", + "views/df_source.xml", + "views/db_config.xml", + "data/demo.xml", + ], + "demo": [ + "data/demo.xml", + ], + "installable": True, +} diff --git a/polars_db_process/data/chinook.sqlite b/polars_db_process/data/chinook.sqlite new file mode 100644 index 0000000000..38a98b391b Binary files /dev/null and b/polars_db_process/data/chinook.sqlite differ diff --git a/polars_db_process/data/demo.xml b/polars_db_process/data/demo.xml new file mode 100644 index 0000000000..cc9f8cdd45 --- /dev/null +++ b/polars_db_process/data/demo.xml @@ -0,0 +1,13 @@ + + + Chinook + sqlite://.../polars_db_schema/tests/files/chinook.sqlite + + + + + Chinook Customers + + diff --git a/polars_db_process/data/files/customers.sql b/polars_db_process/data/files/customers.sql new file mode 100644 index 0000000000..f26584f9ef --- /dev/null +++ b/polars_db_process/data/files/customers.sql @@ -0,0 +1,4 @@ +--{"model": "res.partner"} +SELECT LastName AS name, State AS state, PostalCode AS zip, Email AS mail +-- , Company, Address, City, Country, Phone, Fax, SupportRepId +FROM customers; diff --git a/polars_db_process/models/__init__.py b/polars_db_process/models/__init__.py new file mode 100644 index 0000000000..f79689c656 --- /dev/null +++ b/polars_db_process/models/__init__.py @@ -0,0 +1,2 @@ +from . import df_source +from . import db_config diff --git a/polars_db_process/models/db_config.py b/polars_db_process/models/db_config.py new file mode 100644 index 0000000000..dc2ff46d76 --- /dev/null +++ b/polars_db_process/models/db_config.py @@ -0,0 +1,44 @@ +import connectorx as cx + +from odoo import _, exceptions, fields, models + +HELP = """ +String connexion samples: + +postgres://user:PASSWORD@server:port/database +mssql://user:PASSWORD@server:port/db.encrypt=true&trusted_connection=false +sqlite:///home/user/path/test.db +mysql://user:PASSWORD@server:port/database +oracle://user:PASSWORD@server:port/database +""" + + +class DbConfig(models.Model): + _name = "db.config" + _description = "External db.configuration" + _rec_name = "name" + _order = "name" + _rec_names_search = ["name"] + + name = fields.Char(required=True) + string_connexion = fields.Char(required=True, help=HELP) + password = fields.Char(help="Not required for Sqlite") + + def _get_connexion(self): + return self.string_connexion.replace("PASSWORD", self.password or "") + + def test_connexion(self): + res = self._read_sql(self._get_connexion(), "SELECT 1") + if len(res): + # Not invalid in reality + raise exceptions.ValidationError(_("Connexion OK !")) + + def _read_sql(self, connexion, query): + try: + return cx.read_sql(connexion, query, return_type="polars") + except RuntimeError as err: + raise exceptions.ValidationError(err) from err + except TimeoutError as err: + raise exceptions.ValidationError(err) from err + except Exception as err: + raise exceptions.ValidationError(err) from err diff --git a/polars_db_process/models/df_source.py b/polars_db_process/models/df_source.py new file mode 100644 index 0000000000..ea2fe51bdc --- /dev/null +++ b/polars_db_process/models/df_source.py @@ -0,0 +1,84 @@ +from pathlib import Path + +from odoo import fields, models +from odoo.modules.module import get_module_path +from odoo.tools.safe_eval import safe_eval + +MODULE = __name__[12 : __name__.index(".", 13)] + +HELP = """Supported files: .xlsx and .sql +Sql files may contains a comment on first line +to be mapped automatically with dataframe, i.e:\n +-- {'model_id': 'product.product', 'db_id': mydb} +-- {'code': 'my_delivery_address', 'db_id': mydb} +""" + + +class DfSource(models.Model): + _inherit = "df.source" + + name = fields.Char(help=HELP) + query = fields.Char() + # TODO : -> db_config_id + db_id = fields.Many2one(comodel_name="db.config", help="Database") + + def _file_hook(self, file): + "Map sql file with the right Odoo model via dataframe and the right db.config" + vals = super()._file_hook(file) + if ".sql" in file: + # TODO: improve + content = self._get_file(file).decode("utf-8") + contents = content.split("\n") + if contents: + # we only detect first line + metadata = safe_eval(contents[0].replace("--", "")) + model_name = metadata.get("model") + model = self.env["ir.model"].search([("model", "=", model_name)]) + if model_name: + # we don't want to use these dataframes + dataframes = ( + self.env["df.source"] + .search([]) + .filtered(lambda s: not s.db_id) + .mapped("dataframe_id") + ) + dataframe = self.env["dataframe"].search( + [ + ("id", "not in", dataframes.ids), + ("model_id", "=", model_name), + ] + ) + if dataframe: + # TODO use first + vals["dataframe_id"] = dataframe[0].id + db_config = self.env["db.config"].search( + [("name", "ilike", metadata.get("db_id"))] + ) + vals["db_id"] = db_config and db_config[0].id or False + else: + df = self.env["dataframe"].create( + {"code": model.name, "model_id": model and model[0].id} + ) + vals["dataframe_id"] = df.id + vals["query"] = content + return vals + + def _populate(self): + chinook = self.env.ref(f"{MODULE}.sqlite_chinook") + if chinook: + # Demo behavior only + path = Path(get_module_path(MODULE)) / "data/chinook.sqlite" + chinook.string_connexion = f"sqlite://{str(path)}" + return super()._populate() + + def _get_test_file_paths(self): + res = super()._get_test_file_paths() + res.update( + { + "polars_db_process": { + "relative_path": "data/files", + "xmlid": "polars_db_process.contact_chinook", + } + } + ) + return res diff --git a/polars_db_process/pyproject.toml b/polars_db_process/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/polars_db_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/polars_db_process/readme/CONTRIBUTORS.md b/polars_db_process/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..7997db4f34 --- /dev/null +++ b/polars_db_process/readme/CONTRIBUTORS.md @@ -0,0 +1,3 @@ +- Akretion + + - David BEAL diff --git a/polars_db_process/readme/DESCRIPTION.md b/polars_db_process/readme/DESCRIPTION.md new file mode 100644 index 0000000000..b71632d8c7 --- /dev/null +++ b/polars_db_process/readme/DESCRIPTION.md @@ -0,0 +1,8 @@ +From a db query, this module allows to transform data in Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update or import + +For that you need to transform/arrange data to the same way diff --git a/polars_db_process/security/ir.model.access.xml b/polars_db_process/security/ir.model.access.xml new file mode 100644 index 0000000000..9cc3d07de0 --- /dev/null +++ b/polars_db_process/security/ir.model.access.xml @@ -0,0 +1,11 @@ + + + Database Config + + + + + + + + diff --git a/polars_db_process/static/description/index.html b/polars_db_process/static/description/index.html new file mode 100644 index 0000000000..bd157bb925 --- /dev/null +++ b/polars_db_process/static/description/index.html @@ -0,0 +1,442 @@ + + + + + +Polars Database Process + + + +
+

Polars Database Process

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

From a db query, this module allows to transform data in Polars +dataframe and process them according to rules in order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update or import

+

For that you need to transform/arrange data to the same way

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/polars_db_process/tests/__init__.py b/polars_db_process/tests/__init__.py new file mode 100644 index 0000000000..d9b96c4fa5 --- /dev/null +++ b/polars_db_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/polars_db_process/tests/test_module.py b/polars_db_process/tests/test_module.py new file mode 100644 index 0000000000..ca14680e07 --- /dev/null +++ b/polars_db_process/tests/test_module.py @@ -0,0 +1,7 @@ +from odoo.tests.common import TransactionCase + + +class TestModule(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() diff --git a/polars_db_process/views/dataframe.xml b/polars_db_process/views/dataframe.xml new file mode 100644 index 0000000000..2385074797 --- /dev/null +++ b/polars_db_process/views/dataframe.xml @@ -0,0 +1 @@ + diff --git a/polars_db_process/views/db_config.xml b/polars_db_process/views/db_config.xml new file mode 100644 index 0000000000..74aa7ba564 --- /dev/null +++ b/polars_db_process/views/db_config.xml @@ -0,0 +1,59 @@ + + + db.config + +
+
+
+ + + + + + + + + + + +
+ PASSWORD in string connexion'll be replaced by password field +
+ + +
+ + + + + db.config + + + + + + + + + Database Config + db.config + list,form + db-config + + + + diff --git a/polars_db_process/views/df_field.xml b/polars_db_process/views/df_field.xml new file mode 100644 index 0000000000..2385074797 --- /dev/null +++ b/polars_db_process/views/df_field.xml @@ -0,0 +1 @@ + diff --git a/polars_db_process/views/df_source.xml b/polars_db_process/views/df_source.xml new file mode 100644 index 0000000000..623a488640 --- /dev/null +++ b/polars_db_process/views/df_source.xml @@ -0,0 +1,48 @@ + + + df.source + + + + + + + + + + + + + + df.source + + + + + + + {'accepted_file_extensions': '.xlsx,.sql'} + + + + + + df.source + + + + + + + + + + + diff --git a/polars_db_process/wizards/__init__.py b/polars_db_process/wizards/__init__.py new file mode 100644 index 0000000000..b8df5256dc --- /dev/null +++ b/polars_db_process/wizards/__init__.py @@ -0,0 +1 @@ +from . import df_process diff --git a/polars_db_process/wizards/df_process.py b/polars_db_process/wizards/df_process.py new file mode 100644 index 0000000000..611b4a78e7 --- /dev/null +++ b/polars_db_process/wizards/df_process.py @@ -0,0 +1,16 @@ +from odoo import _, exceptions, models + +MODULE = __name__[12 : __name__.index(".", 13)] + + +class DfProcessWiz(models.TransientModel): + _inherit = "df.process.wiz" + + def _pre_process(self): + res = super()._pre_process() + if not self.file: + self._pre_process_sql() + return res + + def _pre_process_sql(self): + raise exceptions.ValidationError(_("to be continued")) diff --git a/polars_db_process/wizards/df_process.xml b/polars_db_process/wizards/df_process.xml new file mode 100644 index 0000000000..2385074797 --- /dev/null +++ b/polars_db_process/wizards/df_process.xml @@ -0,0 +1 @@ + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..278dea77fc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# generated from manifests external_dependencies +connectorx diff --git a/test-requirement.txt b/test-requirement.txt new file mode 100644 index 0000000000..61712502d4 --- /dev/null +++ b/test-requirement.txt @@ -0,0 +1 @@ +odoo-addon-polars_process @ git+https://github.com/OCA/reporting-engine.git@refs/pull/943/head#subdirectory=polars_process