diff --git a/base_import_async/README.rst b/base_import_async/README.rst new file mode 100644 index 0000000000..bf5d466570 --- /dev/null +++ b/base_import_async/README.rst @@ -0,0 +1,154 @@ +=================== +Asynchronous Import +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:e7bcaae80fd59cc8e45b2784606eb026fba7dfcac892ad4a986f51e0dc248e1e + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png + :target: https://odoo-community.org/page/development-status + :alt: Production/Stable +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github + :target: https://github.com/OCA/queue/tree/18.0/base_import_async + :alt: OCA/queue +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module extends the standard CSV import functionality to import +files in the background using the OCA/queue framework. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +The user is presented with a new checkbox in the import screen. When +selected, the import is delayed in a background job. + +This job in turn splits the CSV file in chunks of minimum 100 lines (or +more to align with record boundaries). Each chunk is then imported in a +separate background job. + +When an import fails, the job is marked as such and the user can read +the error in the job status. The CSV chunk being imported is stored as +an attachment to the job, making it easy to download it, fix it and run +a new import, possibly in synchronous mode since the chunks are small. + +Any file that can be imported by the standard import mechanism can also +be imported in the background. + +This module's scope is limited to making standard imports asynchronous. +It does not attempt to transform the data nor automate ETL flows. + +Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): + +1. create an instance of base_import.import and populate its fields + (res_model, file, file_name), +2. invoke the do method with appropriate options (header, encoding, + separator, quoting, use_queue, chunk_size). + +Known issues / Roadmap +====================== + +- There is currently no user interface to control the chunk size, which + is currently 100 by default. Should this proves to be an issue, it is + easy to add an option to extend the import screen. +- Validation cannot be run in the background. + +Changelog +========= + +13.0.1.0.0 (2019-12-20) +----------------------- + +- [MIGRATION] from 12.0 branched at rev. a7f8031 + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion +* ACSONE SA/NV + +Contributors +------------ + +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments and let the user +control the asynchronous behaviour. + +Other contributors include: + +- Anthony Muschang (ACSONE) + +- David Béal (Akretion) + +- Jonathan Nemry (ACSONE) + +- Laurent Mignon (ACSONE) + +- Dennis Sluijk (Onestein) + +- Guewen Baconnier (Camptocamp) + +- `Trobz `__: + + - Dzung Tran + - Do Anh Duy + +- Daniel Duque (FactorLibre) + +Other credits +------------- + +The migration of this module from 17.0 to 18.0 was financially supported +by Camptocamp + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/queue `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py new file mode 100644 index 0000000000..31660d6a96 --- /dev/null +++ b/base_import_async/__init__.py @@ -0,0 +1,3 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import models diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py new file mode 100644 index 0000000000..b6a054a10b --- /dev/null +++ b/base_import_async/__manifest__.py @@ -0,0 +1,25 @@ +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +{ + "name": "Asynchronous Import", + "summary": "Import CSV files in the background", + "version": "18.0.1.0.0", + "author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)", + "license": "AGPL-3", + "website": "https://github.com/OCA/queue", + "category": "Generic Modules", + "depends": ["base_import", "queue_job"], + "data": [ + "data/queue_job_function_data.xml", + ], + "assets": { + "web.assets_backend": [ + "base_import_async/static/src/js/import_model.esm.js", + "base_import_async/static/src/xml/import_data_sidepanel.xml", + ], + }, + "installable": True, + "development_status": "Production/Stable", +} diff --git a/base_import_async/data/queue_job_function_data.xml b/base_import_async/data/queue_job_function_data.xml new file mode 100644 index 0000000000..22cc8dbab0 --- /dev/null +++ b/base_import_async/data/queue_job_function_data.xml @@ -0,0 +1,21 @@ + + + + _split_file + + + + + _import_one_chunk + + + diff --git a/base_import_async/i18n/base_import_async.pot b/base_import_async/i18n/base_import_async.pot new file mode 100644 index 0000000000..b7990c4b45 --- /dev/null +++ b/base_import_async/i18n/base_import_async.pot @@ -0,0 +1,78 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_async +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/queue_job.py:0 +#, python-format +msgid "Attachment" +msgstr "" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_base_import_import +msgid "Base Import" +msgstr "" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "" +"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to " +"%(to)s" +msgstr "" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "Import %(model)s from file %(from_file)s" +msgstr "" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "Import in the background" +msgstr "" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_queue_job +msgid "Queue Job" +msgstr "" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "" +"When checked, the import will be executed as a background job, after " +"splitting your file in small chunks that will be processed independently. " +"Use this to import very large files." +msgstr "" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "You can check the status of this job in menu 'Queue / Jobs'." +msgstr "" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "Your request is being processed" +msgstr "" diff --git a/base_import_async/i18n/es.po b/base_import_async/i18n/es.po new file mode 100644 index 0000000000..22ea084925 --- /dev/null +++ b/base_import_async/i18n/es.po @@ -0,0 +1,88 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_async +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2023-09-25 09:34+0000\n" +"Last-Translator: Asier Neira \n" +"Language-Team: none\n" +"Language: es\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.17\n" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/queue_job.py:0 +#, python-format +msgid "Attachment" +msgstr "Archivo adjunto" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_base_import_import +msgid "Base Import" +msgstr "Importación Base" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "" +"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to " +"%(to)s" +msgstr "" +"Importar %(model)s del archivo %(file_name)s - #%(chunk)s - líneas %(from)s " +"a %(to)s" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "Import %(model)s from file %(from_file)s" +msgstr "Importar %(model)s del fichero %(from_file)s" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "Import in the background" +msgstr "Importación en segundo plano" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_queue_job +msgid "Queue Job" +msgstr "Cola de trabajo" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "" +"When checked, the import will be executed as a background job, after " +"splitting your file in small chunks that will be processed independently. " +"Use this to import very large files." +msgstr "" +"Si esta opción está seleccionada, la importación se ejecutará como un " +"trabajo en segundo plano, tras dividir el archivo en pequeños trozos que se " +"procesarán de forma independiente. Utilícelo para importar archivos muy " +"grandes." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "You can check the status of this job in menu 'Queue / Jobs'." +msgstr "" +"Puede comprobar el estado de este trabajo en el menú 'Cola / Trabajos'." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "Your request is being processed" +msgstr "Su solicitud está siendo procesada" diff --git a/base_import_async/i18n/it.po b/base_import_async/i18n/it.po new file mode 100644 index 0000000000..a7b840e630 --- /dev/null +++ b/base_import_async/i18n/it.po @@ -0,0 +1,87 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_async +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2024-01-11 09:34+0000\n" +"Last-Translator: mymage \n" +"Language-Team: none\n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.17\n" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/queue_job.py:0 +#, python-format +msgid "Attachment" +msgstr "Allegato" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_base_import_import +msgid "Base Import" +msgstr "Importazione base" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "" +"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to " +"%(to)s" +msgstr "" +"Importa %(model)s dal file %(file_name)s - #%(chunk)s - linee %(from)s a " +"%(to)s" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "Import %(model)s from file %(from_file)s" +msgstr "Importa %(model)s dal file %(from_file)s" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "Import in the background" +msgstr "Importa in background" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_queue_job +msgid "Queue Job" +msgstr "Lavoro in coda" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "" +"When checked, the import will be executed as a background job, after " +"splitting your file in small chunks that will be processed independently. " +"Use this to import very large files." +msgstr "" +"Quando selezionata, l'importazione verrà eseguita com un lavoro in " +"backgroud, dopo la divisine del file in piccole parti che verranno " +"processate indipendentemente. Utilizzare questa opzione per importare file " +"di grandi dimensioni." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "You can check the status of this job in menu 'Queue / Jobs'." +msgstr "Si può controllare lo stato di questo lavoro nel menu 'Coda / Lavori'." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "Your request is being processed" +msgstr "La richiesta è in lavorazione" diff --git a/base_import_async/i18n/tr.po b/base_import_async/i18n/tr.po new file mode 100644 index 0000000000..a72ecf3894 --- /dev/null +++ b/base_import_async/i18n/tr.po @@ -0,0 +1,93 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_async +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 15.0-20221029\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-11-24 07:21+0000\n" +"PO-Revision-Date: 2022-11-24 10:25+0300\n" +"Last-Translator: \n" +"Language-Team: \n" +"Language: tr_TR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Poedit 2.4.2\n" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/queue_job.py:0 +#, python-format +msgid "Attachment" +msgstr "Ek" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_base_import_import +msgid "Base Import" +msgstr "İçe aktarım" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "" +"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to " +"%(to)s" +msgstr "" + +#. module: base_import_async +#. odoo-python +#: code:addons/base_import_async/models/base_import_import.py:0 +#, python-format +msgid "Import %(model)s from file %(from_file)s" +msgstr "" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "Import in the background" +msgstr "Arka planda içe aktarım" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_queue_job +msgid "Queue Job" +msgstr "İş Kuyruğu" + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/xml/import.xml:0 +#, python-format +msgid "" +"When checked, the import will be executed as a background job, after " +"splitting your file in small chunks that will be processed independently. " +"Use this to import very large files." +msgstr "" +"İşaretlendiğinde, içe aktarım işlemi, dosyanızı bağımsız olarak işlenecek " +"küçük parçalara ayırdıktan sonra bir arka plan işi olarak yürütülür. Çok " +"büyük dosyaları içe aktarmak için bunu kullanın." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "You can check the status of this job in menu 'Queue / Jobs'." +msgstr "Bu işin durumunu 'Kuyruk / İşler' menüsünden kontrol edebilirsiniz." + +#. module: base_import_async +#. odoo-javascript +#: code:addons/base_import_async/static/src/js/import.js:0 +#, python-format +msgid "Your request is being processed" +msgstr "İsteğiniz işleniyor" + +#, python-format +#~ msgid "Import %s from file %s" +#~ msgstr "%s dosyasından %s içe aktar" + +#, python-format +#~ msgid "Import %s from file %s - #%s - lines %s to %s" +#~ msgstr "%s dosyasından %s - #%s - %s - %s satırlarını içe aktar" diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py new file mode 100644 index 0000000000..2af21f4102 --- /dev/null +++ b/base_import_async/models/__init__.py @@ -0,0 +1,4 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import base_import_import +from . import queue_job diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py new file mode 100644 index 0000000000..f77a4bbc59 --- /dev/null +++ b/base_import_async/models/base_import_import.py @@ -0,0 +1,196 @@ +# Copyright 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 +import csv +from io import BytesIO, StringIO, TextIOWrapper +from os.path import splitext + +from odoo import _, api, models +from odoo.models import fix_import_export_id_paths + +from odoo.addons.base_import.models.base_import import ImportValidationError +from odoo.addons.queue_job.exception import FailedJobError + +# options defined in base_import/import.js +OPT_HAS_HEADER = "has_headers" +OPT_SEPARATOR = "separator" +OPT_QUOTING = "quoting" +OPT_ENCODING = "encoding" +# options defined in base_import_async/import.js +OPT_USE_QUEUE = "use_queue" +OPT_CHUNK_SIZE = "chunk_size" +# option not available in UI, but usable from scripts +OPT_PRIORITY = "priority" + +INIT_PRIORITY = 100 +DEFAULT_CHUNK_SIZE = 100 + + +class BaseImportImport(models.TransientModel): + _inherit = "base_import.import" + + def execute_import(self, fields, columns, options, dryrun=False): + if dryrun or not options.get(OPT_USE_QUEUE): + # normal import + return super().execute_import(fields, columns, options, dryrun=dryrun) + + # asynchronous import + try: + data, import_fields = self._convert_import_data(fields, options) + # Parse date and float field + data = self._parse_import_data(data, import_fields, options) + except (ImportValidationError, ValueError) as e: + return {"messages": [e.__dict__]} + + # get the translated model name to build + # a meaningful job description + search_result = ( + self.env["ir.model"].sudo().name_search(self.res_model, operator="=") + ) + if search_result: + translated_model_name = search_result[0][1] + else: + translated_model_name = self._description + description = _("Import %(model)s from file %(from_file)s") % { + "model": translated_model_name, + "from_file": self.file_name, + } + attachment = self._create_csv_attachment( + import_fields, data, options, self.file_name + ) + delayed_job = self.with_delay(description=description)._split_file( + model_name=self.res_model, + translated_model_name=translated_model_name, + attachment=attachment, + options=options, + file_name=self.file_name, + ) + self._link_attachment_to_job(delayed_job, attachment) + return [] + + def _link_attachment_to_job(self, delayed_job, attachment): + queue_job = self.env["queue.job"].search( + [("uuid", "=", delayed_job.uuid)], limit=1 + ) + attachment.write({"res_model": "queue.job", "res_id": queue_job.id}) + + @api.returns("ir.attachment") + def _create_csv_attachment(self, fields, data, options, file_name): + # write csv + f = StringIO() + writer = csv.writer( + f, + delimiter=str(options.get(OPT_SEPARATOR)) or ",", + quotechar=str(options.get(OPT_QUOTING)), + ) + encoding = options.get(OPT_ENCODING) or "utf-8" + writer.writerow(fields) + for row in data: + writer.writerow(row) + # create attachment. Remove default values from context + context = self.env.context + context_copy = {} + for key in context.keys(): + if not key.startswith("default_"): + context_copy[key] = context[key] + datas = base64.encodebytes(f.getvalue().encode(encoding)) + attachment = ( + self.env["ir.attachment"] + .with_context(**context_copy) + .create({"name": file_name, "datas": datas}) + ) + return attachment + + @staticmethod + def _read_csv_attachment(attachment, options): + decoded_datas = base64.decodebytes(attachment.datas) + encoding = options.get(OPT_ENCODING) or "utf-8" + f = TextIOWrapper(BytesIO(decoded_datas), encoding=encoding) + reader = csv.reader( + f, + delimiter=str(options.get(OPT_SEPARATOR)) or ",", + quotechar=str(options.get(OPT_QUOTING)), + ) + + fields = next(reader) + data = [row for row in reader] + return fields, data + + @staticmethod + def _extract_chunks(model_obj, fields, data, chunk_size): + """Split the data on record boundaries, in chunks of minimum chunk_size""" + fields = list(map(fix_import_export_id_paths, fields)) + row_from = 0 + for rows in model_obj._extract_records(fields, data): + rows = rows[1]["rows"] + if rows["to"] - row_from + 1 >= chunk_size: + yield row_from, rows["to"] + row_from = rows["to"] + 1 + if row_from < len(data): + yield row_from, len(data) - 1 + + def _split_file( + self, + model_name, + translated_model_name, + attachment, + options, + file_name="file.csv", + ): + """Split a CSV attachment in smaller import jobs""" + model_obj = self.env[model_name] + fields, data = self._read_csv_attachment(attachment, options) + padding = len(str(len(data))) + priority = options.get(OPT_PRIORITY, INIT_PRIORITY) + if options.get(OPT_HAS_HEADER): + header_offset = 1 + else: + header_offset = 0 + chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE + for row_from, row_to in self._extract_chunks( + model_obj, fields, data, chunk_size + ): + chunk = str(priority - INIT_PRIORITY).zfill(padding) + description = _( + "Import %(model)s from file %(file_name)s - " + "#%(chunk)s - lines %(from)s to %(to)s" + ) + description = description % { + "model": translated_model_name, + "file_name": file_name, + "chunk": chunk, + "from": row_from + 1 + header_offset, + "to": row_to + 1 + header_offset, + } + # create a CSV attachment and enqueue the job + root, ext = splitext(file_name) + attachment = self._create_csv_attachment( + fields, + data[row_from : row_to + 1], + options, + file_name=root + "-" + chunk + ext, + ) + delayed_job = self.with_delay( + description=description, priority=priority + )._import_one_chunk( + model_name=model_name, attachment=attachment, options=options + ) + self._link_attachment_to_job(delayed_job, attachment) + priority += 1 + + def _import_one_chunk(self, model_name, attachment, options): + model_obj = self.env[model_name] + fields, data = self._read_csv_attachment(attachment, options) + result = model_obj.load(fields, data) + error_message = [ + message["message"] + for message in result["messages"] + if message["type"] == "error" + ] + if error_message: + raise FailedJobError("\n".join(error_message)) + return result diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py new file mode 100644 index 0000000000..b7313505f3 --- /dev/null +++ b/base_import_async/models/queue_job.py @@ -0,0 +1,19 @@ +# Copyright 2017 ACSONE SA/NV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo import _, models + + +class QueueJob(models.Model): + """Job status and result""" + + _inherit = "queue.job" + + def _related_action_attachment(self): + return { + "name": _("Attachment"), + "type": "ir.actions.act_window", + "res_model": "ir.attachment", + "view_mode": "form", + "res_id": self.kwargs.get("att_id"), + } diff --git a/base_import_async/pyproject.toml b/base_import_async/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/base_import_async/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/base_import_async/readme/CONTRIBUTORS.md b/base_import_async/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..729ae17015 --- /dev/null +++ b/base_import_async/readme/CONTRIBUTORS.md @@ -0,0 +1,25 @@ +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments and let the user +control the asynchronous behaviour. + +Other contributors include: + +- Anthony Muschang (ACSONE) + +- David Béal (Akretion) + +- Jonathan Nemry (ACSONE) + +- Laurent Mignon (ACSONE) + +- Dennis Sluijk (Onestein) + +- Guewen Baconnier (Camptocamp) + +- [Trobz](https://trobz.com): + - Dzung Tran \<\> + - Do Anh Duy \<\> + +- Daniel Duque (FactorLibre) diff --git a/base_import_async/readme/CREDITS.md b/base_import_async/readme/CREDITS.md new file mode 100644 index 0000000000..76edf63081 --- /dev/null +++ b/base_import_async/readme/CREDITS.md @@ -0,0 +1,2 @@ +The migration of this module from 17.0 to 18.0 was financially supported +by Camptocamp diff --git a/base_import_async/readme/DESCRIPTION.md b/base_import_async/readme/DESCRIPTION.md new file mode 100644 index 0000000000..77f62de26f --- /dev/null +++ b/base_import_async/readme/DESCRIPTION.md @@ -0,0 +1,2 @@ +This module extends the standard CSV import functionality to import +files in the background using the OCA/queue framework. diff --git a/base_import_async/readme/HISTORY.md b/base_import_async/readme/HISTORY.md new file mode 100644 index 0000000000..98691d391e --- /dev/null +++ b/base_import_async/readme/HISTORY.md @@ -0,0 +1,3 @@ +## 13.0.1.0.0 (2019-12-20) + +- \[MIGRATION\] from 12.0 branched at rev. a7f8031 diff --git a/base_import_async/readme/ROADMAP.md b/base_import_async/readme/ROADMAP.md new file mode 100644 index 0000000000..1e0af37223 --- /dev/null +++ b/base_import_async/readme/ROADMAP.md @@ -0,0 +1,4 @@ +- There is currently no user interface to control the chunk size, which + is currently 100 by default. Should this proves to be an issue, it is + easy to add an option to extend the import screen. +- Validation cannot be run in the background. diff --git a/base_import_async/readme/USAGE.md b/base_import_async/readme/USAGE.md new file mode 100644 index 0000000000..5366aee78e --- /dev/null +++ b/base_import_async/readme/USAGE.md @@ -0,0 +1,25 @@ +The user is presented with a new checkbox in the import screen. When +selected, the import is delayed in a background job. + +This job in turn splits the CSV file in chunks of minimum 100 lines (or +more to align with record boundaries). Each chunk is then imported in a +separate background job. + +When an import fails, the job is marked as such and the user can read +the error in the job status. The CSV chunk being imported is stored as +an attachment to the job, making it easy to download it, fix it and run +a new import, possibly in synchronous mode since the chunks are small. + +Any file that can be imported by the standard import mechanism can also +be imported in the background. + +This module's scope is limited to making standard imports asynchronous. +It does not attempt to transform the data nor automate ETL flows. + +Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): + +1. create an instance of base_import.import and populate its fields + (res_model, file, file_name), +2. invoke the do method with appropriate options (header, encoding, + separator, quoting, use_queue, chunk_size). diff --git a/base_import_async/static/description/icon.png b/base_import_async/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/base_import_async/static/description/icon.png differ diff --git a/base_import_async/static/description/index.html b/base_import_async/static/description/index.html new file mode 100644 index 0000000000..475ce3219d --- /dev/null +++ b/base_import_async/static/description/index.html @@ -0,0 +1,495 @@ + + + + + +Asynchronous Import + + + +
+

Asynchronous Import

+ + +

Production/Stable License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

+

This module extends the standard CSV import functionality to import +files in the background using the OCA/queue framework.

+

Table of contents

+ +
+

Usage

+

The user is presented with a new checkbox in the import screen. When +selected, the import is delayed in a background job.

+

This job in turn splits the CSV file in chunks of minimum 100 lines (or +more to align with record boundaries). Each chunk is then imported in a +separate background job.

+

When an import fails, the job is marked as such and the user can read +the error in the job status. The CSV chunk being imported is stored as +an attachment to the job, making it easy to download it, fix it and run +a new import, possibly in synchronous mode since the chunks are small.

+

Any file that can be imported by the standard import mechanism can also +be imported in the background.

+

This module’s scope is limited to making standard imports asynchronous. +It does not attempt to transform the data nor automate ETL flows.

+

Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite):

+
    +
  1. create an instance of base_import.import and populate its fields +(res_model, file, file_name),
    2. +
  2. invoke the do method with appropriate options (header, encoding, +separator, quoting, use_queue, chunk_size).
    3. +
+
+
+

Known issues / Roadmap

+
    +
  • There is currently no user interface to control the chunk size, which +is currently 100 by default. Should this proves to be an issue, it is +easy to add an option to extend the import screen.
    • +
  • Validation cannot be run in the background.
    • +
+
+
+

Changelog

+
+

13.0.1.0.0 (2019-12-20)

+
    +
  • [MIGRATION] from 12.0 branched at rev. a7f8031
    • +
+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+

Sébastien Beau (Akretion) authored the initial prototype.

+

Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments and let the user +control the asynchronous behaviour.

+

Other contributors include:

+
    +
  • Anthony Muschang (ACSONE)
    • +
  • David Béal (Akretion)
    • +
  • Jonathan Nemry (ACSONE)
    • +
  • Laurent Mignon (ACSONE)
    • +
  • Dennis Sluijk (Onestein)
    • +
  • Guewen Baconnier (Camptocamp)
    • +
  • Trobz: +
    • +
  • Daniel Duque (FactorLibre)
    • +
+
+
+

Other credits

+

The migration of this module from 17.0 to 18.0 was financially supported +by Camptocamp

+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/queue project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/base_import_async/static/src/js/import_model.esm.js b/base_import_async/static/src/js/import_model.esm.js new file mode 100644 index 0000000000..9c8907cdcd --- /dev/null +++ b/base_import_async/static/src/js/import_model.esm.js @@ -0,0 +1,26 @@ +import {BaseImportModel} from "@base_import/import_model"; +import {_t} from "@web/core/l10n/translation"; +import {patch} from "@web/core/utils/patch"; + +const {document} = globalThis; + +patch(BaseImportModel.prototype, { + get importOptions() { + const options = super.importOptions; + const checkbox = document.querySelector("#oe_import_queue"); + options.use_queue = checkbox ? checkbox.checked : false; + return options; + }, + + async executeImport(isTest, totalSteps, importProgress) { + const def = super.executeImport(isTest, totalSteps, importProgress); + const checkbox = document.querySelector("#oe_import_queue"); + if (checkbox && checkbox.checked && !isTest) { + this._addMessage("warning", [ + _t("Your request is being processed"), + _t("You can check the status of this job in menu 'Queue / Jobs'."), + ]); + } + return def; + }, +}); diff --git a/base_import_async/static/src/xml/import_data_sidepanel.xml b/base_import_async/static/src/xml/import_data_sidepanel.xml new file mode 100644 index 0000000000..8c3ef0f5c8 --- /dev/null +++ b/base_import_async/static/src/xml/import_data_sidepanel.xml @@ -0,0 +1,24 @@ + + + + +
+ + +
+ + + diff --git a/base_import_async/tests/__init__.py b/base_import_async/tests/__init__.py new file mode 100644 index 0000000000..629d1e39e1 --- /dev/null +++ b/base_import_async/tests/__init__.py @@ -0,0 +1 @@ +from . import test_base_import_import diff --git a/base_import_async/tests/test_base_import_import.py b/base_import_async/tests/test_base_import_import.py new file mode 100644 index 0000000000..4a815d6707 --- /dev/null +++ b/base_import_async/tests/test_base_import_import.py @@ -0,0 +1,99 @@ +# Copyright 2024 Camptocamp +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.tests.common import RecordCapturer, TransactionCase + +from ..models.base_import_import import OPT_USE_QUEUE + + +class TestBaseImportImport(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.res_partners = cls.env["res.partner"] + cls.import_wizard = cls.env["base_import.import"] + + def test_normal_import_res_partners(self): + values = [ + [ + "name", + "email", + "is_company", + ], + [ + "partner 1", + "partner1@example.com", + "1", + ], + [ + "partner 2", + "partner2@example.com", + "0", + ], + ] + import_vals = { + "res_model": self.res_partners._name, + "file": "\n".join([";".join(values) for values in values]), + "file_type": "text/csv", + } + self.import_wizard |= self.import_wizard.create(import_vals) + opts = {"quoting": '"', "separator": ";", "has_headers": True} + preview = self.import_wizard.parse_preview(opts) + self.assertEqual( + preview["matches"], + { + 0: ["name"], + 1: ["email"], + 2: ["is_company"], + }, + ) + with RecordCapturer(self.res_partners, []) as capture: + results = self.import_wizard.execute_import( + [fnames[0] for fnames in preview["matches"].values()], + [], + opts, + ) + # if result is empty, no import error + self.assertItemsEqual(results["messages"], []) + records_created = capture.records + self.assertEqual(len(records_created), 2) + self.assertIn("partner1", records_created[0].email) + + def test_wrong_import_res_partners(self): + values = [ + [ + "name", + "email", + "date", # Adding date field to trigger parsing error + ], + [ + "partner 1", + "partner1@example.com", + "21-13-2024", + ], + [ + "partner 2", + "partner2@example.com", + "2024-13-45", + ], + ] + opts = { + "quoting": '"', + "separator": ";", + "has_headers": True, + "date_format": "%Y-%m-%d", # Set specific date format + OPT_USE_QUEUE: True, + } + import_vals = { + "res_model": self.res_partners._name, + "file": "\n".join([";".join(row) for row in values]), + "file_type": "text/csv", + } + import_wizard = self.import_wizard.create(import_vals) + preview = import_wizard.parse_preview(opts) + results = import_wizard.execute_import( + [field[0] for field in preview["matches"].values()], + ["name", "email", "date"], # Include date in fields to import + opts, + ) + self.assertTrue(any(msg["type"] == "error" for msg in results["messages"]))