From 18219d5c664564327d567f54e79047794e9d9cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 14 Nov 2014 11:09:28 +0100 Subject: [PATCH 01/77] [IMP] rename base_import_connector to base_import_async --- base_import_async/README.md | 62 +++++ base_import_async/__init__.py | 25 ++ base_import_async/__openerp__.py | 42 +++ base_import_async/models/__init__.py | 25 ++ base_import_async/models/base_import_async.py | 239 ++++++++++++++++++ base_import_async/static/src/js/import.js | 23 ++ base_import_async/static/src/xml/import.xml | 13 + base_import_async/views/base_import_async.xml | 10 + 8 files changed, 439 insertions(+) create mode 100644 base_import_async/README.md create mode 100644 base_import_async/__init__.py create mode 100644 base_import_async/__openerp__.py create mode 100644 base_import_async/models/__init__.py create mode 100644 base_import_async/models/base_import_async.py create mode 100644 base_import_async/static/src/js/import.js create mode 100644 base_import_async/static/src/xml/import.xml create mode 100644 base_import_async/views/base_import_async.xml diff --git a/base_import_async/README.md b/base_import_async/README.md new file mode 100644 index 000000000..7e8d4fcdf --- /dev/null +++ b/base_import_async/README.md @@ -0,0 +1,62 @@ +Odoo Asynchronous import module +=============================== + +This module extends the standard CSV import functionality +to import files in the background using the OCA/connector +framework. + +The user is presented with a new checkbox in the import +screen. When selected the import is delayed in a background +job. + +This job in turn splits the CSV file in chunks of minimum +100 lines (or more to align with record boundaries). Each +chunk is then imported in a separate background job. + +When an import fails, the job is marked as such and the +user can read the error in the job status. The CSV chunk +being imported is stored as an attachment to the job, making +it easy to download it, fix it and run a new import, possibly +in synchronous mode since the chunks are small. + +Scope +===== + +Any file that can be imported by the standard import mechanism +can also be imported in the background. + +This module's scope is limited to making standard imports +asynchronous. It does not attempt to transform the data nor +automate ETL flows. + +Other modules may benefit from this infrastructure in the following way + +1. create an instance of `base_import.import` and populate its fields + (`res_model`, `file`, `file_name`), +2. invoke the `do` method with appropriate options + (`header`, `encoding`, `separator`, `quoting`, + `use_connector`, `chunk_size`). + +Known limitations +================= + +* There is currently no user interface to control the chunk size, + which is currently 100 by default. Should this proves to be an issue, + it is easy to add an option to extend the import screen. +* Validation cannot be run in the background. + +Credits +======= + +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments +and let the user control the asynchronous behaviour. + +Other contributors include: + +* Anthony Muschang (ACSONE) +* David Béal (Akretion) +* Jonathan Nemry (ACSONE) +* Laurent Mignon (ACSONE) diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py new file mode 100644 index 000000000..e5f986cdf --- /dev/null +++ b/base_import_async/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for OpenERP +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### + +from . import models diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py new file mode 100644 index 000000000..1eae99e97 --- /dev/null +++ b/base_import_async/__openerp__.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for OpenERP +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### +{ + 'name': 'Asynchronous Import', + 'version': '1.0', + 'author': ['Akretion', 'ACSONE SA/NV'], + 'license': 'AGPL-3', + 'category': 'Generic Modules', + 'depends': [ + 'base_import', + 'connector', + ], + 'data': [ + 'views/base_import_async.xml', + ], + 'qweb': [ + 'static/src/xml/import.xml', + ], + 'installable': True, + 'application': False, +} diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py new file mode 100644 index 000000000..51f967457 --- /dev/null +++ b/base_import_async/models/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for OpenERP +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### + +from . import base_import_async diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py new file mode 100644 index 000000000..14df61901 --- /dev/null +++ b/base_import_async/models/base_import_async.py @@ -0,0 +1,239 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for OpenERP +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### + +import csv +import os +from cStringIO import StringIO + +from openerp.models import TransientModel +from openerp.models import fix_import_export_id_paths +from openerp.tools.translate import _ + +from openerp.addons.connector.queue.job import job, related_action +from openerp.addons.connector.session import ConnectorSession +from openerp.addons.connector.exception import FailedJobError + +# options defined in base_import/import.js +OPT_HAS_HEADER = 'headers' +OPT_SEPARATOR = 'separator' +OPT_QUOTING = 'quoting' +OPT_ENCODING = 'encoding' +# options defined in base_import_async/import.js +OPT_USE_CONNECTOR = 'use_connector' +OPT_CHUNK_SIZE = 'chunk_size' + +INIT_PRIORITY = 100 +DEFAULT_CHUNK_SIZE = 100 + + +def _encode(row, encoding): + return [cell.encode(encoding) for cell in row] + + +def _decode(row, encoding): + return [cell.decode(encoding) for cell in row] + + +def _create_csv_attachment(session, fields, data, options, file_name): + # write csv + f = StringIO() + writer = csv.writer(f, + delimiter=options.get(OPT_SEPARATOR), + quotechar=options.get(OPT_QUOTING)) + encoding = options.get(OPT_ENCODING, 'utf-8') + writer.writerow(_encode(fields, encoding)) + for row in data: + writer.writerow(_encode(row, encoding)) + # create attachment + att_id = session.pool['ir.attachment'].create(session.cr, session.uid, { + 'name': file_name, + # TODO: better way? + 'datas': f.getvalue().encode('base64'), + }, context=session.context) + return att_id + + +def _read_csv_attachment(session, att_id, options): + att = session.pool['ir.attachment'].\ + browse(session.cr, session.uid, att_id, context=session.context) + f = StringIO(att.datas.decode('base64')) + reader = csv.reader(f, + delimiter=options.get(OPT_SEPARATOR), + quotechar=options.get(OPT_QUOTING)) + encoding = options.get(OPT_ENCODING, 'utf-8') + fields = _decode(reader.next(), encoding) + data = [_decode(row, encoding) for row in reader] + return fields, data + + +def _link_attachment_to_job(session, job_uuid, att_id): + job_ids = session.pool['queue.job'].\ + search(session.cr, session.uid, [('uuid', '=', job_uuid)], + context=session.context) + session.pool['ir.attachment'].write(session.cr, session.uid, att_id, { + 'res_model': 'queue.job', + 'res_id': job_ids[0], + }, context=session.context) + + +def _extract_records(session, model_obj, fields, data, chunk_size): + """ Split the data on record boundaries, + in chunks of minimum chunk_size """ + fields = map(fix_import_export_id_paths, fields) + row_from = 0 + for rows in model_obj._extract_records(session.cr, + session.uid, + fields, + data, + context=session.context): + rows = rows[1]['rows'] + if rows['to'] - row_from + 1 >= chunk_size: + yield row_from, rows['to'] + row_from = rows['to'] + 1 + if row_from < len(data): + yield row_from, len(data) - 1 + + +def related_attachment(session, job): + attachment_id = job.args[1] + + action = { + 'name': _("Attachment"), + 'type': 'ir.actions.act_window', + 'res_model': "ir.attachment", + 'view_type': 'form', + 'view_mode': 'form', + 'res_id': attachment_id, + } + return action + + +@job +@related_action(action=related_attachment) +def import_one_chunk(session, res_model, att_id, options): + model_obj = session.pool[res_model] + fields, data = _read_csv_attachment(session, att_id, options) + result = model_obj.load(session.cr, + session.uid, + fields, + data, + context=session.context) + error_message = [message['message'] for message in result['messages'] + if message['type'] == 'error'] + if error_message: + raise FailedJobError('\n'.join(error_message)) + return result + + +@job +def split_file(session, model_name, translated_model_name, + att_id, options, file_name="file.csv"): + """ Split a CSV attachment in smaller import jobs """ + model_obj = session.pool[model_name] + fields, data = _read_csv_attachment(session, att_id, options) + padding = len(str(len(data))) + priority = INIT_PRIORITY + if options.get(OPT_HAS_HEADER): + header_offset = 1 + else: + header_offset = 0 + chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE + for row_from, row_to in _extract_records(session, + model_obj, + fields, + data, + chunk_size): + chunk = str(priority - INIT_PRIORITY).zfill(padding) + description = _("Import %s from file %s - #%s - lines %s to %s") % \ + (translated_model_name, + file_name, + chunk, + row_from + 1 + header_offset, + row_to + 1 + header_offset) + # create a CSV attachment and enqueue the job + root, ext = os.path.splitext(file_name) + att_id = _create_csv_attachment(session, + fields, + data[row_from:row_to + 1], + options, + file_name=root + '-' + chunk + ext) + job_uuid = import_one_chunk.delay(session, + model_name, + att_id, + options, + description=description, + priority=priority) + _link_attachment_to_job(session, job_uuid, att_id) + priority += 1 + + +class BaseImportConnector(TransientModel): + _inherit = 'base_import.import' + + def do(self, cr, uid, id, fields, options, dryrun=False, context=None): + if dryrun or not options.get(OPT_USE_CONNECTOR): + # normal import + return super(BaseImportConnector, self).do( + cr, uid, id, fields, options, dryrun=dryrun, context=context) + + # asynchronous import + (record,) = self.browse(cr, uid, [id], context=context) + try: + data, import_fields = self._convert_import_data( + record, fields, options, context=context) + except ValueError, e: + return [{ + 'type': 'error', + 'message': unicode(e), + 'record': False, + }] + + # get the translated model name to build + # a meaningful job description + search_result = self.pool['ir.model'].name_search( + cr, uid, args=[('model', '=', record.res_model)], context=context) + if search_result: + translated_model_name = search_result[0][1] + else: + self.pool[record.res_model]._description + description = _("Import %s from file %s") % \ + (translated_model_name, record.file_name) + + # create a CSV attachment and enqueue the job + session = ConnectorSession(cr, uid, context) + att_id = _create_csv_attachment(session, + import_fields, + data, + options, + record.file_name) + job_uuid = split_file.delay(session, + record.res_model, + translated_model_name, + att_id, + options, + file_name=record.file_name, + description=description) + _link_attachment_to_job(session, job_uuid, att_id) + + return [] diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js new file mode 100644 index 000000000..05e87e6c6 --- /dev/null +++ b/base_import_async/static/src/js/import.js @@ -0,0 +1,23 @@ +openerp.base_import_async = function (instance) { + + var QWeb = instance.web.qweb; + var _t = instance.web._t; + + instance.web.DataImport.include({ + + import_options: function () { + var options = this._super.apply(this, arguments); + options.use_connector = this.$('input.oe_import_connector').prop('checked'); + return options; + }, + + onimported: function () { + var self = this; + if (this.$('input.oe_import_connector').prop('checked')) { + this.do_notify(_t("Your request is being processed"), _t("You can check the status of this job in menu 'Connector / Jobs'.")); + } + this._super.apply(this, arguments); + }, + + }); +}; diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml new file mode 100644 index 000000000..08692f084 --- /dev/null +++ b/base_import_async/static/src/xml/import.xml @@ -0,0 +1,13 @@ + + + +
+ + +
+ + + \ No newline at end of file diff --git a/base_import_async/views/base_import_async.xml b/base_import_async/views/base_import_async.xml new file mode 100644 index 000000000..da49cf8c9 --- /dev/null +++ b/base_import_async/views/base_import_async.xml @@ -0,0 +1,10 @@ + + + + + + From 18a9dc1f4fd99c1edf9eee0069b0b906d77da8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 14 Nov 2014 12:27:04 +0100 Subject: [PATCH 02/77] [FIX] silence pylint, which btw discovered a genuine bug --- base_import_async/models/base_import_async.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 14df61901..8a1722ef3 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -115,8 +115,8 @@ def _extract_records(session, model_obj, fields, data, chunk_size): yield row_from, len(data) - 1 -def related_attachment(session, job): - attachment_id = job.args[1] +def related_attachment(session, thejob): + attachment_id = thejob.args[1] action = { 'name': _("Attachment"), @@ -191,14 +191,14 @@ def split_file(session, model_name, translated_model_name, class BaseImportConnector(TransientModel): _inherit = 'base_import.import' - def do(self, cr, uid, id, fields, options, dryrun=False, context=None): + def do(self, cr, uid, id_, fields, options, dryrun=False, context=None): if dryrun or not options.get(OPT_USE_CONNECTOR): # normal import return super(BaseImportConnector, self).do( - cr, uid, id, fields, options, dryrun=dryrun, context=context) + cr, uid, id_, fields, options, dryrun=dryrun, context=context) # asynchronous import - (record,) = self.browse(cr, uid, [id], context=context) + (record,) = self.browse(cr, uid, [id_], context=context) try: data, import_fields = self._convert_import_data( record, fields, options, context=context) @@ -216,7 +216,7 @@ def do(self, cr, uid, id, fields, options, dryrun=False, context=None): if search_result: translated_model_name = search_result[0][1] else: - self.pool[record.res_model]._description + translated_model_name = self.pool[record.res_model]._description description = _("Import %s from file %s") % \ (translated_model_name, record.file_name) From d33fcc969b0b12dd666685762f1849570f1eabbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 14 Nov 2014 13:36:48 +0100 Subject: [PATCH 03/77] [IMP] code style using session shortcuts --- base_import_async/models/base_import_async.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 8a1722ef3..fd2f1d158 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -66,17 +66,15 @@ def _create_csv_attachment(session, fields, data, options, file_name): for row in data: writer.writerow(_encode(row, encoding)) # create attachment - att_id = session.pool['ir.attachment'].create(session.cr, session.uid, { + att_id = session.create('ir.attachment', { 'name': file_name, - # TODO: better way? - 'datas': f.getvalue().encode('base64'), - }, context=session.context) + 'datas': f.getvalue().encode('base64') + }) return att_id def _read_csv_attachment(session, att_id, options): - att = session.pool['ir.attachment'].\ - browse(session.cr, session.uid, att_id, context=session.context) + att = session.browse('ir.attachment', att_id) f = StringIO(att.datas.decode('base64')) reader = csv.reader(f, delimiter=options.get(OPT_SEPARATOR), @@ -88,13 +86,11 @@ def _read_csv_attachment(session, att_id, options): def _link_attachment_to_job(session, job_uuid, att_id): - job_ids = session.pool['queue.job'].\ - search(session.cr, session.uid, [('uuid', '=', job_uuid)], - context=session.context) - session.pool['ir.attachment'].write(session.cr, session.uid, att_id, { + job_ids = session.search('queue.job', [('uuid', '=', job_uuid)]) + session.write('ir.attachment', att_id, { 'res_model': 'queue.job', 'res_id': job_ids[0], - }, context=session.context) + }) def _extract_records(session, model_obj, fields, data, chunk_size): From 1d5f7e871854f556d3f797f1fdcb45e49f6caa31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 14 Nov 2014 14:04:33 +0100 Subject: [PATCH 04/77] [IMP] slightly improve README --- base_import_async/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/base_import_async/README.md b/base_import_async/README.md index 7e8d4fcdf..9ba626483 100644 --- a/base_import_async/README.md +++ b/base_import_async/README.md @@ -1,5 +1,4 @@ -Odoo Asynchronous import module -=============================== +# Odoo Asynchronous import module This module extends the standard CSV import functionality to import files in the background using the OCA/connector @@ -19,8 +18,7 @@ being imported is stored as an attachment to the job, making it easy to download it, fix it and run a new import, possibly in synchronous mode since the chunks are small. -Scope -===== +## Scope Any file that can be imported by the standard import mechanism can also be imported in the background. @@ -30,6 +28,7 @@ asynchronous. It does not attempt to transform the data nor automate ETL flows. Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): 1. create an instance of `base_import.import` and populate its fields (`res_model`, `file`, `file_name`), @@ -37,16 +36,14 @@ Other modules may benefit from this infrastructure in the following way (`header`, `encoding`, `separator`, `quoting`, `use_connector`, `chunk_size`). -Known limitations -================= +## Known limitations * There is currently no user interface to control the chunk size, which is currently 100 by default. Should this proves to be an issue, it is easy to add an option to extend the import screen. * Validation cannot be run in the background. -Credits -======= +## Credits Sébastien Beau (Akretion) authored the initial prototype. From 52a43525c421031e7cced43db872abd1ae08816f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 14 Nov 2014 16:18:56 +0100 Subject: [PATCH 05/77] [IMP] add comma in README --- base_import_async/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/README.md b/base_import_async/README.md index 9ba626483..c187e9747 100644 --- a/base_import_async/README.md +++ b/base_import_async/README.md @@ -5,7 +5,7 @@ to import files in the background using the OCA/connector framework. The user is presented with a new checkbox in the import -screen. When selected the import is delayed in a background +screen. When selected, the import is delayed in a background job. This job in turn splits the CSV file in chunks of minimum From be79ba251971b49a87d5060a98c34723f1735990 Mon Sep 17 00:00:00 2001 From: Alexandre Fayolle Date: Tue, 3 Mar 2015 16:40:41 +0100 Subject: [PATCH 06/77] Add OCA as author of OCA addons In order to get visibility on https://www.odoo.com/apps the OCA board has decided to add the OCA as author of all the addons maintained as part of the association. --- base_import_async/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index 1eae99e97..17314e34a 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -24,7 +24,7 @@ { 'name': 'Asynchronous Import', 'version': '1.0', - 'author': ['Akretion', 'ACSONE SA/NV'], + 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Generic Modules', 'depends': [ From f340fccaeaab3a29482a63987cca0a699cc81d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 20 Apr 2015 16:35:33 +0200 Subject: [PATCH 07/77] [IMP] OpenERP -> Odoo in comments --- base_import_async/__init__.py | 2 +- base_import_async/__openerp__.py | 2 +- base_import_async/models/__init__.py | 2 +- base_import_async/models/base_import_async.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index e5f986cdf..184a3e03c 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################### # -# Module for OpenERP +# Module for Odoo # Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). # Copyright (C) 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index 17314e34a..b69d4151b 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################### # -# Module for OpenERP +# Module for Odoo # Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). # Copyright (C) 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py index 51f967457..d9d2a9344 100644 --- a/base_import_async/models/__init__.py +++ b/base_import_async/models/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################### # -# Module for OpenERP +# Module for Odoo # Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). # Copyright (C) 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index fd2f1d158..3e2837ef7 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################### # -# Module for OpenERP +# Module for Odoo # Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). # Copyright (C) 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul From d951f0a0035ec0d9f3aaf81f279f05c8a7b4bd32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 20 Apr 2015 16:36:23 +0200 Subject: [PATCH 08/77] [FIX] base_import_async: properly install connector module Cfr. http://odoo-connector.com/guides/bootstrap_connector.html#install-the-module-in-the-connector --- base_import_async/__init__.py | 1 + base_import_async/connector.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 base_import_async/connector.py diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index 184a3e03c..1d41bee0b 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -23,3 +23,4 @@ ############################################################################### from . import models +from . import connector diff --git a/base_import_async/connector.py b/base_import_async/connector.py new file mode 100644 index 000000000..b7914b40b --- /dev/null +++ b/base_import_async/connector.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for Odoo +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### + +from openerp.addons.connector.connector import install_in_connector + +install_in_connector() From ff6b6313bac16cb37afab0c9472491027686f62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Apr 2015 23:04:46 +0200 Subject: [PATCH 09/77] [IMP] base_import_async: update README --- base_import_async/{README.md => README.rst} | 33 ++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) rename base_import_async/{README.md => README.rst} (73%) diff --git a/base_import_async/README.md b/base_import_async/README.rst similarity index 73% rename from base_import_async/README.md rename to base_import_async/README.rst index c187e9747..b4e25a7f8 100644 --- a/base_import_async/README.md +++ b/base_import_async/README.rst @@ -1,4 +1,8 @@ -# Odoo Asynchronous import module +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License: AGPL-3 + +Odoo Asynchronous import module +=============================== This module extends the standard CSV import functionality to import files in the background using the OCA/connector @@ -18,7 +22,8 @@ being imported is stored as an attachment to the job, making it easy to download it, fix it and run a new import, possibly in synchronous mode since the chunks are small. -## Scope +Scope +----- Any file that can be imported by the standard import mechanism can also be imported in the background. @@ -36,14 +41,19 @@ Other modules may benefit from this infrastructure in the following way (`header`, `encoding`, `separator`, `quoting`, `use_connector`, `chunk_size`). -## Known limitations +Known limitations +================= * There is currently no user interface to control the chunk size, which is currently 100 by default. Should this proves to be an issue, it is easy to add an option to extend the import screen. * Validation cannot be run in the background. -## Credits +Credits +======= + +Contributors +------------ Sébastien Beau (Akretion) authored the initial prototype. @@ -57,3 +67,18 @@ Other contributors include: * David Béal (Akretion) * Jonathan Nemry (ACSONE) * Laurent Mignon (ACSONE) + +Maintainer +---------- + +.. image:: http://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: http://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. From ea10594e5ed67c6236423b3c97a23ae293e2163e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 21 Apr 2015 11:56:21 +0200 Subject: [PATCH 10/77] [IMP] base_import_async: further align README with OCA template --- base_import_async/README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base_import_async/README.rst b/base_import_async/README.rst index b4e25a7f8..ad6b818d8 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -8,6 +8,9 @@ This module extends the standard CSV import functionality to import files in the background using the OCA/connector framework. +Usage +===== + The user is presented with a new checkbox in the import screen. When selected, the import is delayed in a background job. @@ -22,9 +25,6 @@ being imported is stored as an attachment to the job, making it easy to download it, fix it and run a new import, possibly in synchronous mode since the chunks are small. -Scope ------ - Any file that can be imported by the standard import mechanism can also be imported in the background. @@ -41,8 +41,8 @@ Other modules may benefit from this infrastructure in the following way (`header`, `encoding`, `separator`, `quoting`, `use_connector`, `chunk_size`). -Known limitations -================= +Known issues / Roadmap +====================== * There is currently no user interface to control the chunk size, which is currently 100 by default. Should this proves to be an issue, From d8f3586c31c52ac544bc6af8ce260e5f8d7e9dbe Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Fri, 15 May 2015 20:56:02 +0200 Subject: [PATCH 11/77] base_import_async: allow to set the base priority In the options of an import. This option is too technical to be displayed on the frontend, but the wizard can be called with the option. It is useful when we have to run several imports one after the other and the 'other' has dependencies on the 'one'. --- base_import_async/models/base_import_async.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 3e2837ef7..06c1d4de4 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -42,6 +42,8 @@ # options defined in base_import_async/import.js OPT_USE_CONNECTOR = 'use_connector' OPT_CHUNK_SIZE = 'chunk_size' +# option not available in UI, but usable from scripts +OPT_PRIORITY = 'priority' INIT_PRIORITY = 100 DEFAULT_CHUNK_SIZE = 100 @@ -149,7 +151,7 @@ def split_file(session, model_name, translated_model_name, model_obj = session.pool[model_name] fields, data = _read_csv_attachment(session, att_id, options) padding = len(str(len(data))) - priority = INIT_PRIORITY + priority = options.get(OPT_PRIORITY, INIT_PRIORITY) if options.get(OPT_HAS_HEADER): header_offset = 1 else: From 1fc86694209991736bb6b208b0230af72bd933fc Mon Sep 17 00:00:00 2001 From: Yannick Vaucher Date: Fri, 22 May 2015 19:45:49 +0200 Subject: [PATCH 12/77] Add bug tracker link on README.rst --- base_import_async/README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/base_import_async/README.rst b/base_import_async/README.rst index ad6b818d8..26629481f 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -49,6 +49,16 @@ Known issues / Roadmap it is easy to add an option to extend the import screen. * Validation cannot be run in the background. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + + Credits ======= From dcc7eae6e1d4fb70c84c0cb194842435bfe64133 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 10 Jun 2015 14:38:24 +0200 Subject: [PATCH 13/77] Replace deprecated functions Shortcut methods session.create(), session.write(), session.browse(), session.search() should now be directly called on session.env['model'] --- base_import_async/models/base_import_async.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 06c1d4de4..a7d133061 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -68,15 +68,15 @@ def _create_csv_attachment(session, fields, data, options, file_name): for row in data: writer.writerow(_encode(row, encoding)) # create attachment - att_id = session.create('ir.attachment', { + attachment = session.env['ir.attachment'].create({ 'name': file_name, 'datas': f.getvalue().encode('base64') }) - return att_id + return attachment.id def _read_csv_attachment(session, att_id, options): - att = session.browse('ir.attachment', att_id) + att = session.env['ir.attachment'].browse(att_id) f = StringIO(att.datas.decode('base64')) reader = csv.reader(f, delimiter=options.get(OPT_SEPARATOR), @@ -88,10 +88,10 @@ def _read_csv_attachment(session, att_id, options): def _link_attachment_to_job(session, job_uuid, att_id): - job_ids = session.search('queue.job', [('uuid', '=', job_uuid)]) - session.write('ir.attachment', att_id, { + job = session.env['queue.job'].search([('uuid', '=', job_uuid)], limit=1) + session.env['ir.attachment'].browse(att_id).write({ 'res_model': 'queue.job', - 'res_id': job_ids[0], + 'res_id': job.id, }) From 147093282a8b32c34cd3cebe476bf3dc08e9773d Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 10 Jun 2015 14:45:43 +0200 Subject: [PATCH 14/77] Remove call to install_in_connector() The function has been deprecated in https://github.com/OCA/connector/commit/6e3ab38c788b003e7cdccbc949d340bd724500d7 --- base_import_async/__init__.py | 1 - base_import_async/connector.py | 27 --------------------------- 2 files changed, 28 deletions(-) delete mode 100644 base_import_async/connector.py diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index 1d41bee0b..184a3e03c 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -23,4 +23,3 @@ ############################################################################### from . import models -from . import connector diff --git a/base_import_async/connector.py b/base_import_async/connector.py deleted file mode 100644 index b7914b40b..000000000 --- a/base_import_async/connector.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### - -from openerp.addons.connector.connector import install_in_connector - -install_in_connector() From 57e53123ed407425b52f1aff51aae7d726827a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 9 Oct 2015 09:59:51 +0200 Subject: [PATCH 15/77] [UPD] prefix versions with 8.0 --- base_import_async/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index b69d4151b..2d3a8dd13 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -23,7 +23,7 @@ ############################################################################### { 'name': 'Asynchronous Import', - 'version': '1.0', + 'version': '8.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Generic Modules', From 8deda99aaa3395d72f4b902b1bd4ca374e7b690a Mon Sep 17 00:00:00 2001 From: "Pedro M. Baeza" Date: Wed, 14 Oct 2015 02:23:55 +0200 Subject: [PATCH 16/77] [MIG] Make modules uninstallable --- base_import_async/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index 2d3a8dd13..73f30062d 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -37,6 +37,6 @@ 'qweb': [ 'static/src/xml/import.xml', ], - 'installable': True, + 'installable': False, 'application': False, } From 8392c0a06de47eb3b934969a8b1c68d9da82e03d Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 19 Oct 2015 17:10:27 +0200 Subject: [PATCH 17/77] [FIX] Fix new/old api compatibility api decorator only recognize 'res_id' or 'id' as keyword args --- base_import_async/models/base_import_async.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index a7d133061..a6958b700 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -189,14 +189,15 @@ def split_file(session, model_name, translated_model_name, class BaseImportConnector(TransientModel): _inherit = 'base_import.import' - def do(self, cr, uid, id_, fields, options, dryrun=False, context=None): + def do(self, cr, uid, res_id, fields, options, dryrun=False, context=None): if dryrun or not options.get(OPT_USE_CONNECTOR): # normal import return super(BaseImportConnector, self).do( - cr, uid, id_, fields, options, dryrun=dryrun, context=context) + cr, uid, res_id, fields, options, dryrun=dryrun, + context=context) # asynchronous import - (record,) = self.browse(cr, uid, [id_], context=context) + (record,) = self.browse(cr, uid, [res_id], context=context) try: data, import_fields = self._convert_import_data( record, fields, options, context=context) From 6564148942c25b27cb0c06871d8782476a6e343e Mon Sep 17 00:00:00 2001 From: waveyeung Date: Thu, 19 Nov 2015 08:47:38 +0800 Subject: [PATCH 18/77] Update base_import_async.py --- base_import_async/models/base_import_async.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index a6958b700..2a2c60748 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -61,8 +61,8 @@ def _create_csv_attachment(session, fields, data, options, file_name): # write csv f = StringIO() writer = csv.writer(f, - delimiter=options.get(OPT_SEPARATOR), - quotechar=options.get(OPT_QUOTING)) + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING))) encoding = options.get(OPT_ENCODING, 'utf-8') writer.writerow(_encode(fields, encoding)) for row in data: From 83841968915da2d9382516cea5dafb4ed48ead92 Mon Sep 17 00:00:00 2001 From: waveyeung Date: Thu, 19 Nov 2015 18:50:46 +0800 Subject: [PATCH 19/77] Update base_import_async.py --- base_import_async/models/base_import_async.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 2a2c60748..54530fce6 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -79,8 +79,8 @@ def _read_csv_attachment(session, att_id, options): att = session.env['ir.attachment'].browse(att_id) f = StringIO(att.datas.decode('base64')) reader = csv.reader(f, - delimiter=options.get(OPT_SEPARATOR), - quotechar=options.get(OPT_QUOTING)) + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING))) encoding = options.get(OPT_ENCODING, 'utf-8') fields = _decode(reader.next(), encoding) data = [_decode(row, encoding) for row in reader] From 2085617f96189a88fce5dc1ecbcda35bd0f88d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 3 Mar 2016 16:56:40 +0100 Subject: [PATCH 20/77] migrate base_import_async and it's test to 9.0 --- base_import_async/__openerp__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index 73f30062d..a5ad02995 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -23,7 +23,7 @@ ############################################################################### { 'name': 'Asynchronous Import', - 'version': '8.0.1.0.0', + 'version': '9.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Generic Modules', @@ -37,6 +37,6 @@ 'qweb': [ 'static/src/xml/import.xml', ], - 'installable': False, + 'installable': True, 'application': False, } From d862023ab5b057b7dee52fb1ff1499a7c0951acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 3 Mar 2016 18:07:35 +0100 Subject: [PATCH 21/77] migrate javascript --- base_import_async/static/src/js/import.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js index 05e87e6c6..9fd35f03f 100644 --- a/base_import_async/static/src/js/import.js +++ b/base_import_async/static/src/js/import.js @@ -1,9 +1,13 @@ -openerp.base_import_async = function (instance) { +odoo.define('base_import_async.import', function (require) { + "use strict"; - var QWeb = instance.web.qweb; - var _t = instance.web._t; + var core = require('web.core'); + var _t = core._t; + require('base_import.import'); - instance.web.DataImport.include({ + var DataImport = core.action_registry.get('import'); + + DataImport = DataImport.include({ import_options: function () { var options = this._super.apply(this, arguments); @@ -20,4 +24,5 @@ openerp.base_import_async = function (instance) { }, }); -}; + +}); From 54160a71df58be79300dc04c28a1da943cd4eeb4 Mon Sep 17 00:00:00 2001 From: "Pedro M. Baeza" Date: Thu, 6 Oct 2016 14:49:20 +0200 Subject: [PATCH 22/77] [MIG] Make modules uninstallable --- base_import_async/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/__openerp__.py b/base_import_async/__openerp__.py index a5ad02995..d6651aa6e 100644 --- a/base_import_async/__openerp__.py +++ b/base_import_async/__openerp__.py @@ -37,6 +37,6 @@ 'qweb': [ 'static/src/xml/import.xml', ], - 'installable': True, + 'installable': False, 'application': False, } From e345a85c33f233ba39542151692ed3800cfd33e7 Mon Sep 17 00:00:00 2001 From: "Pedro M. Baeza" Date: Thu, 6 Oct 2016 14:49:21 +0200 Subject: [PATCH 23/77] [MIG] Rename manifest files --- base_import_async/{__openerp__.py => __manifest__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename base_import_async/{__openerp__.py => __manifest__.py} (100%) diff --git a/base_import_async/__openerp__.py b/base_import_async/__manifest__.py similarity index 100% rename from base_import_async/__openerp__.py rename to base_import_async/__manifest__.py From 6636c628186361f29a32b2062558b1430105c4c7 Mon Sep 17 00:00:00 2001 From: Daniel Torres Date: Fri, 4 Nov 2016 13:36:36 -0200 Subject: [PATCH 24/77] Porting to new API --- base_import_async/__manifest__.py | 4 +- base_import_async/models/base_import_async.py | 58 ++++++++----------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index d6651aa6e..2cbe38710 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -23,7 +23,7 @@ ############################################################################### { 'name': 'Asynchronous Import', - 'version': '9.0.1.0.0', + 'version': '10.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Generic Modules', @@ -37,6 +37,6 @@ 'qweb': [ 'static/src/xml/import.xml', ], - 'installable': False, + 'installable': True, 'application': False, } diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index 54530fce6..b8c21b5a1 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -26,13 +26,13 @@ import os from cStringIO import StringIO -from openerp.models import TransientModel -from openerp.models import fix_import_export_id_paths -from openerp.tools.translate import _ +from odoo.models import TransientModel +from odoo.models import fix_import_export_id_paths +from odoo.tools.translate import _ -from openerp.addons.connector.queue.job import job, related_action -from openerp.addons.connector.session import ConnectorSession -from openerp.addons.connector.exception import FailedJobError +from odoo.addons.connector.queue.job import job, related_action +from odoo.addons.connector.session import ConnectorSession +from odoo.addons.connector.exception import FailedJobError # options defined in base_import/import.js OPT_HAS_HEADER = 'headers' @@ -100,11 +100,8 @@ def _extract_records(session, model_obj, fields, data, chunk_size): in chunks of minimum chunk_size """ fields = map(fix_import_export_id_paths, fields) row_from = 0 - for rows in model_obj._extract_records(session.cr, - session.uid, - fields, - data, - context=session.context): + for rows in model_obj._extract_records(fields, + data): rows = rows[1]['rows'] if rows['to'] - row_from + 1 >= chunk_size: yield row_from, rows['to'] @@ -130,13 +127,10 @@ def related_attachment(session, thejob): @job @related_action(action=related_attachment) def import_one_chunk(session, res_model, att_id, options): - model_obj = session.pool[res_model] + model_obj = session.env[res_model] fields, data = _read_csv_attachment(session, att_id, options) - result = model_obj.load(session.cr, - session.uid, - fields, - data, - context=session.context) + result = model_obj.load(fields, + data) error_message = [message['message'] for message in result['messages'] if message['type'] == 'error'] if error_message: @@ -148,7 +142,7 @@ def import_one_chunk(session, res_model, att_id, options): def split_file(session, model_name, translated_model_name, att_id, options, file_name="file.csv"): """ Split a CSV attachment in smaller import jobs """ - model_obj = session.pool[model_name] + model_obj = session.env[model_name] fields, data = _read_csv_attachment(session, att_id, options) padding = len(str(len(data))) priority = options.get(OPT_PRIORITY, INIT_PRIORITY) @@ -189,19 +183,18 @@ def split_file(session, model_name, translated_model_name, class BaseImportConnector(TransientModel): _inherit = 'base_import.import' - def do(self, cr, uid, res_id, fields, options, dryrun=False, context=None): + def do(self, fields, options, dryrun=False): if dryrun or not options.get(OPT_USE_CONNECTOR): # normal import return super(BaseImportConnector, self).do( - cr, uid, res_id, fields, options, dryrun=dryrun, - context=context) + fields, options, dryrun=dryrun) # asynchronous import - (record,) = self.browse(cr, uid, [res_id], context=context) try: - data, import_fields = self._convert_import_data( - record, fields, options, context=context) - except ValueError, e: + data, import_fields = self._convert_import_data(fields, options) + # Parse date and float field + data = self._parse_import_data(data, import_fields, options) + except ValueError as e: return [{ 'type': 'error', 'message': unicode(e), @@ -210,28 +203,27 @@ def do(self, cr, uid, res_id, fields, options, dryrun=False, context=None): # get the translated model name to build # a meaningful job description - search_result = self.pool['ir.model'].name_search( - cr, uid, args=[('model', '=', record.res_model)], context=context) + search_result = self.env['ir.model'].name_search(self._name) if search_result: translated_model_name = search_result[0][1] else: - translated_model_name = self.pool[record.res_model]._description + translated_model_name = self._description description = _("Import %s from file %s") % \ - (translated_model_name, record.file_name) + (translated_model_name, self.file_name) # create a CSV attachment and enqueue the job - session = ConnectorSession(cr, uid, context) + session = ConnectorSession(self.env.cr, self.env.uid, context=self.env.context) att_id = _create_csv_attachment(session, import_fields, data, options, - record.file_name) + self.file_name) job_uuid = split_file.delay(session, - record.res_model, + self.res_model, translated_model_name, att_id, options, - file_name=record.file_name, + file_name=self.file_name, description=description) _link_attachment_to_job(session, job_uuid, att_id) From f52533fd1487d458e478f33ead972b5ef4f73763 Mon Sep 17 00:00:00 2001 From: Daniel Torres Date: Mon, 7 Nov 2016 12:01:40 -0200 Subject: [PATCH 25/77] Whitespace --- base_import_async/models/base_import_async.py | 3 ++- base_import_async/static/src/xml/import.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py index b8c21b5a1..bf00a464a 100644 --- a/base_import_async/models/base_import_async.py +++ b/base_import_async/models/base_import_async.py @@ -212,7 +212,8 @@ def do(self, fields, options, dryrun=False): (translated_model_name, self.file_name) # create a CSV attachment and enqueue the job - session = ConnectorSession(self.env.cr, self.env.uid, context=self.env.context) + session = ConnectorSession(self.env.cr, self.env.uid, + context=self.env.context) att_id = _create_csv_attachment(session, import_fields, data, diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml index 08692f084..11167bb56 100644 --- a/base_import_async/static/src/xml/import.xml +++ b/base_import_async/static/src/xml/import.xml @@ -10,4 +10,4 @@ Use this to import very large files."> - \ No newline at end of file + From c7579561034dc4c431a0decec33193fe02c68999 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Tue, 21 Mar 2017 18:03:27 +0100 Subject: [PATCH 26/77] [MIG] base_import_async: replace connector by queue_job --- base_import_async/__manifest__.py | 3 +- base_import_async/models/__init__.py | 3 +- base_import_async/models/base_import_async.py | 231 ------------------ .../models/base_import_import.py | 205 ++++++++++++++++ base_import_async/models/queue_job.py | 23 ++ base_import_async/static/src/js/import.js | 6 +- base_import_async/static/src/xml/import.xml | 6 +- 7 files changed, 237 insertions(+), 240 deletions(-) delete mode 100644 base_import_async/models/base_import_async.py create mode 100644 base_import_async/models/base_import_import.py create mode 100644 base_import_async/models/queue_job.py diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 2cbe38710..758696214 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -29,7 +29,7 @@ 'category': 'Generic Modules', 'depends': [ 'base_import', - 'connector', + 'queue_job', ], 'data': [ 'views/base_import_async.xml', @@ -38,5 +38,4 @@ 'static/src/xml/import.xml', ], 'installable': True, - 'application': False, } diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py index d9d2a9344..29ab48105 100644 --- a/base_import_async/models/__init__.py +++ b/base_import_async/models/__init__.py @@ -22,4 +22,5 @@ # ############################################################################### -from . import base_import_async +from . import base_import_import +from . import queue_job diff --git a/base_import_async/models/base_import_async.py b/base_import_async/models/base_import_async.py deleted file mode 100644 index bf00a464a..000000000 --- a/base_import_async/models/base_import_async.py +++ /dev/null @@ -1,231 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### - -import csv -import os -from cStringIO import StringIO - -from odoo.models import TransientModel -from odoo.models import fix_import_export_id_paths -from odoo.tools.translate import _ - -from odoo.addons.connector.queue.job import job, related_action -from odoo.addons.connector.session import ConnectorSession -from odoo.addons.connector.exception import FailedJobError - -# options defined in base_import/import.js -OPT_HAS_HEADER = 'headers' -OPT_SEPARATOR = 'separator' -OPT_QUOTING = 'quoting' -OPT_ENCODING = 'encoding' -# options defined in base_import_async/import.js -OPT_USE_CONNECTOR = 'use_connector' -OPT_CHUNK_SIZE = 'chunk_size' -# option not available in UI, but usable from scripts -OPT_PRIORITY = 'priority' - -INIT_PRIORITY = 100 -DEFAULT_CHUNK_SIZE = 100 - - -def _encode(row, encoding): - return [cell.encode(encoding) for cell in row] - - -def _decode(row, encoding): - return [cell.decode(encoding) for cell in row] - - -def _create_csv_attachment(session, fields, data, options, file_name): - # write csv - f = StringIO() - writer = csv.writer(f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING))) - encoding = options.get(OPT_ENCODING, 'utf-8') - writer.writerow(_encode(fields, encoding)) - for row in data: - writer.writerow(_encode(row, encoding)) - # create attachment - attachment = session.env['ir.attachment'].create({ - 'name': file_name, - 'datas': f.getvalue().encode('base64') - }) - return attachment.id - - -def _read_csv_attachment(session, att_id, options): - att = session.env['ir.attachment'].browse(att_id) - f = StringIO(att.datas.decode('base64')) - reader = csv.reader(f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING))) - encoding = options.get(OPT_ENCODING, 'utf-8') - fields = _decode(reader.next(), encoding) - data = [_decode(row, encoding) for row in reader] - return fields, data - - -def _link_attachment_to_job(session, job_uuid, att_id): - job = session.env['queue.job'].search([('uuid', '=', job_uuid)], limit=1) - session.env['ir.attachment'].browse(att_id).write({ - 'res_model': 'queue.job', - 'res_id': job.id, - }) - - -def _extract_records(session, model_obj, fields, data, chunk_size): - """ Split the data on record boundaries, - in chunks of minimum chunk_size """ - fields = map(fix_import_export_id_paths, fields) - row_from = 0 - for rows in model_obj._extract_records(fields, - data): - rows = rows[1]['rows'] - if rows['to'] - row_from + 1 >= chunk_size: - yield row_from, rows['to'] - row_from = rows['to'] + 1 - if row_from < len(data): - yield row_from, len(data) - 1 - - -def related_attachment(session, thejob): - attachment_id = thejob.args[1] - - action = { - 'name': _("Attachment"), - 'type': 'ir.actions.act_window', - 'res_model': "ir.attachment", - 'view_type': 'form', - 'view_mode': 'form', - 'res_id': attachment_id, - } - return action - - -@job -@related_action(action=related_attachment) -def import_one_chunk(session, res_model, att_id, options): - model_obj = session.env[res_model] - fields, data = _read_csv_attachment(session, att_id, options) - result = model_obj.load(fields, - data) - error_message = [message['message'] for message in result['messages'] - if message['type'] == 'error'] - if error_message: - raise FailedJobError('\n'.join(error_message)) - return result - - -@job -def split_file(session, model_name, translated_model_name, - att_id, options, file_name="file.csv"): - """ Split a CSV attachment in smaller import jobs """ - model_obj = session.env[model_name] - fields, data = _read_csv_attachment(session, att_id, options) - padding = len(str(len(data))) - priority = options.get(OPT_PRIORITY, INIT_PRIORITY) - if options.get(OPT_HAS_HEADER): - header_offset = 1 - else: - header_offset = 0 - chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE - for row_from, row_to in _extract_records(session, - model_obj, - fields, - data, - chunk_size): - chunk = str(priority - INIT_PRIORITY).zfill(padding) - description = _("Import %s from file %s - #%s - lines %s to %s") % \ - (translated_model_name, - file_name, - chunk, - row_from + 1 + header_offset, - row_to + 1 + header_offset) - # create a CSV attachment and enqueue the job - root, ext = os.path.splitext(file_name) - att_id = _create_csv_attachment(session, - fields, - data[row_from:row_to + 1], - options, - file_name=root + '-' + chunk + ext) - job_uuid = import_one_chunk.delay(session, - model_name, - att_id, - options, - description=description, - priority=priority) - _link_attachment_to_job(session, job_uuid, att_id) - priority += 1 - - -class BaseImportConnector(TransientModel): - _inherit = 'base_import.import' - - def do(self, fields, options, dryrun=False): - if dryrun or not options.get(OPT_USE_CONNECTOR): - # normal import - return super(BaseImportConnector, self).do( - fields, options, dryrun=dryrun) - - # asynchronous import - try: - data, import_fields = self._convert_import_data(fields, options) - # Parse date and float field - data = self._parse_import_data(data, import_fields, options) - except ValueError as e: - return [{ - 'type': 'error', - 'message': unicode(e), - 'record': False, - }] - - # get the translated model name to build - # a meaningful job description - search_result = self.env['ir.model'].name_search(self._name) - if search_result: - translated_model_name = search_result[0][1] - else: - translated_model_name = self._description - description = _("Import %s from file %s") % \ - (translated_model_name, self.file_name) - - # create a CSV attachment and enqueue the job - session = ConnectorSession(self.env.cr, self.env.uid, - context=self.env.context) - att_id = _create_csv_attachment(session, - import_fields, - data, - options, - self.file_name) - job_uuid = split_file.delay(session, - self.res_model, - translated_model_name, - att_id, - options, - file_name=self.file_name, - description=description) - _link_attachment_to_job(session, job_uuid, att_id) - - return [] diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py new file mode 100644 index 000000000..403816c32 --- /dev/null +++ b/base_import_async/models/base_import_import.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +############################################################################### +# +# Module for Odoo +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################### + +import csv +import os +from cStringIO import StringIO + +from odoo import api, _ +from odoo.models import TransientModel +from odoo.models import fix_import_export_id_paths + +from odoo.addons.queue_job.job import job, related_action +from odoo.addons.queue_job.exception import FailedJobError + +# options defined in base_import/import.js +OPT_HAS_HEADER = 'headers' +OPT_SEPARATOR = 'separator' +OPT_QUOTING = 'quoting' +OPT_ENCODING = 'encoding' +# options defined in base_import_async/import.js +OPT_USE_QUEUE = 'use_queue' +OPT_CHUNK_SIZE = 'chunk_size' +# option not available in UI, but usable from scripts +OPT_PRIORITY = 'priority' + +INIT_PRIORITY = 100 +DEFAULT_CHUNK_SIZE = 100 + + +def _encode(row, encoding): + return [cell.encode(encoding) for cell in row] + + +def _decode(row, encoding): + return [cell.decode(encoding) for cell in row] + + +class BaseImportImport(TransientModel): + _inherit = 'base_import.import' + + @api.multi + def do(self, fields, options, dryrun=False): + if dryrun or not options.get(OPT_USE_QUEUE): + # normal import + return super(BaseImportImport, self).do( + fields, options, dryrun=dryrun) + + # asynchronous import + try: + data, import_fields = self._convert_import_data(fields, options) + # Parse date and float field + data = self._parse_import_data(data, import_fields, options) + except ValueError as e: + return [{ + 'type': 'error', + 'message': unicode(e), + 'record': False, + }] + + # get the translated model name to build + # a meaningful job description + search_result = self.env['ir.model'].name_search( + self.res_model, operator='=') + if search_result: + translated_model_name = search_result[0][1] + else: + translated_model_name = self._description + description = _("Import %s from file %s") % \ + (translated_model_name, self.file_name) + att_id = self._create_csv_attachment( + import_fields, data, options, self.file_name) + job_uuid = self.with_delay(description=description)._split_file( + model_name=self.res_model, + translated_model_name=translated_model_name, + att_id=att_id, + options=options, + file_name=self.file_name + ) + self._link_attachment_to_job(job_uuid, att_id) + return [] + + @api.model + def _link_attachment_to_job(self, job_uuid, att_id): + job = self.env['queue.job'].search([('uuid', '=', job_uuid)], limit=1) + self.env['ir.attachment'].browse(att_id).write({ + 'res_model': 'queue.job', + 'res_id': job.id, + }) + + @api.model + def _create_csv_attachment(self, fields, data, options, file_name): + # write csv + f = StringIO() + writer = csv.writer(f, + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING))) + encoding = options.get(OPT_ENCODING, 'utf-8') + writer.writerow(_encode(fields, encoding)) + for row in data: + writer.writerow(_encode(row, encoding)) + # create attachment + attachment = self.env['ir.attachment'].create({ + 'name': file_name, + 'datas': f.getvalue().encode('base64'), + 'datas_fname': file_name + }) + return attachment.id + + @api.model + def _read_csv_attachment(self, att_id, options): + att = self.env['ir.attachment'].browse(att_id) + f = StringIO(att.datas.decode('base64')) + reader = csv.reader(f, + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING))) + encoding = options.get(OPT_ENCODING, 'utf-8') + fields = _decode(reader.next(), encoding) + data = [_decode(row, encoding) for row in reader] + return fields, data + + @api.model + def _extract_records(self, model_obj, fields, data, chunk_size): + """ Split the data on record boundaries, + in chunks of minimum chunk_size """ + fields = map(fix_import_export_id_paths, fields) + row_from = 0 + for rows in model_obj._extract_records(fields, + data): + rows = rows[1]['rows'] + if rows['to'] - row_from + 1 >= chunk_size: + yield row_from, rows['to'] + row_from = rows['to'] + 1 + if row_from < len(data): + yield row_from, len(data) - 1 + + @api.model + @job + @related_action('_related_action_attachment') + def _split_file(self, model_name, translated_model_name, + att_id, options, file_name="file.csv"): + """ Split a CSV attachment in smaller import jobs """ + model_obj = self.env[model_name] + fields, data = self._read_csv_attachment(att_id, options) + padding = len(str(len(data))) + priority = options.get(OPT_PRIORITY, INIT_PRIORITY) + if options.get(OPT_HAS_HEADER): + header_offset = 1 + else: + header_offset = 0 + chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE + for row_from, row_to in self._extract_records( + model_obj, fields, data, chunk_size): + chunk = str(priority - INIT_PRIORITY).zfill(padding) + description = _("Import %s from file %s - #%s - lines %s to %s") + description = description % (translated_model_name, + file_name, + chunk, + row_from + 1 + header_offset, + row_to + 1 + header_offset) + # create a CSV attachment and enqueue the job + root, ext = os.path.splitext(file_name) + att_id = self._create_csv_attachment( + fields, data[row_from:row_to + 1], options, + file_name=root + '-' + chunk + ext) + job_uuid = self.with_delay( + description=description, priority=priority)._import_one_chunk( + model_name=model_name, + att_id=att_id, + options=options) + self._link_attachment_to_job(job_uuid, att_id) + priority += 1 + + @api.model + @job + @related_action('_related_action_attachment') + def _import_one_chunk(self, model_name, att_id, options): + model_obj = self.env[model_name] + fields, data = self._read_csv_attachment(att_id, options) + result = model_obj.load(fields, data) + error_message = [message['message'] for message in result['messages'] + if message['type'] == 'error'] + if error_message: + raise FailedJobError('\n'.join(error_message)) + return result diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py new file mode 100644 index 000000000..b9bcb79cc --- /dev/null +++ b/base_import_async/models/queue_job.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 ACSONE SA/NV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models, _ + + +class QueueJob(models.Model): + """ Job status and result """ + _inherit = 'queue.job' + + @api.multi + def _related_action_attachment(self): + res_id = self.kwargs.get('att_id') + action = { + 'name': _("Attachment"), + 'type': 'ir.actions.act_window', + 'res_model': "ir.attachment", + 'view_type': 'form', + 'view_mode': 'form', + 'res_id': res_id, + } + return action diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js index 9fd35f03f..7ab7d08f7 100644 --- a/base_import_async/static/src/js/import.js +++ b/base_import_async/static/src/js/import.js @@ -11,14 +11,14 @@ odoo.define('base_import_async.import', function (require) { import_options: function () { var options = this._super.apply(this, arguments); - options.use_connector = this.$('input.oe_import_connector').prop('checked'); + options.use_queue = this.$('input.oe_import_queue').prop('checked'); return options; }, onimported: function () { var self = this; - if (this.$('input.oe_import_connector').prop('checked')) { - this.do_notify(_t("Your request is being processed"), _t("You can check the status of this job in menu 'Connector / Jobs'.")); + if (this.$('input.oe_import_queue').prop('checked')) { + this.do_notify(_t("Your request is being processed"), _t("You can check the status of this job in menu 'Queue / Jobs'.")); } this._super.apply(this, arguments); }, diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml index 11167bb56..22300db51 100644 --- a/base_import_async/static/src/xml/import.xml +++ b/base_import_async/static/src/xml/import.xml @@ -4,9 +4,9 @@
- - + +
From 62f39cc8ea39a6adc88200efc286f54b8f3540db Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Thu, 23 Mar 2017 10:12:29 +0100 Subject: [PATCH 27/77] [FIX] base_import_async: with_delay returns an instance of Job --- base_import_async/models/base_import_import.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 403816c32..063d5e49b 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -90,22 +90,23 @@ def do(self, fields, options, dryrun=False): (translated_model_name, self.file_name) att_id = self._create_csv_attachment( import_fields, data, options, self.file_name) - job_uuid = self.with_delay(description=description)._split_file( + delayed_job = self.with_delay(description=description)._split_file( model_name=self.res_model, translated_model_name=translated_model_name, att_id=att_id, options=options, file_name=self.file_name ) - self._link_attachment_to_job(job_uuid, att_id) + self._link_attachment_to_job(delayed_job, att_id) return [] @api.model - def _link_attachment_to_job(self, job_uuid, att_id): - job = self.env['queue.job'].search([('uuid', '=', job_uuid)], limit=1) + def _link_attachment_to_job(self, delayed_job, att_id): + queue_job = self.env['queue.job'].search( + [('uuid', '=', delayed_job.uuid)], limit=1) self.env['ir.attachment'].browse(att_id).write({ 'res_model': 'queue.job', - 'res_id': job.id, + 'res_id': queue_job.id, }) @api.model @@ -183,12 +184,12 @@ def _split_file(self, model_name, translated_model_name, att_id = self._create_csv_attachment( fields, data[row_from:row_to + 1], options, file_name=root + '-' + chunk + ext) - job_uuid = self.with_delay( + delayed_job = self.with_delay( description=description, priority=priority)._import_one_chunk( model_name=model_name, att_id=att_id, options=options) - self._link_attachment_to_job(job_uuid, att_id) + self._link_attachment_to_job(delayed_job, att_id) priority += 1 @api.model From 007ec3157c92060d1dc45b7e4e8838f3977f9de1 Mon Sep 17 00:00:00 2001 From: tarteo Date: Thu, 21 Jun 2018 15:45:34 +0200 Subject: [PATCH 28/77] [MIG] base_import_async and test_base_import_async to 11.0 --- base_import_async/README.rst | 94 ------------------- base_import_async/__init__.py | 24 +---- base_import_async/__manifest__.py | 34 ++----- base_import_async/models/__init__.py | 24 +---- .../models/base_import_import.py | 69 +++++--------- base_import_async/models/queue_job.py | 3 +- base_import_async/readme/CONTRIBUTORS.rst | 12 +++ base_import_async/readme/DESCRIPTION.rst | 3 + base_import_async/readme/ROADMAP.rst | 4 + base_import_async/readme/USAGE.rst | 29 ++++++ base_import_async/static/src/js/import.js | 12 +-- base_import_async/static/src/xml/import.xml | 1 + base_import_async/views/base_import_async.xml | 18 ++-- 13 files changed, 100 insertions(+), 227 deletions(-) delete mode 100644 base_import_async/README.rst create mode 100644 base_import_async/readme/CONTRIBUTORS.rst create mode 100644 base_import_async/readme/DESCRIPTION.rst create mode 100644 base_import_async/readme/ROADMAP.rst create mode 100644 base_import_async/readme/USAGE.rst diff --git a/base_import_async/README.rst b/base_import_async/README.rst deleted file mode 100644 index 26629481f..000000000 --- a/base_import_async/README.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License: AGPL-3 - -Odoo Asynchronous import module -=============================== - -This module extends the standard CSV import functionality -to import files in the background using the OCA/connector -framework. - -Usage -===== - -The user is presented with a new checkbox in the import -screen. When selected, the import is delayed in a background -job. - -This job in turn splits the CSV file in chunks of minimum -100 lines (or more to align with record boundaries). Each -chunk is then imported in a separate background job. - -When an import fails, the job is marked as such and the -user can read the error in the job status. The CSV chunk -being imported is stored as an attachment to the job, making -it easy to download it, fix it and run a new import, possibly -in synchronous mode since the chunks are small. - -Any file that can be imported by the standard import mechanism -can also be imported in the background. - -This module's scope is limited to making standard imports -asynchronous. It does not attempt to transform the data nor -automate ETL flows. - -Other modules may benefit from this infrastructure in the following way -(as illustrated in the test suite): - -1. create an instance of `base_import.import` and populate its fields - (`res_model`, `file`, `file_name`), -2. invoke the `do` method with appropriate options - (`header`, `encoding`, `separator`, `quoting`, - `use_connector`, `chunk_size`). - -Known issues / Roadmap -====================== - -* There is currently no user interface to control the chunk size, - which is currently 100 by default. Should this proves to be an issue, - it is easy to add an option to extend the import screen. -* Validation cannot be run in the background. - - -Bug Tracker -=========== - -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. - - -Credits -======= - -Contributors ------------- - -Sébastien Beau (Akretion) authored the initial prototype. - -Stéphane Bidoul (ACSONE) extended it to version 1.0 to support -multi-line records, store data to import as attachments -and let the user control the asynchronous behaviour. - -Other contributors include: - -* Anthony Muschang (ACSONE) -* David Béal (Akretion) -* Jonathan Nemry (ACSONE) -* Laurent Mignon (ACSONE) - -Maintainer ----------- - -.. image:: http://odoo-community.org/logo.png - :alt: Odoo Community Association - :target: http://odoo-community.org - -This module is maintained by the OCA. - -OCA, or the Odoo Community Association, is a nonprofit organization whose -mission is to support the collaborative development of Odoo features and -promote its widespread use. - -To contribute to this module, please visit http://odoo-community.org. diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index 184a3e03c..31660d6a9 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -1,25 +1,3 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import models diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 758696214..ea0a3ad5f 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -1,31 +1,17 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + { 'name': 'Asynchronous Import', - 'version': '10.0.1.0.0', + 'summary': 'This module extends the standard CSV import ' + 'functionality to import files in the background', + 'version': '11.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', + 'website': 'https://github.com/OCA/queue', 'category': 'Generic Modules', 'depends': [ 'base_import', diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py index 29ab48105..2af21f410 100644 --- a/base_import_async/models/__init__.py +++ b/base_import_async/models/__init__.py @@ -1,26 +1,4 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import base_import_import from . import queue_job diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 063d5e49b..997641c3f 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -1,33 +1,15 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). import csv -import os -from cStringIO import StringIO +from os.path import splitext +from base64 import encodebytes, decodebytes +from io import StringIO -from odoo import api, _ -from odoo.models import TransientModel +from odoo import api, models, _ from odoo.models import fix_import_export_id_paths from odoo.addons.queue_job.job import job, related_action @@ -48,15 +30,7 @@ DEFAULT_CHUNK_SIZE = 100 -def _encode(row, encoding): - return [cell.encode(encoding) for cell in row] - - -def _decode(row, encoding): - return [cell.decode(encoding) for cell in row] - - -class BaseImportImport(TransientModel): +class BaseImportImport(models.TransientModel): _inherit = 'base_import.import' @api.multi @@ -74,7 +48,7 @@ def do(self, fields, options, dryrun=False): except ValueError as e: return [{ 'type': 'error', - 'message': unicode(e), + 'message': str(e), 'record': False, }] @@ -117,13 +91,14 @@ def _create_csv_attachment(self, fields, data, options, file_name): delimiter=str(options.get(OPT_SEPARATOR)), quotechar=str(options.get(OPT_QUOTING))) encoding = options.get(OPT_ENCODING, 'utf-8') - writer.writerow(_encode(fields, encoding)) + writer.writerow(fields) for row in data: - writer.writerow(_encode(row, encoding)) + writer.writerow(row) # create attachment + datas = encodebytes(f.getvalue().encode(encoding)) attachment = self.env['ir.attachment'].create({ 'name': file_name, - 'datas': f.getvalue().encode('base64'), + 'datas': datas, 'datas_fname': file_name }) return attachment.id @@ -131,20 +106,22 @@ def _create_csv_attachment(self, fields, data, options, file_name): @api.model def _read_csv_attachment(self, att_id, options): att = self.env['ir.attachment'].browse(att_id) - f = StringIO(att.datas.decode('base64')) + decoded_datas = decodebytes(att.datas) + encoding = options.get(OPT_ENCODING, 'utf-8') + f = StringIO(decoded_datas.decode(encoding)) reader = csv.reader(f, delimiter=str(options.get(OPT_SEPARATOR)), quotechar=str(options.get(OPT_QUOTING))) - encoding = options.get(OPT_ENCODING, 'utf-8') - fields = _decode(reader.next(), encoding) - data = [_decode(row, encoding) for row in reader] + + fields = next(reader) + data = [row for row in reader] return fields, data @api.model def _extract_records(self, model_obj, fields, data, chunk_size): """ Split the data on record boundaries, in chunks of minimum chunk_size """ - fields = map(fix_import_export_id_paths, fields) + fields = list(map(fix_import_export_id_paths, fields)) row_from = 0 for rows in model_obj._extract_records(fields, data): @@ -180,7 +157,7 @@ def _split_file(self, model_name, translated_model_name, row_from + 1 + header_offset, row_to + 1 + header_offset) # create a CSV attachment and enqueue the job - root, ext = os.path.splitext(file_name) + root, ext = splitext(file_name) att_id = self._create_csv_attachment( fields, data[row_from:row_to + 1], options, file_name=root + '-' + chunk + ext) diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py index b9bcb79cc..f1476c575 100644 --- a/base_import_async/models/queue_job.py +++ b/base_import_async/models/queue_job.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright 2017 ACSONE SA/NV -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from odoo import api, models, _ diff --git a/base_import_async/readme/CONTRIBUTORS.rst b/base_import_async/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000..48e3feaae --- /dev/null +++ b/base_import_async/readme/CONTRIBUTORS.rst @@ -0,0 +1,12 @@ +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments +and let the user control the asynchronous behaviour. + +Other contributors include: + +* Anthony Muschang (ACSONE) +* David Béal (Akretion) +* Jonathan Nemry (ACSONE) +* Laurent Mignon (ACSONE) diff --git a/base_import_async/readme/DESCRIPTION.rst b/base_import_async/readme/DESCRIPTION.rst new file mode 100644 index 000000000..b5996832e --- /dev/null +++ b/base_import_async/readme/DESCRIPTION.rst @@ -0,0 +1,3 @@ +This module extends the standard CSV import functionality +to import files in the background using the OCA/connector +framework. diff --git a/base_import_async/readme/ROADMAP.rst b/base_import_async/readme/ROADMAP.rst new file mode 100644 index 000000000..eccbbb6e4 --- /dev/null +++ b/base_import_async/readme/ROADMAP.rst @@ -0,0 +1,4 @@ +* There is currently no user interface to control the chunk size, + which is currently 100 by default. Should this proves to be an issue, + it is easy to add an option to extend the import screen. +* Validation cannot be run in the background. diff --git a/base_import_async/readme/USAGE.rst b/base_import_async/readme/USAGE.rst new file mode 100644 index 000000000..3618b71b4 --- /dev/null +++ b/base_import_async/readme/USAGE.rst @@ -0,0 +1,29 @@ +The user is presented with a new checkbox in the import +screen. When selected, the import is delayed in a background +job. + +This job in turn splits the CSV file in chunks of minimum +100 lines (or more to align with record boundaries). Each +chunk is then imported in a separate background job. + +When an import fails, the job is marked as such and the +user can read the error in the job status. The CSV chunk +being imported is stored as an attachment to the job, making +it easy to download it, fix it and run a new import, possibly +in synchronous mode since the chunks are small. + +Any file that can be imported by the standard import mechanism +can also be imported in the background. + +This module's scope is limited to making standard imports +asynchronous. It does not attempt to transform the data nor +automate ETL flows. + +Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): + +1. create an instance of `base_import.import` and populate its fields + (`res_model`, `file`, `file_name`), +2. invoke the `do` method with appropriate options + (`header`, `encoding`, `separator`, `quoting`, + `use_connector`, `chunk_size`). diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js index 7ab7d08f7..9cff3b58a 100644 --- a/base_import_async/static/src/js/import.js +++ b/base_import_async/static/src/js/import.js @@ -3,11 +3,9 @@ odoo.define('base_import_async.import', function (require) { var core = require('web.core'); var _t = core._t; - require('base_import.import'); + var DataImport = require('base_import.import').DataImport; - var DataImport = core.action_registry.get('import'); - - DataImport = DataImport.include({ + DataImport.include({ import_options: function () { var options = this._super.apply(this, arguments); @@ -16,9 +14,11 @@ odoo.define('base_import_async.import', function (require) { }, onimported: function () { - var self = this; if (this.$('input.oe_import_queue').prop('checked')) { - this.do_notify(_t("Your request is being processed"), _t("You can check the status of this job in menu 'Queue / Jobs'.")); + this.do_notify( + _t("Your request is being processed"), + _t("You can check the status of this job in menu 'Queue / Jobs'.") + ); } this._super.apply(this, arguments); }, diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml index 22300db51..7819613d5 100644 --- a/base_import_async/static/src/xml/import.xml +++ b/base_import_async/static/src/xml/import.xml @@ -1,3 +1,4 @@ + diff --git a/base_import_async/views/base_import_async.xml b/base_import_async/views/base_import_async.xml index da49cf8c9..56e5645a7 100644 --- a/base_import_async/views/base_import_async.xml +++ b/base_import_async/views/base_import_async.xml @@ -1,10 +1,10 @@ - - - - - + + + + + From 4b241b0bf57e592a033ef70886de4b8c13ecea66 Mon Sep 17 00:00:00 2001 From: tarteo Date: Fri, 22 Jun 2018 13:02:03 +0200 Subject: [PATCH 29/77] [IMP] Processed review --- base_import_async/README.rst | 129 ++++++++++++++++++ base_import_async/__manifest__.py | 3 +- .../models/base_import_import.py | 42 +++--- base_import_async/readme/CONTRIBUTORS.rst | 1 + base_import_async/readme/DESCRIPTION.rst | 2 +- base_import_async/readme/USAGE.rst | 2 +- 6 files changed, 154 insertions(+), 25 deletions(-) create mode 100644 base_import_async/README.rst diff --git a/base_import_async/README.rst b/base_import_async/README.rst new file mode 100644 index 000000000..b192bb2a4 --- /dev/null +++ b/base_import_async/README.rst @@ -0,0 +1,129 @@ +=================== +Asynchronous Import +=================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github + :target: https://github.com/OCA/queue/tree/11.0/base_import_async + :alt: OCA/queue +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/queue-11-0/queue-11-0-base_import_async + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/230/11.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module extends the standard CSV import functionality +to import files in the background using the OCA/queue +framework. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +The user is presented with a new checkbox in the import +screen. When selected, the import is delayed in a background +job. + +This job in turn splits the CSV file in chunks of minimum +100 lines (or more to align with record boundaries). Each +chunk is then imported in a separate background job. + +When an import fails, the job is marked as such and the +user can read the error in the job status. The CSV chunk +being imported is stored as an attachment to the job, making +it easy to download it, fix it and run a new import, possibly +in synchronous mode since the chunks are small. + +Any file that can be imported by the standard import mechanism +can also be imported in the background. + +This module's scope is limited to making standard imports +asynchronous. It does not attempt to transform the data nor +automate ETL flows. + +Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): + +1. create an instance of `base_import.import` and populate its fields + (`res_model`, `file`, `file_name`), +2. invoke the `do` method with appropriate options + (`header`, `encoding`, `separator`, `quoting`, + `use_queue`, `chunk_size`). + +Known issues / Roadmap +====================== + +* There is currently no user interface to control the chunk size, + which is currently 100 by default. Should this proves to be an issue, + it is easy to add an option to extend the import screen. +* Validation cannot be run in the background. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Akretion +* ACSONE SA/NV + +Contributors +~~~~~~~~~~~~ + +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments +and let the user control the asynchronous behaviour. + +Other contributors include: + +* Anthony Muschang (ACSONE) +* David Béal (Akretion) +* Jonathan Nemry (ACSONE) +* Laurent Mignon (ACSONE) +* Dennis Sluijk (Onestein) + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/queue `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index ea0a3ad5f..b94a74e65 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -6,8 +6,7 @@ { 'name': 'Asynchronous Import', - 'summary': 'This module extends the standard CSV import ' - 'functionality to import files in the background', + 'summary': 'Import CSV files in the background', 'version': '11.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 997641c3f..27610da32 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -1,5 +1,5 @@ -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). +# Copyright 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul # @author Sébastien BEAU # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). @@ -62,28 +62,29 @@ def do(self, fields, options, dryrun=False): translated_model_name = self._description description = _("Import %s from file %s") % \ (translated_model_name, self.file_name) - att_id = self._create_csv_attachment( + attachment = self._create_csv_attachment( import_fields, data, options, self.file_name) delayed_job = self.with_delay(description=description)._split_file( model_name=self.res_model, translated_model_name=translated_model_name, - att_id=att_id, + attachment=attachment, options=options, file_name=self.file_name ) - self._link_attachment_to_job(delayed_job, att_id) + self._link_attachment_to_job(delayed_job, attachment) return [] @api.model - def _link_attachment_to_job(self, delayed_job, att_id): + def _link_attachment_to_job(self, delayed_job, attachment): queue_job = self.env['queue.job'].search( [('uuid', '=', delayed_job.uuid)], limit=1) - self.env['ir.attachment'].browse(att_id).write({ + attachment.write({ 'res_model': 'queue.job', 'res_id': queue_job.id, }) @api.model + @api.returns('ir.attachment') def _create_csv_attachment(self, fields, data, options, file_name): # write csv f = StringIO() @@ -101,12 +102,11 @@ def _create_csv_attachment(self, fields, data, options, file_name): 'datas': datas, 'datas_fname': file_name }) - return attachment.id + return attachment @api.model - def _read_csv_attachment(self, att_id, options): - att = self.env['ir.attachment'].browse(att_id) - decoded_datas = decodebytes(att.datas) + def _read_csv_attachment(self, attachment, options): + decoded_datas = decodebytes(attachment.datas) encoding = options.get(OPT_ENCODING, 'utf-8') f = StringIO(decoded_datas.decode(encoding)) reader = csv.reader(f, @@ -117,8 +117,8 @@ def _read_csv_attachment(self, att_id, options): data = [row for row in reader] return fields, data - @api.model - def _extract_records(self, model_obj, fields, data, chunk_size): + @staticmethod + def _extract_chunks(model_obj, fields, data, chunk_size): """ Split the data on record boundaries, in chunks of minimum chunk_size """ fields = list(map(fix_import_export_id_paths, fields)) @@ -136,10 +136,10 @@ def _extract_records(self, model_obj, fields, data, chunk_size): @job @related_action('_related_action_attachment') def _split_file(self, model_name, translated_model_name, - att_id, options, file_name="file.csv"): + attachment, options, file_name="file.csv"): """ Split a CSV attachment in smaller import jobs """ model_obj = self.env[model_name] - fields, data = self._read_csv_attachment(att_id, options) + fields, data = self._read_csv_attachment(attachment, options) padding = len(str(len(data))) priority = options.get(OPT_PRIORITY, INIT_PRIORITY) if options.get(OPT_HAS_HEADER): @@ -147,7 +147,7 @@ def _split_file(self, model_name, translated_model_name, else: header_offset = 0 chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE - for row_from, row_to in self._extract_records( + for row_from, row_to in self._extract_chunks( model_obj, fields, data, chunk_size): chunk = str(priority - INIT_PRIORITY).zfill(padding) description = _("Import %s from file %s - #%s - lines %s to %s") @@ -158,23 +158,23 @@ def _split_file(self, model_name, translated_model_name, row_to + 1 + header_offset) # create a CSV attachment and enqueue the job root, ext = splitext(file_name) - att_id = self._create_csv_attachment( + attachment = self._create_csv_attachment( fields, data[row_from:row_to + 1], options, file_name=root + '-' + chunk + ext) delayed_job = self.with_delay( description=description, priority=priority)._import_one_chunk( model_name=model_name, - att_id=att_id, + attachment=attachment, options=options) - self._link_attachment_to_job(delayed_job, att_id) + self._link_attachment_to_job(delayed_job, attachment) priority += 1 @api.model @job @related_action('_related_action_attachment') - def _import_one_chunk(self, model_name, att_id, options): + def _import_one_chunk(self, model_name, attachment, options): model_obj = self.env[model_name] - fields, data = self._read_csv_attachment(att_id, options) + fields, data = self._read_csv_attachment(attachment, options) result = model_obj.load(fields, data) error_message = [message['message'] for message in result['messages'] if message['type'] == 'error'] diff --git a/base_import_async/readme/CONTRIBUTORS.rst b/base_import_async/readme/CONTRIBUTORS.rst index 48e3feaae..34370afc8 100644 --- a/base_import_async/readme/CONTRIBUTORS.rst +++ b/base_import_async/readme/CONTRIBUTORS.rst @@ -10,3 +10,4 @@ Other contributors include: * David Béal (Akretion) * Jonathan Nemry (ACSONE) * Laurent Mignon (ACSONE) +* Dennis Sluijk (Onestein) diff --git a/base_import_async/readme/DESCRIPTION.rst b/base_import_async/readme/DESCRIPTION.rst index b5996832e..1ac82746c 100644 --- a/base_import_async/readme/DESCRIPTION.rst +++ b/base_import_async/readme/DESCRIPTION.rst @@ -1,3 +1,3 @@ This module extends the standard CSV import functionality -to import files in the background using the OCA/connector +to import files in the background using the OCA/queue framework. diff --git a/base_import_async/readme/USAGE.rst b/base_import_async/readme/USAGE.rst index 3618b71b4..ed7b5d8a3 100644 --- a/base_import_async/readme/USAGE.rst +++ b/base_import_async/readme/USAGE.rst @@ -26,4 +26,4 @@ Other modules may benefit from this infrastructure in the following way (`res_model`, `file`, `file_name`), 2. invoke the `do` method with appropriate options (`header`, `encoding`, `separator`, `quoting`, - `use_connector`, `chunk_size`). + `use_queue`, `chunk_size`). From 6f82da12cb7bf70f0becc2efc83b2cffed26c312 Mon Sep 17 00:00:00 2001 From: tarteo Date: Tue, 26 Jun 2018 15:45:59 +0200 Subject: [PATCH 30/77] [IMP] Readability, speed, and added HISTORY.rst --- base_import_async/README.rst | 8 + .../models/base_import_import.py | 10 +- base_import_async/readme/HISTORY.rst | 4 + .../static/description/index.html | 483 ++++++++++++++++++ 4 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 base_import_async/readme/HISTORY.rst create mode 100644 base_import_async/static/description/index.html diff --git a/base_import_async/README.rst b/base_import_async/README.rst index b192bb2a4..466173b9c 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -75,6 +75,14 @@ Known issues / Roadmap it is easy to add an option to extend the import screen. * Validation cannot be run in the background. +Changelog +========= + +11.0.1.0.0 (2018-06-26) +~~~~~~~~~~~~~~~~~~~~~~~ + +* [BREAKING] In the `do` method the `use_connector` option has changed to `use_queue`. + Bug Tracker =========== diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 27610da32..850ea6fba 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -5,9 +5,9 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). import csv +import base64 from os.path import splitext -from base64 import encodebytes, decodebytes -from io import StringIO +from io import StringIO, TextIOWrapper, BytesIO from odoo import api, models, _ from odoo.models import fix_import_export_id_paths @@ -96,7 +96,7 @@ def _create_csv_attachment(self, fields, data, options, file_name): for row in data: writer.writerow(row) # create attachment - datas = encodebytes(f.getvalue().encode(encoding)) + datas = base64.encodebytes(f.getvalue().encode(encoding)) attachment = self.env['ir.attachment'].create({ 'name': file_name, 'datas': datas, @@ -106,9 +106,9 @@ def _create_csv_attachment(self, fields, data, options, file_name): @api.model def _read_csv_attachment(self, attachment, options): - decoded_datas = decodebytes(attachment.datas) + decoded_datas = base64.decodebytes(attachment.datas) encoding = options.get(OPT_ENCODING, 'utf-8') - f = StringIO(decoded_datas.decode(encoding)) + f = TextIOWrapper(BytesIO(decoded_datas), encoding=encoding) reader = csv.reader(f, delimiter=str(options.get(OPT_SEPARATOR)), quotechar=str(options.get(OPT_QUOTING))) diff --git a/base_import_async/readme/HISTORY.rst b/base_import_async/readme/HISTORY.rst new file mode 100644 index 000000000..630d9ebaa --- /dev/null +++ b/base_import_async/readme/HISTORY.rst @@ -0,0 +1,4 @@ +11.0.1.0.0 (2018-06-26) +~~~~~~~~~~~~~~~~~~~~~~~ + +* [BREAKING] In the `do` method the `use_connector` option has changed to `use_queue`. diff --git a/base_import_async/static/description/index.html b/base_import_async/static/description/index.html new file mode 100644 index 000000000..2ce5026e7 --- /dev/null +++ b/base_import_async/static/description/index.html @@ -0,0 +1,483 @@ + + + + + + +Asynchronous Import + + + +
+

Asynchronous Import

+ + +

Beta License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runbot

+

This module extends the standard CSV import functionality +to import files in the background using the OCA/queue +framework.

+

Table of contents

+ +
+

Usage

+

The user is presented with a new checkbox in the import +screen. When selected, the import is delayed in a background +job.

+

This job in turn splits the CSV file in chunks of minimum +100 lines (or more to align with record boundaries). Each +chunk is then imported in a separate background job.

+

When an import fails, the job is marked as such and the +user can read the error in the job status. The CSV chunk +being imported is stored as an attachment to the job, making +it easy to download it, fix it and run a new import, possibly +in synchronous mode since the chunks are small.

+

Any file that can be imported by the standard import mechanism +can also be imported in the background.

+

This module’s scope is limited to making standard imports +asynchronous. It does not attempt to transform the data nor +automate ETL flows.

+

Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite):

+
    +
  1. create an instance of base_import.import and populate its fields +(res_model, file, file_name),
    2. +
  2. invoke the do method with appropriate options +(header, encoding, separator, quoting, +use_queue, chunk_size).
    3. +
+
+
+

Known issues / Roadmap

+
    +
  • There is currently no user interface to control the chunk size, +which is currently 100 by default. Should this proves to be an issue, +it is easy to add an option to extend the import screen.
    • +
  • Validation cannot be run in the background.
    • +
+
+
+

Changelog

+
+

11.0.1.0.0 (2018-06-26)

+
    +
  • [BREAKING] In the do method the use_connector option has changed to use_queue.
    • +
+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+

Sébastien Beau (Akretion) authored the initial prototype.

+

Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments +and let the user control the asynchronous behaviour.

+

Other contributors include:

+
    +
  • Anthony Muschang (ACSONE)
    • +
  • David Béal (Akretion)
    • +
  • Jonathan Nemry (ACSONE)
    • +
  • Laurent Mignon (ACSONE)
    • +
  • Dennis Sluijk (Onestein)
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/queue project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + From f601115676c4ee4b5aefd5bf4f4c6cf4a2f4fc5d Mon Sep 17 00:00:00 2001 From: OCA git bot Date: Thu, 27 Sep 2018 02:00:50 +0200 Subject: [PATCH 31/77] [MIG] Make modules uninstallable --- base_import_async/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index b94a74e65..735f5b09a 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -22,5 +22,5 @@ 'qweb': [ 'static/src/xml/import.xml', ], - 'installable': True, + 'installable': False, } From 3aeb04f6be2f013c121603bef5f3f68054f99180 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Tue, 1 Oct 2019 18:32:32 +0200 Subject: [PATCH 32/77] Set modules as uninstallable --- base_import_async/__manifest__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 735f5b09a..142b291d8 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -1,5 +1,3 @@ -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). # @author Stéphane Bidoul # @author Sébastien BEAU # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). From 01b4f4a2e540eabc0a997e5c68c29d80f5c9a58b Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Fri, 4 Oct 2019 10:47:58 +0200 Subject: [PATCH 33/77] Run pre-commit with black, isort, ... --- base_import_async/__manifest__.py | 29 ++-- .../models/base_import_import.py | 154 ++++++++++-------- base_import_async/models/queue_job.py | 19 ++- base_import_async/static/src/xml/import.xml | 4 +- 4 files changed, 105 insertions(+), 101 deletions(-) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 142b291d8..11cb0d083 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -3,22 +3,15 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { - 'name': 'Asynchronous Import', - 'summary': 'Import CSV files in the background', - 'version': '11.0.1.0.0', - 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'website': 'https://github.com/OCA/queue', - 'category': 'Generic Modules', - 'depends': [ - 'base_import', - 'queue_job', - ], - 'data': [ - 'views/base_import_async.xml', - ], - 'qweb': [ - 'static/src/xml/import.xml', - ], - 'installable': False, + "name": "Asynchronous Import", + "summary": "Import CSV files in the background", + "version": "11.0.1.0.0", + "author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)", + "license": "AGPL-3", + "website": "https://github.com/OCA/queue", + "category": "Generic Modules", + "depends": ["base_import", "queue_job"], + "data": ["views/base_import_async.xml"], + "qweb": ["static/src/xml/import.xml"], + "installable": False, } diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 850ea6fba..11e6a05a3 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -4,41 +4,39 @@ # @author Sébastien BEAU # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -import csv import base64 +import csv +from io import BytesIO, StringIO, TextIOWrapper from os.path import splitext -from io import StringIO, TextIOWrapper, BytesIO - -from odoo import api, models, _ -from odoo.models import fix_import_export_id_paths -from odoo.addons.queue_job.job import job, related_action +from odoo import _, api, models from odoo.addons.queue_job.exception import FailedJobError +from odoo.addons.queue_job.job import job, related_action +from odoo.models import fix_import_export_id_paths # options defined in base_import/import.js -OPT_HAS_HEADER = 'headers' -OPT_SEPARATOR = 'separator' -OPT_QUOTING = 'quoting' -OPT_ENCODING = 'encoding' +OPT_HAS_HEADER = "headers" +OPT_SEPARATOR = "separator" +OPT_QUOTING = "quoting" +OPT_ENCODING = "encoding" # options defined in base_import_async/import.js -OPT_USE_QUEUE = 'use_queue' -OPT_CHUNK_SIZE = 'chunk_size' +OPT_USE_QUEUE = "use_queue" +OPT_CHUNK_SIZE = "chunk_size" # option not available in UI, but usable from scripts -OPT_PRIORITY = 'priority' +OPT_PRIORITY = "priority" INIT_PRIORITY = 100 DEFAULT_CHUNK_SIZE = 100 class BaseImportImport(models.TransientModel): - _inherit = 'base_import.import' + _inherit = "base_import.import" @api.multi def do(self, fields, options, dryrun=False): if dryrun or not options.get(OPT_USE_QUEUE): # normal import - return super(BaseImportImport, self).do( - fields, options, dryrun=dryrun) + return super(BaseImportImport, self).do(fields, options, dryrun=dryrun) # asynchronous import try: @@ -46,72 +44,70 @@ def do(self, fields, options, dryrun=False): # Parse date and float field data = self._parse_import_data(data, import_fields, options) except ValueError as e: - return [{ - 'type': 'error', - 'message': str(e), - 'record': False, - }] + return [{"type": "error", "message": str(e), "record": False}] # get the translated model name to build # a meaningful job description - search_result = self.env['ir.model'].name_search( - self.res_model, operator='=') + search_result = self.env["ir.model"].name_search(self.res_model, operator="=") if search_result: translated_model_name = search_result[0][1] else: translated_model_name = self._description - description = _("Import %s from file %s") % \ - (translated_model_name, self.file_name) + description = _("Import %s from file %s") % ( + translated_model_name, + self.file_name, + ) attachment = self._create_csv_attachment( - import_fields, data, options, self.file_name) + import_fields, data, options, self.file_name + ) delayed_job = self.with_delay(description=description)._split_file( model_name=self.res_model, translated_model_name=translated_model_name, attachment=attachment, options=options, - file_name=self.file_name + file_name=self.file_name, ) self._link_attachment_to_job(delayed_job, attachment) return [] @api.model def _link_attachment_to_job(self, delayed_job, attachment): - queue_job = self.env['queue.job'].search( - [('uuid', '=', delayed_job.uuid)], limit=1) - attachment.write({ - 'res_model': 'queue.job', - 'res_id': queue_job.id, - }) + queue_job = self.env["queue.job"].search( + [("uuid", "=", delayed_job.uuid)], limit=1 + ) + attachment.write({"res_model": "queue.job", "res_id": queue_job.id}) @api.model - @api.returns('ir.attachment') + @api.returns("ir.attachment") def _create_csv_attachment(self, fields, data, options, file_name): # write csv f = StringIO() - writer = csv.writer(f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING))) - encoding = options.get(OPT_ENCODING, 'utf-8') + writer = csv.writer( + f, + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING)), + ) + encoding = options.get(OPT_ENCODING, "utf-8") writer.writerow(fields) for row in data: writer.writerow(row) # create attachment datas = base64.encodebytes(f.getvalue().encode(encoding)) - attachment = self.env['ir.attachment'].create({ - 'name': file_name, - 'datas': datas, - 'datas_fname': file_name - }) + attachment = self.env["ir.attachment"].create( + {"name": file_name, "datas": datas, "datas_fname": file_name} + ) return attachment @api.model def _read_csv_attachment(self, attachment, options): decoded_datas = base64.decodebytes(attachment.datas) - encoding = options.get(OPT_ENCODING, 'utf-8') + encoding = options.get(OPT_ENCODING, "utf-8") f = TextIOWrapper(BytesIO(decoded_datas), encoding=encoding) - reader = csv.reader(f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING))) + reader = csv.reader( + f, + delimiter=str(options.get(OPT_SEPARATOR)), + quotechar=str(options.get(OPT_QUOTING)), + ) fields = next(reader) data = [row for row in reader] @@ -123,20 +119,25 @@ def _extract_chunks(model_obj, fields, data, chunk_size): in chunks of minimum chunk_size """ fields = list(map(fix_import_export_id_paths, fields)) row_from = 0 - for rows in model_obj._extract_records(fields, - data): - rows = rows[1]['rows'] - if rows['to'] - row_from + 1 >= chunk_size: - yield row_from, rows['to'] - row_from = rows['to'] + 1 + for rows in model_obj._extract_records(fields, data): + rows = rows[1]["rows"] + if rows["to"] - row_from + 1 >= chunk_size: + yield row_from, rows["to"] + row_from = rows["to"] + 1 if row_from < len(data): yield row_from, len(data) - 1 @api.model @job - @related_action('_related_action_attachment') - def _split_file(self, model_name, translated_model_name, - attachment, options, file_name="file.csv"): + @related_action("_related_action_attachment") + def _split_file( + self, + model_name, + translated_model_name, + attachment, + options, + file_name="file.csv", + ): """ Split a CSV attachment in smaller import jobs """ model_obj = self.env[model_name] fields, data = self._read_csv_attachment(attachment, options) @@ -148,36 +149,45 @@ def _split_file(self, model_name, translated_model_name, header_offset = 0 chunk_size = options.get(OPT_CHUNK_SIZE) or DEFAULT_CHUNK_SIZE for row_from, row_to in self._extract_chunks( - model_obj, fields, data, chunk_size): + model_obj, fields, data, chunk_size + ): chunk = str(priority - INIT_PRIORITY).zfill(padding) description = _("Import %s from file %s - #%s - lines %s to %s") - description = description % (translated_model_name, - file_name, - chunk, - row_from + 1 + header_offset, - row_to + 1 + header_offset) + description = description % ( + translated_model_name, + file_name, + chunk, + row_from + 1 + header_offset, + row_to + 1 + header_offset, + ) # create a CSV attachment and enqueue the job root, ext = splitext(file_name) attachment = self._create_csv_attachment( - fields, data[row_from:row_to + 1], options, - file_name=root + '-' + chunk + ext) + fields, + data[row_from : row_to + 1], + options, + file_name=root + "-" + chunk + ext, + ) delayed_job = self.with_delay( - description=description, priority=priority)._import_one_chunk( - model_name=model_name, - attachment=attachment, - options=options) + description=description, priority=priority + )._import_one_chunk( + model_name=model_name, attachment=attachment, options=options + ) self._link_attachment_to_job(delayed_job, attachment) priority += 1 @api.model @job - @related_action('_related_action_attachment') + @related_action("_related_action_attachment") def _import_one_chunk(self, model_name, attachment, options): model_obj = self.env[model_name] fields, data = self._read_csv_attachment(attachment, options) result = model_obj.load(fields, data) - error_message = [message['message'] for message in result['messages'] - if message['type'] == 'error'] + error_message = [ + message["message"] + for message in result["messages"] + if message["type"] == "error" + ] if error_message: - raise FailedJobError('\n'.join(error_message)) + raise FailedJobError("\n".join(error_message)) return result diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py index f1476c575..9988a3251 100644 --- a/base_import_async/models/queue_job.py +++ b/base_import_async/models/queue_job.py @@ -1,22 +1,23 @@ # Copyright 2017 ACSONE SA/NV # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from odoo import api, models, _ +from odoo import _, api, models class QueueJob(models.Model): """ Job status and result """ - _inherit = 'queue.job' + + _inherit = "queue.job" @api.multi def _related_action_attachment(self): - res_id = self.kwargs.get('att_id') + res_id = self.kwargs.get("att_id") action = { - 'name': _("Attachment"), - 'type': 'ir.actions.act_window', - 'res_model': "ir.attachment", - 'view_type': 'form', - 'view_mode': 'form', - 'res_id': res_id, + "name": _("Attachment"), + "type": "ir.actions.act_window", + "res_model": "ir.attachment", + "view_type": "form", + "view_mode": "form", + "res_id": res_id, } return action diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml index 7819613d5..fc122dff0 100644 --- a/base_import_async/static/src/xml/import.xml +++ b/base_import_async/static/src/xml/import.xml @@ -2,8 +2,8 @@ -
From 9d9bd0126d53a3639cbc3dd7bada398ffcfb6322 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Tue, 2 Oct 2018 12:08:12 +0200 Subject: [PATCH 34/77] Migrate base_import_async to 12.0 --- base_import_async/__manifest__.py | 29 ++++++++++++------- .../models/base_import_import.py | 5 ++-- base_import_async/readme/HISTORY.rst | 4 +-- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 11cb0d083..f044245bf 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -3,15 +3,22 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { - "name": "Asynchronous Import", - "summary": "Import CSV files in the background", - "version": "11.0.1.0.0", - "author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)", - "license": "AGPL-3", - "website": "https://github.com/OCA/queue", - "category": "Generic Modules", - "depends": ["base_import", "queue_job"], - "data": ["views/base_import_async.xml"], - "qweb": ["static/src/xml/import.xml"], - "installable": False, + 'name': 'Asynchronous Import', + 'summary': 'Import CSV files in the background', + 'version': '12.0.1.0.0', + 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'website': 'https://github.com/OCA/queue', + 'category': 'Generic Modules', + 'depends': [ + 'base_import', + 'queue_job', + ], + 'data': [ + 'views/base_import_async.xml', + ], + 'qweb': [ + 'static/src/xml/import.xml', + ], + 'installable': True, } diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 11e6a05a3..3b2f3c474 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -33,10 +33,11 @@ class BaseImportImport(models.TransientModel): _inherit = "base_import.import" @api.multi - def do(self, fields, options, dryrun=False): + def do(self, fields, columns, options, dryrun=False): if dryrun or not options.get(OPT_USE_QUEUE): # normal import - return super(BaseImportImport, self).do(fields, options, dryrun=dryrun) + return super(BaseImportImport, self).do( + fields, columns, options, dryrun=dryrun) # asynchronous import try: diff --git a/base_import_async/readme/HISTORY.rst b/base_import_async/readme/HISTORY.rst index 630d9ebaa..6ef5491de 100644 --- a/base_import_async/readme/HISTORY.rst +++ b/base_import_async/readme/HISTORY.rst @@ -1,4 +1,4 @@ -11.0.1.0.0 (2018-06-26) +12.0.1.0.0 (2018-10-20) ~~~~~~~~~~~~~~~~~~~~~~~ -* [BREAKING] In the `do` method the `use_connector` option has changed to `use_queue`. +* [MIGRATION] from 11.0 branched at rev. b0945be From f195df348de5d31cf030b4688a86b9ac2cf1c6ed Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Fri, 5 Oct 2018 08:35:51 +0200 Subject: [PATCH 35/77] Add OCA development status --- base_import_async/__manifest__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index f044245bf..9a234da01 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -21,4 +21,5 @@ 'static/src/xml/import.xml', ], 'installable': True, + 'development_status': 'Stable', } From f860ba1ad8a73293ce518923afec840ccd7b119e Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Fri, 15 Nov 2019 16:42:59 +0100 Subject: [PATCH 36/77] Exit the import screen after importing file in async When calling super, the super method would fail with a 'state machine' error, appearing confusingly as a CORS error client side. We do not have to call super when we delayed import asynchronously, as anyway the 'super' call would send an inaccurate confirmation notification and exit. --- base_import_async/static/src/js/import.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js index 9cff3b58a..3fcab6fc9 100644 --- a/base_import_async/static/src/js/import.js +++ b/base_import_async/static/src/js/import.js @@ -19,8 +19,10 @@ odoo.define('base_import_async.import', function (require) { _t("Your request is being processed"), _t("You can check the status of this job in menu 'Queue / Jobs'.") ); + this.exit(); + } else { + this._super.apply(this, arguments); } - this._super.apply(this, arguments); }, }); From 14b03407859bfb94dd5c79bf988d1dd09435dc15 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Sun, 17 Nov 2019 17:57:38 +0100 Subject: [PATCH 37/77] Set default values for encoding and separator Import of Excel files do not set these options, although the keys are in the dict. Courtesy of @GSLabIt for finding the issue and solution. --- base_import_async/README.rst | 27 ++++--- base_import_async/i18n/base_import_async.pot | 71 +++++++++++++++++++ .../models/base_import_import.py | 20 +++--- .../static/description/index.html | 14 ++-- 4 files changed, 98 insertions(+), 34 deletions(-) create mode 100644 base_import_async/i18n/base_import_async.pot diff --git a/base_import_async/README.rst b/base_import_async/README.rst index 466173b9c..d23b790df 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -7,23 +7,20 @@ Asynchronous Import !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png - :target: https://odoo-community.org/page/development-status - :alt: Beta -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png +.. |badge1| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 -.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github - :target: https://github.com/OCA/queue/tree/11.0/base_import_async +.. |badge2| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github + :target: https://github.com/OCA/queue/tree/12.0/base_import_async :alt: OCA/queue -.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/queue-11-0/queue-11-0-base_import_async +.. |badge3| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/queue-12-0/queue-12-0-base_import_async :alt: Translate me on Weblate -.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/230/11.0 +.. |badge4| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/230/12.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| This module extends the standard CSV import functionality to import files in the background using the OCA/queue @@ -78,10 +75,10 @@ Known issues / Roadmap Changelog ========= -11.0.1.0.0 (2018-06-26) +12.0.1.0.0 (2018-10-20) ~~~~~~~~~~~~~~~~~~~~~~~ -* [BREAKING] In the `do` method the `use_connector` option has changed to `use_queue`. +* [MIGRATION] from 11.0 branched at rev. b0945be Bug Tracker =========== @@ -89,7 +86,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -132,6 +129,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/queue `_ project on GitHub. +This module is part of the `OCA/queue `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_async/i18n/base_import_async.pot b/base_import_async/i18n/base_import_async.pot new file mode 100644 index 000000000..8abe94322 --- /dev/null +++ b/base_import_async/i18n/base_import_async.pot @@ -0,0 +1,71 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_async +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 12.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: base_import_async +#: code:addons/base_import_async/models/queue_job.py:15 +#, python-format +msgid "Attachment" +msgstr "" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_base_import_import +msgid "Base Import" +msgstr "" + +#. module: base_import_async +#: code:addons/base_import_async/models/base_import_import.py:63 +#, python-format +msgid "Import %s from file %s" +msgstr "" + +#. module: base_import_async +#: code:addons/base_import_async/models/base_import_import.py:153 +#, python-format +msgid "Import %s from file %s - #%s - lines %s to %s" +msgstr "" + +#. module: base_import_async +#. openerp-web +#: code:addons/base_import_async/static/src/xml/import.xml:10 +#, python-format +msgid "Import in the background" +msgstr "" + +#. module: base_import_async +#: model:ir.model,name:base_import_async.model_queue_job +msgid "Queue Job" +msgstr "" + +#. module: base_import_async +#. openerp-web +#: code:addons/base_import_async/static/src/xml/import.xml:7 +#, python-format +msgid "When checked, the import will be executed as a background job, after splitting your file in small chunks that will be processed independently. Use this to import very large files." +msgstr "" + +#. module: base_import_async +#. openerp-web +#: code:addons/base_import_async/static/src/js/import.js:20 +#, python-format +msgid "You can check the status of this job in menu 'Queue / Jobs'." +msgstr "" + +#. module: base_import_async +#. openerp-web +#: code:addons/base_import_async/static/src/js/import.js:19 +#, python-format +msgid "Your request is being processed" +msgstr "" + diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 3b2f3c474..40d31e97c 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -83,12 +83,10 @@ def _link_attachment_to_job(self, delayed_job, attachment): def _create_csv_attachment(self, fields, data, options, file_name): # write csv f = StringIO() - writer = csv.writer( - f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING)), - ) - encoding = options.get(OPT_ENCODING, "utf-8") + writer = csv.writer(f, + delimiter=str(options.get(OPT_SEPARATOR)) or ',', + quotechar=str(options.get(OPT_QUOTING))) + encoding = options.get(OPT_ENCODING) or 'utf-8' writer.writerow(fields) for row in data: writer.writerow(row) @@ -102,13 +100,11 @@ def _create_csv_attachment(self, fields, data, options, file_name): @api.model def _read_csv_attachment(self, attachment, options): decoded_datas = base64.decodebytes(attachment.datas) - encoding = options.get(OPT_ENCODING, "utf-8") + encoding = options.get(OPT_ENCODING) or 'utf-8' f = TextIOWrapper(BytesIO(decoded_datas), encoding=encoding) - reader = csv.reader( - f, - delimiter=str(options.get(OPT_SEPARATOR)), - quotechar=str(options.get(OPT_QUOTING)), - ) + reader = csv.reader(f, + delimiter=str(options.get(OPT_SEPARATOR)) or ',', + quotechar=str(options.get(OPT_QUOTING))) fields = next(reader) data = [row for row in reader] diff --git a/base_import_async/static/description/index.html b/base_import_async/static/description/index.html index 2ce5026e7..2dbd718e6 100644 --- a/base_import_async/static/description/index.html +++ b/base_import_async/static/description/index.html @@ -3,7 +3,7 @@ - + Asynchronous Import