Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
exclude: |
(?x)
# NOT INSTALLABLE ADDONS
^base_import_async/|
^queue_job_batch/|
^queue_job_cron/|
^queue_job_cron_jobrunner/|
Expand Down
16 changes: 10 additions & 6 deletions base_import_async/README.rst
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
.. image:: https://odoo-community.org/readme-banner-image
:target: https://odoo-community.org/get-involved?utm_source=readme
:alt: Odoo Community Association

===================
Asynchronous Import
===================
Expand All @@ -13,17 +17,17 @@ Asynchronous Import
.. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png
:target: https://odoo-community.org/page/development-status
:alt: Production/Stable
.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
:alt: License: AGPL-3
.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github
:target: https://github.com/OCA/queue/tree/18.0/base_import_async
:target: https://github.com/OCA/queue/tree/19.0/base_import_async
:alt: OCA/queue
.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
:target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async
:target: https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-base_import_async
:alt: Translate me on Weblate
.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=19.0
:alt: Try me on Runboat

|badge1| |badge2| |badge3| |badge4| |badge5|
Expand Down Expand Up @@ -87,7 +91,7 @@ Bug Tracker
Bugs are tracked on `GitHub Issues <https://github.com/OCA/queue/issues>`_.
In case of trouble, please check there if your issue has already been reported.
If you spotted it first, help us to smash it by providing a detailed and welcomed
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2019.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.

Do not contact contributors directly about support or help with technical issues.

Expand Down Expand Up @@ -149,6 +153,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose
mission is to support the collaborative development of Odoo features and
promote its widespread use.

This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/18.0/base_import_async>`_ project on GitHub.
This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/19.0/base_import_async>`_ project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.
1 change: 1 addition & 0 deletions base_import_async/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from . import models
from . import wizard
4 changes: 2 additions & 2 deletions base_import_async/__manifest__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
{
"name": "Asynchronous Import",
"summary": "Import CSV files in the background",
"version": "18.0.1.0.0",
"version": "19.0.1.0.0",
"author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)",
"license": "AGPL-3",
"website": "https://github.com/OCA/queue",
Expand All @@ -20,6 +20,6 @@
"base_import_async/static/src/xml/import_data_sidepanel.xml",
],
},
"installable": False,
"installable": True,
"development_status": "Production/Stable",
}
1 change: 0 additions & 1 deletion base_import_async/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from . import base_import_import
from . import queue_job
4 changes: 2 additions & 2 deletions base_import_async/models/queue_job.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright 2017 ACSONE SA/NV
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from odoo import _, models
from odoo import models


class QueueJob(models.Model):
Expand All @@ -11,7 +11,7 @@ class QueueJob(models.Model):

def _related_action_attachment(self):
return {
"name": _("Attachment"),
"name": self.env._("Attachment"),
"type": "ir.actions.act_window",
"res_model": "ir.attachment",
"view_mode": "form",
Expand Down
38 changes: 22 additions & 16 deletions base_import_async/static/description/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="Docutils: https://docutils.sourceforge.io/" />
<title>Asynchronous Import</title>
<title>README.rst</title>
<style type="text/css">

/*
Expand Down Expand Up @@ -360,16 +360,21 @@
</style>
</head>
<body>
<div class="document" id="asynchronous-import">
<h1 class="title">Asynchronous Import</h1>
<div class="document">


<a class="reference external image-reference" href="https://odoo-community.org/get-involved?utm_source=readme">
<img alt="Odoo Community Association" src="https://odoo-community.org/readme-banner-image" />
</a>
<div class="section" id="asynchronous-import">
<h1>Asynchronous Import</h1>
<!-- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!! This file is generated by oca-gen-addon-readme !!
!! changes will be overwritten. !!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!! source digest: sha256:299c64b165f6348dfa062b8925a95235390daa6063964f21a5a1d0757cdf1614
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->
<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Production/Stable" src="https://img.shields.io/badge/maturity-Production%2FStable-green.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/queue/tree/18.0/base_import_async"><img alt="OCA/queue" src="https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github" /></a> <a class="reference external image-reference" href="https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external image-reference" href="https://runboat.odoo-community.org/builds?repo=OCA/queue&amp;target_branch=18.0"><img alt="Try me on Runboat" src="https://img.shields.io/badge/runboat-Try%20me-875A7B.png" /></a></p>
<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Production/Stable" src="https://img.shields.io/badge/maturity-Production%2FStable-green.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/license-AGPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/queue/tree/19.0/base_import_async"><img alt="OCA/queue" src="https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github" /></a> <a class="reference external image-reference" href="https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-base_import_async"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external image-reference" href="https://runboat.odoo-community.org/builds?repo=OCA/queue&amp;target_branch=19.0"><img alt="Try me on Runboat" src="https://img.shields.io/badge/runboat-Try%20me-875A7B.png" /></a></p>
<p>This module extends the standard CSV import functionality to import
files in the background using the OCA/queue framework.</p>
<p><strong>Table of contents</strong></p>
Expand All @@ -392,7 +397,7 @@ <h1 class="title">Asynchronous Import</h1>
</ul>
</div>
<div class="section" id="usage">
<h1><a class="toc-backref" href="#toc-entry-1">Usage</a></h1>
<h2><a class="toc-backref" href="#toc-entry-1">Usage</a></h2>
<p>The user is presented with a new checkbox in the import screen. When
selected, the import is delayed in a background job.</p>
<p>This job in turn splits the CSV file in chunks of minimum 100 lines (or
Expand All @@ -416,7 +421,7 @@ <h1><a class="toc-backref" href="#toc-entry-1">Usage</a></h1>
</ol>
</div>
<div class="section" id="known-issues-roadmap">
<h1><a class="toc-backref" href="#toc-entry-2">Known issues / Roadmap</a></h1>
<h2><a class="toc-backref" href="#toc-entry-2">Known issues / Roadmap</a></h2>
<ul class="simple">
<li>There is currently no user interface to control the chunk size, which
is currently 100 by default. Should this proves to be an issue, it is
Expand All @@ -425,33 +430,33 @@ <h1><a class="toc-backref" href="#toc-entry-2">Known issues / Roadmap</a></h1>
</ul>
</div>
<div class="section" id="changelog">
<h1><a class="toc-backref" href="#toc-entry-3">Changelog</a></h1>
<h2><a class="toc-backref" href="#toc-entry-3">Changelog</a></h2>
<div class="section" id="section-1">
<h2><a class="toc-backref" href="#toc-entry-4">13.0.1.0.0 (2019-12-20)</a></h2>
<h3><a class="toc-backref" href="#toc-entry-4">13.0.1.0.0 (2019-12-20)</a></h3>
<ul class="simple">
<li>[MIGRATION] from 12.0 branched at rev. a7f8031</li>
</ul>
</div>
</div>
<div class="section" id="bug-tracker">
<h1><a class="toc-backref" href="#toc-entry-5">Bug Tracker</a></h1>
<h2><a class="toc-backref" href="#toc-entry-5">Bug Tracker</a></h2>
<p>Bugs are tracked on <a class="reference external" href="https://github.com/OCA/queue/issues">GitHub Issues</a>.
In case of trouble, please check there if your issue has already been reported.
If you spotted it first, help us to smash it by providing a detailed and welcomed
<a class="reference external" href="https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p>
<a class="reference external" href="https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2019.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p>
<p>Do not contact contributors directly about support or help with technical issues.</p>
</div>
<div class="section" id="credits">
<h1><a class="toc-backref" href="#toc-entry-6">Credits</a></h1>
<h2><a class="toc-backref" href="#toc-entry-6">Credits</a></h2>
<div class="section" id="authors">
<h2><a class="toc-backref" href="#toc-entry-7">Authors</a></h2>
<h3><a class="toc-backref" href="#toc-entry-7">Authors</a></h3>
<ul class="simple">
<li>Akretion</li>
<li>ACSONE SA/NV</li>
</ul>
</div>
<div class="section" id="contributors">
<h2><a class="toc-backref" href="#toc-entry-8">Contributors</a></h2>
<h3><a class="toc-backref" href="#toc-entry-8">Contributors</a></h3>
<p>Sébastien Beau (Akretion) authored the initial prototype.</p>
<p>Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
multi-line records, store data to import as attachments and let the user
Expand All @@ -473,23 +478,24 @@ <h2><a class="toc-backref" href="#toc-entry-8">Contributors</a></h2>
</ul>
</div>
<div class="section" id="other-credits">
<h2><a class="toc-backref" href="#toc-entry-9">Other credits</a></h2>
<h3><a class="toc-backref" href="#toc-entry-9">Other credits</a></h3>
<p>The migration of this module from 17.0 to 18.0 was financially supported
by Camptocamp</p>
</div>
<div class="section" id="maintainers">
<h2><a class="toc-backref" href="#toc-entry-10">Maintainers</a></h2>
<h3><a class="toc-backref" href="#toc-entry-10">Maintainers</a></h3>
<p>This module is maintained by the OCA.</p>
<a class="reference external image-reference" href="https://odoo-community.org">
<img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" />
</a>
<p>OCA, or the Odoo Community Association, is a nonprofit organization whose
mission is to support the collaborative development of Odoo features and
promote its widespread use.</p>
<p>This module is part of the <a class="reference external" href="https://github.com/OCA/queue/tree/18.0/base_import_async">OCA/queue</a> project on GitHub.</p>
<p>This module is part of the <a class="reference external" href="https://github.com/OCA/queue/tree/19.0/base_import_async">OCA/queue</a> project on GitHub.</p>
<p>You are welcome to contribute. To learn how please visit <a class="reference external" href="https://odoo-community.org/page/Contribute">https://odoo-community.org/page/Contribute</a>.</p>
</div>
</div>
</div>
</div>
</body>
</html>
143 changes: 142 additions & 1 deletion base_import_async/tests/test_base_import_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@

from odoo.tests.common import RecordCapturer, TransactionCase

from ..models.base_import_import import OPT_USE_QUEUE
from odoo.addons.queue_job.exception import FailedJobError

from ..wizard.base_import_import import (
INIT_PRIORITY,
OPT_CHUNK_SIZE,
OPT_HAS_HEADER,
OPT_QUOTING,
OPT_SEPARATOR,
OPT_USE_QUEUE,
)


class TestBaseImportImport(TransactionCase):
Expand Down Expand Up @@ -59,6 +68,54 @@ def test_normal_import_res_partners(self):
self.assertEqual(len(records_created), 2)
self.assertIn("partner1", records_created[0].email)

def test_execute_import_res_partners_async(self):
values = [
[
"name",
"email",
"is_company",
],
[
"partner 1",
"[email protected]",
"1",
],
[
"partner 2",
"[email protected]",
"0",
],
]
import_vals = {
"res_model": self.res_partners._name,
"file": "\n".join([";".join(values) for values in values]),
"file_type": "text/csv",
"file_name": "partners.csv",
}
import_wizard = self.import_wizard.create(import_vals)
opts = {
OPT_QUOTING: '"',
OPT_SEPARATOR: ";",
OPT_HAS_HEADER: True,
OPT_USE_QUEUE: True,
}
preview = import_wizard.parse_preview(opts)
with RecordCapturer(self.env["queue.job"], []) as capture:
result = import_wizard.execute_import(
[field[0] for field in preview["matches"].values()],
[],
opts,
)
self.assertEqual(result, [])
self.assertEqual(len(capture.records), 1)
job = capture.records[0]
self.assertEqual(job.method_name, "_split_file")
self.assertEqual(job.model_name, "base_import.import")
attachment = self.env["ir.attachment"].search(
[("res_model", "=", "queue.job"), ("res_id", "=", job.id)], limit=1
)
self.assertTrue(attachment)

def test_wrong_import_res_partners(self):
values = [
[
Expand Down Expand Up @@ -97,3 +154,87 @@ def test_wrong_import_res_partners(self):
opts,
)
self.assertTrue(any(msg["type"] == "error" for msg in results["messages"]))

def test_split_file_creates_chunk_jobs_and_links_attachments(self):
import_wizard = self.import_wizard.create(
{
"res_model": self.res_partners._name,
"file": "dummy",
"file_type": "text/csv",
}
)
fields = ["name", "email", "is_company"]
data = [
["partner 1", "[email protected]", "1"],
["partner 2", "[email protected]", "0"],
["partner 3", "[email protected]", "0"],
]
opts = {
OPT_QUOTING: '"',
OPT_SEPARATOR: ";",
OPT_HAS_HEADER: True,
OPT_CHUNK_SIZE: 2,
}
attachment = import_wizard._create_csv_attachment(
fields, data, opts, "partners.csv"
)
with RecordCapturer(self.env["queue.job"], []) as capture:
import_wizard._split_file(
model_name=self.res_partners._name,
translated_model_name="Contacts",
attachment=attachment,
options=opts,
file_name="partners.csv",
)
jobs = capture.records.sorted("priority")
self.assertEqual(len(jobs), 2)
self.assertEqual(jobs.mapped("method_name"), ["_import_one_chunk"] * 2)
self.assertEqual(jobs.mapped("priority"), [INIT_PRIORITY, INIT_PRIORITY + 1])
self.assertEqual(
jobs.mapped("name"),
[
"Import Contacts from file partners.csv - #0 - lines 2 to 3",
"Import Contacts from file partners.csv - #1 - lines 4 to 4",
],
)
chunk_attachments = self.env["ir.attachment"].search(
[("name", "in", ["partners-0.csv", "partners-1.csv"])]
)
self.assertEqual(len(chunk_attachments), 2)
self.assertTrue(all(att.res_model == "queue.job" for att in chunk_attachments))
self.assertTrue(all(att.res_id in jobs.ids for att in chunk_attachments))

def test_import_one_chunk_success_and_error(self):
import_wizard = self.import_wizard.create(
{
"res_model": self.res_partners._name,
"file": "dummy",
"file_type": "text/csv",
}
)
opts = {OPT_QUOTING: '"', OPT_SEPARATOR: ";"}

fields = ["name", "email", "is_company"]
data = [
["partner 1", "[email protected]", "1"],
["partner 2", "[email protected]", "0"],
]
attachment = import_wizard._create_csv_attachment(
fields, data, opts, "partners.csv"
)
with RecordCapturer(self.res_partners, []) as capture:
result = import_wizard._import_one_chunk(
self.res_partners._name, attachment, opts
)
self.assertEqual(result["messages"], [])
self.assertEqual(len(capture.records), 2)

invalid_fields = ["name", "company_type"]
invalid_data = [["partner 3", "invalid"]]
invalid_attachment = import_wizard._create_csv_attachment(
invalid_fields, invalid_data, opts, "invalid.csv"
)
with self.assertRaises(FailedJobError):
import_wizard._import_one_chunk(
self.res_partners._name, invalid_attachment, opts
)
3 changes: 3 additions & 0 deletions base_import_async/wizard/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from . import base_import_import
Loading