[IMP] account_bank_statement_import_txt_xlsx: CSV

[IMP] add csv meta data management
[FIX] exclude footer meta data
[IMP] import separated credit/debit column file
[FIX] make comptatible with new version of multi_step_wizard module & add migration file
[FIX] all not provided value are handled in_parse_decimal method
[REF] Remove unnecessary \n

Renamed to account_statement_import_sheet_file

Co-authored-by: zaoral

account_statement_import_sheet_file/README.rst

@@ -14,10 +14,10 @@ Bank Statement TXT/CSV/XLSX Import
     :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
     :alt: License: AGPL-3
 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fbank--statement--import-lightgray.png?logo=github
-    :target: https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_txt_xlsx
+    :target: https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_sheet_file
     :alt: OCA/bank-statement-import
 .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
-    :target: https://translation.odoo-community.org/projects/bank-statement-import-16-0/bank-statement-import-16-0-account_statement_import_txt_xlsx
+    :target: https://translation.odoo-community.org/projects/bank-statement-import-16-0/bank-statement-import-16-0-account_statement_import_sheet_file
     :alt: Translate me on Weblate
 .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png
     :target: https://runbot.odoo-community.org/runbot/174/16.0
@@ -66,7 +66,7 @@ Bug Tracker
 Bugs are tracked on `GitHub Issues <https://github.com/OCA/bank-statement-import/issues>`_.
 In case of trouble, please check there if your issue has already been reported.
 If you spotted it first, help us smashing it by providing a detailed and welcomed
-`feedback <https://github.com/OCA/bank-statement-import/issues/new?body=module:%20account_statement_import_txt_xlsx%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+`feedback <https://github.com/OCA/bank-statement-import/issues/new?body=module:%20account_statement_import_sheet_file%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
 
 Do not contact contributors directly about support or help with technical issues.
 
@@ -84,11 +84,12 @@ Contributors
 
 * Alexis de Lattre <alexis.delattre@akretion.com>
 * Sebastien BEAU <sebastien.beau@akretion.com>
-* Mourad EL HADJ MIMOUNE <mourad.elhadj.mimoune@akretion.com>
+* Katherine Zaoral
 * Tecnativa (https://www.tecnativa.com)
 
   * Vicent Cubells
   * Victor M.M. Torres
+  * Víctor Martínez
 
 * ForgeFlow (https://www.forgeflow.com)
 
@@ -120,6 +121,6 @@ Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
 
 |maintainer-alexey-pelykh| 
 
-This module is part of the `OCA/bank-statement-import <https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_txt_xlsx>`_ project on GitHub.
+This module is part of the `OCA/bank-statement-import <https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_sheet_file>`_ project on GitHub.
 
 You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

account_statement_import_sheet_file/__init__.py

@@ -1,2 +1 @@
 from . import models
-from . import wizards

account_statement_import_sheet_file/__manifest__.py

@@ -13,9 +13,7 @@
     "license": "AGPL-3",
     "installable": True,
     "depends": [
-        "account_statement_import",
-        "multi_step_wizard",
-        "web_widget_dropdown_dynamic",
+        "account_statement_import_file",
     ],
     "external_dependencies": {"python": ["xlrd", "chardet"]},
     "data": [
@@ -24,6 +22,5 @@
         "views/account_statement_import_sheet_mapping.xml",
         "views/account_statement_import.xml",
         "views/account_journal_views.xml",
-        "wizards/account_statement_import_sheet_mapping_wizard.xml",
     ],
 }

account_statement_import_sheet_file/data/map_data.xml

@@ -7,12 +7,15 @@
 <odoo noupdate="1">
     <record id="sample_statement_map" model="account.statement.import.sheet.mapping">
         <field name="name">Sample Statement</field>
+        <field name="footer_lines_skip_count">0</field>
+        <field name="header_lines_skip_count">1</field>
         <field name="float_thousands_sep">comma</field>
         <field name="float_decimal_sep">dot</field>
         <field name="delimiter">comma</field>
         <field name="quotechar">"</field>
         <field name="timestamp_format">%m/%d/%Y</field>
         <field name="timestamp_column">Date</field>
+        <field name="amount_type">simple_value</field>
         <field name="amount_column">Amount</field>
         <field name="original_currency_column">Currency</field>
         <field name="original_amount_column">Amount Currency</field>

account_statement_import_sheet_file/migrations/16.0.1.0.0/pre-migration.py

@@ -0,0 +1,23 @@
+# Copyright 2023 Tecnativa - Víctor Martínez
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from openupgradelib import openupgrade
+
+_field_renames = [
+    (
+        "account.statement.import.sheet.mapping",
+        "account_statement_import_sheet_mapping",
+        "footer_lines_count",
+        "footer_lines_skip_count",
+    )(
+        "account.statement.import.sheet.mapping",
+        "account_statement_import_sheet_mapping",
+        "column_labels_row",
+        "header_lines_skip_count",
+    )
+]
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+    openupgrade.rename_fields(env, _field_renames)

account_statement_import_sheet_file/models/account_statement_import.py

@@ -34,7 +34,7 @@ def _parse_file(self, data_file):
                     data_file, self.sheet_mapping_id, self.statement_filename
                 )
             except BaseException as exc:
-                if self.env.context.get("account_statement_import_txt_xlsx_test"):
+                if self.env.context.get("account_statement_import_sheet_file_test"):
                     raise
                 _logger.warning("Sheet parser error", exc_info=True)
                 raise UserError(_("Bad file/mapping: ") + str(exc)) from exc

account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py

@@ -70,6 +70,7 @@ class AccountStatementImportSheetMapping(models.Model):
     amount_column = fields.Char(
         help="Amount of transaction in journal's currency",
     )
+
     amount_debit_column = fields.Char(
         string="Debit amount column",
         help="Debit amount of transaction in journal's currency",
@@ -95,10 +96,35 @@ class AccountStatementImportSheetMapping(models.Model):
             "transaction amount in original transaction currency from"
         ),
     )
+    amount_type = fields.Selection(
+        selection=[
+            ("simple_value", "Simple value"),
+            ("absolute_value", "Absolute value"),
+            ("distinct_credit_debit", "Distinct Credit/debit Column"),
+        ],
+        string="Amount type",
+        required=True,
+        default="simple_value",
+        help=(
+            "Simple value: use igned amount in ammount comlumn\n"
+            "Absolute Value: use a same comlumn for debit and credit\n"
+            "(absolute value + indicate sign)\n"
+            "Distinct Credit/debit Column: use a distinct comlumn for debit and credit"
+        ),
+    )
+    amount_column = fields.Char(
+        string="Amount column",
+        help=(
+            'Used if amount type is "Simple value" or "Absolute value"\n'
+            "Amount of transaction in journal's currency\n"
+            "Some statement formats use credit/debit columns"
+        ),
+    )
     debit_credit_column = fields.Char(
         string="Debit/credit column",
         help=(
-            "Some statement formats use absolute amount value and indicate sign"
+            'Used if amount type is "Absolute value"\n'
+            "Some statement formats use absolute amount value and indicate sign\n"
             "of the transaction by specifying if it was a debit or a credit one"
         ),
     )
@@ -123,6 +149,18 @@ class AccountStatementImportSheetMapping(models.Model):
     bank_account_column = fields.Char(
         help="Partner's bank account",
     )
+    footer_lines_skip_count = fields.Integer(
+        string="Footer lines skip count",
+        help="Set the Footer lines number."
+        "Used in some csv/xlsx file that integrate meta data in"
+        "last lines.",
+        default="0",
+    )
+    header_lines_skip_count = fields.Integer(
+        string="Header lines skip count",
+        help="Set the Header lines number.",
+        default="0",
+    )
 
     _sql_constraints = [
         (

account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py

@@ -36,21 +36,23 @@ class AccountStatementImportSheetParser(models.TransientModel):
     _description = "Bank Statement Import Sheet Parser"
 
     @api.model
-    def parse_header(self, data_file, encoding, csv_options):
+    def parse_header(self, data_file, encoding, csv_options, header_lines_skip_count=1):
         try:
             workbook = xlrd.open_workbook(
                 file_contents=data_file,
                 encoding_override=encoding if encoding else None,
             )
             sheet = workbook.sheet_by_index(0)
-            values = sheet.row_values(0)
+            values = sheet.row_values(header_lines_skip_count - 1)
             return [str(value) for value in values]
         except xlrd.XLRDError:
             _logger.error("Pass this method")
 
         data = StringIO(data_file.decode(encoding or "utf-8"))
         csv_data = reader(data, **csv_options)
-        return list(next(csv_data))
+        csv_data_lst = list(csv_data)
+        header = [value.strip() for value in csv_data_lst[header_lines_skip_count - 1]]
+        return header
 
     @api.model
     def parse(self, data_file, mapping, filename):
@@ -62,9 +64,12 @@ def parse(self, data_file, mapping, filename):
         if not lines:
             return currency_code, account_number, [{"transactions": []}]
 
-        lines = list(sorted(lines, key=lambda line: line["timestamp"]))
-        first_line = lines[0]
-        last_line = lines[-1]
+        if lines[0]["timestamp"] > lines[-1]["timestamp"]:
+            first_line = lines[-1]
+            last_line = lines[0]
+        else:
+            first_line = lines[0]
+            last_line = lines[-1]
         data = {
             "date": first_line["timestamp"].date(),
             "name": _("%(code)s: %(filename)s")
@@ -171,14 +176,25 @@ def _parse_lines(self, mapping, data_file, currency_code):
         header = False
         if not mapping.no_header:
             if isinstance(csv_or_xlsx, tuple):
-                header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+                header = [
+                    str(value).strip()
+                    for value in csv_or_xlsx[1].row_values(
+                        mapping.header_lines_skip_count - 1
+                    )
+                ]
             else:
+                for _i in range(mapping.header_lines_skip_count - 1):
+                    next(csv_or_xlsx)
                 header = [value.strip() for value in next(csv_or_xlsx)]
+
+        # NOTE no seria necesario debit_column y credit_column ya que tenemos los
+        # respectivos campos related
         for column_name in self._get_column_names():
             columns[column_name] = self._get_column_indexes(
                 header, column_name, mapping
             )
-        return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
+        data = csv_or_xlsx, data_file
+        return self._parse_rows(mapping, currency_code, data, columns)
 
     def _get_values_from_column(self, values, columns, column_name):
         indexes = columns[column_name]
@@ -195,25 +211,38 @@ def _get_values_from_column(self, values, columns, column_name):
             return " ".join(content_l)
         return content_l[0]
 
-    def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
+    def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
+        csv_or_xlsx, data_file = data
+
+        # Get the numbers of rows of the file
+        if isinstance(csv_or_xlsx, tuple):
+            numrows = csv_or_xlsx[1].nrows
+        else:
+            numrows = len(str(data_file.strip()).split("\\n"))
+
+        label_line = mapping.header_lines_skip_count
+        footer_line = numrows - mapping.footer_lines_skip_count
+
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(1, csv_or_xlsx[1].nrows)
+            rows = range(mapping.header_lines_skip_count, footer_line)
         else:
             rows = csv_or_xlsx
 
         lines = []
-        for row in rows:
+        for index, row in enumerate(rows, label_line):
             if isinstance(csv_or_xlsx, tuple):
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(sheet.row_len(row)):
+                for col_index in range(0, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
                         cell_value = xldate_as_datetime(cell_value, book.datemode)
                     values.append(cell_value)
             else:
+                if index >= footer_line:
+                    continue
                 values = list(row)
 
             timestamp = self._get_values_from_column(
@@ -307,7 +336,7 @@ def _decimal(column_name):
             else:
                 balance = None
 
-            if debit_credit:
+            if debit_credit is not None:
                 amount = amount.copy_abs()
                 if debit_credit == mapping.debit_value:
                     amount = -amount
@@ -429,6 +458,7 @@ def _parse_decimal(self, value, mapping):
             return value
         elif isinstance(value, float):
             return Decimal(value)
+        value = value or "0"
         thousands, decimal = mapping._get_float_separators()
         value = value.replace(thousands, "")
         value = value.replace(decimal, ".")

account_statement_import_sheet_file/readme/CONTRIBUTORS.rst

@@ -1,9 +1,11 @@
 * Alexis de Lattre <alexis.delattre@akretion.com>
 * Sebastien BEAU <sebastien.beau@akretion.com>
+* Katherine Zaoral
 * Tecnativa (https://www.tecnativa.com)
 
   * Vicent Cubells
   * Victor M.M. Torres
+  * Víctor Martínez
 
 * ForgeFlow (https://www.forgeflow.com)
 

account_statement_import_sheet_file/security/ir.model.access.csv

@@ -2,4 +2,3 @@
 access_account_statement_import_sheet_mapping_manager,account.statement.import.sheet.mapping:account.group_account_manager,model_account_statement_import_sheet_mapping,account.group_account_manager,1,1,1,1
 access_account_statement_import_sheet_mapping_user,account.statement.import.sheet.mapping:account.group_account_user,model_account_statement_import_sheet_mapping,account.group_account_user,1,0,0,0
 access_account_statement_import_sheet_parser,account.statement.import.sheet.parser:account.group_account_user,model_account_statement_import_sheet_parser,account.group_account_user,1,1,1,1
-access_account_statement_import_sheet_mapping_wizard,Full access on account.statement.import.sheet.mapping.wizard,model_account_statement_import_sheet_mapping_wizard,account.group_account_user,1,1,1,1

account_statement_import_sheet_file/static/description/index.html

@@ -367,7 +367,7 @@ <h1 class="title">Bank Statement TXT/CSV/XLSX Import</h1>
 !! This file is generated by oca-gen-addon-readme !!
 !! changes will be overwritten.                   !!
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->
-<p><a class="reference external" href="https://odoo-community.org/page/development-status"><img alt="Beta" src="https://img.shields.io/badge/maturity-Beta-yellow.png" /></a> <a class="reference external" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external" href="https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_txt_xlsx"><img alt="OCA/bank-statement-import" src="https://img.shields.io/badge/github-OCA%2Fbank--statement--import-lightgray.png?logo=github" /></a> <a class="reference external" href="https://translation.odoo-community.org/projects/bank-statement-import-16-0/bank-statement-import-16-0-account_statement_import_txt_xlsx"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external" href="https://runbot.odoo-community.org/runbot/174/16.0"><img alt="Try me on Runbot" src="https://img.shields.io/badge/runbot-Try%20me-875A7B.png" /></a></p>
+<p><a class="reference external" href="https://odoo-community.org/page/development-status"><img alt="Beta" src="https://img.shields.io/badge/maturity-Beta-yellow.png" /></a> <a class="reference external" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external" href="https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_sheet_file"><img alt="OCA/bank-statement-import" src="https://img.shields.io/badge/github-OCA%2Fbank--statement--import-lightgray.png?logo=github" /></a> <a class="reference external" href="https://translation.odoo-community.org/projects/bank-statement-import-16-0/bank-statement-import-16-0-account_statement_import_sheet_file"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external" href="https://runbot.odoo-community.org/runbot/174/16.0"><img alt="Try me on Runbot" src="https://img.shields.io/badge/runbot-Try%20me-875A7B.png" /></a></p>
 <p>This module allows you to import any TXT/CSV or XLSX file in Odoo as bank
 statements.</p>
 <p><strong>Table of contents</strong></p>
@@ -421,7 +421,7 @@ <h1><a class="toc-backref" href="#id6">Bug Tracker</a></h1>
 <p>Bugs are tracked on <a class="reference external" href="https://github.com/OCA/bank-statement-import/issues">GitHub Issues</a>.
 In case of trouble, please check there if your issue has already been reported.
 If you spotted it first, help us smashing it by providing a detailed and welcomed
-<a class="reference external" href="https://github.com/OCA/bank-statement-import/issues/new?body=module:%20account_statement_import_txt_xlsx%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p>
+<a class="reference external" href="https://github.com/OCA/bank-statement-import/issues/new?body=module:%20account_statement_import_sheet_file%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p>
 <p>Do not contact contributors directly about support or help with technical issues.</p>
 </div>
 <div class="section" id="credits">
@@ -438,10 +438,11 @@ <h2><a class="toc-backref" href="#id9">Contributors</a></h2>
 <ul class="simple">
 <li>Alexis de Lattre &lt;<a class="reference external" href="mailto:alexis.delattre&#64;akretion.com">alexis.delattre&#64;akretion.com</a>&gt;</li>
 <li>Sebastien BEAU &lt;<a class="reference external" href="mailto:sebastien.beau&#64;akretion.com">sebastien.beau&#64;akretion.com</a>&gt;</li>
-<li>Mourad EL HADJ MIMOUNE &lt;<a class="reference external" href="mailto:mourad.elhadj.mimoune&#64;akretion.com">mourad.elhadj.mimoune&#64;akretion.com</a>&gt;</li>
+<li>Katherine Zaoral</li>
 <li>Tecnativa (<a class="reference external" href="https://www.tecnativa.com">https://www.tecnativa.com</a>)<ul>
 <li>Vicent Cubells</li>
 <li>Victor M.M. Torres</li>
+<li>Víctor Martínez</li>
 </ul>
 </li>
 <li>ForgeFlow (<a class="reference external" href="https://www.forgeflow.com">https://www.forgeflow.com</a>)<ul>
@@ -464,7 +465,7 @@ <h2><a class="toc-backref" href="#id10">Maintainers</a></h2>
 promote its widespread use.</p>
 <p>Current <a class="reference external" href="https://odoo-community.org/page/maintainer-role">maintainer</a>:</p>
 <p><a class="reference external" href="https://github.com/alexey-pelykh"><img alt="alexey-pelykh" src="https://github.com/alexey-pelykh.png?size=40px" /></a></p>
-<p>This module is part of the <a class="reference external" href="https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_txt_xlsx">OCA/bank-statement-import</a> project on GitHub.</p>
+<p>This module is part of the <a class="reference external" href="https://github.com/OCA/bank-statement-import/tree/16.0/account_statement_import_sheet_file">OCA/bank-statement-import</a> project on GitHub.</p>
 <p>You are welcome to contribute. To learn how please visit <a class="reference external" href="https://odoo-community.org/page/Contribute">https://odoo-community.org/page/Contribute</a>.</p>
 </div>
 </div>

account_statement_import_sheet_file/tests/__init__.py

@@ -1,3 +1,3 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
-from . import test_account_statement_import_txt_xlsx
+from . import test_account_statement_import_sheet_file