diff --git a/account_statement_import_sheet_file/README.rst b/account_statement_import_sheet_file/README.rst index b74219224e..410f00510f 100644 --- a/account_statement_import_sheet_file/README.rst +++ b/account_statement_import_sheet_file/README.rst @@ -84,11 +84,12 @@ Contributors * Alexis de Lattre * Sebastien BEAU -* Mourad EL HADJ MIMOUNE +* Katherine Zaoral * Tecnativa (https://www.tecnativa.com) * Vicent Cubells * Victor M.M. Torres + * Víctor Martínez * ForgeFlow (https://www.forgeflow.com) diff --git a/account_statement_import_sheet_file/__init__.py b/account_statement_import_sheet_file/__init__.py index aee8895e7a..0650744f6b 100644 --- a/account_statement_import_sheet_file/__init__.py +++ b/account_statement_import_sheet_file/__init__.py @@ -1,2 +1 @@ from . import models -from . import wizards diff --git a/account_statement_import_sheet_file/__manifest__.py b/account_statement_import_sheet_file/__manifest__.py index 6eab5d7bdd..34446897ee 100644 --- a/account_statement_import_sheet_file/__manifest__.py +++ b/account_statement_import_sheet_file/__manifest__.py @@ -13,9 +13,7 @@ "license": "AGPL-3", "installable": True, "depends": [ - "account_statement_import", - "multi_step_wizard", - "web_widget_dropdown_dynamic", + "account_statement_import_file", ], "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ @@ -24,6 +22,5 @@ "views/account_statement_import_sheet_mapping.xml", "views/account_statement_import.xml", "views/account_journal_views.xml", - "wizards/account_statement_import_sheet_mapping_wizard.xml", ], } diff --git a/account_statement_import_sheet_file/data/map_data.xml b/account_statement_import_sheet_file/data/map_data.xml index f874660158..ea50132080 100644 --- a/account_statement_import_sheet_file/data/map_data.xml +++ b/account_statement_import_sheet_file/data/map_data.xml @@ -7,12 +7,15 @@ Sample Statement + 0 + 1 comma dot comma " %m/%d/%Y Date + simple_value Amount Currency Amount Currency diff --git a/account_statement_import_sheet_file/migrations/16.0.1.0.0/pre-migration.py b/account_statement_import_sheet_file/migrations/16.0.1.0.0/pre-migration.py new file mode 100644 index 0000000000..4be5b89c08 --- /dev/null +++ b/account_statement_import_sheet_file/migrations/16.0.1.0.0/pre-migration.py @@ -0,0 +1,23 @@ +# Copyright 2023 Tecnativa - Víctor Martínez +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade + +_field_renames = [ + ( + "account.statement.import.sheet.mapping", + "account_statement_import_sheet_mapping", + "footer_lines_count", + "footer_lines_skip_count", + )( + "account.statement.import.sheet.mapping", + "account_statement_import_sheet_mapping", + "column_labels_row", + "header_lines_skip_count", + ) +] + + +@openupgrade.migrate() +def migrate(env, version): + openupgrade.rename_fields(env, _field_renames) diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py index caefa7ac19..59213e69d0 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py @@ -70,6 +70,7 @@ class AccountStatementImportSheetMapping(models.Model): amount_column = fields.Char( help="Amount of transaction in journal's currency", ) + amount_debit_column = fields.Char( string="Debit amount column", help="Debit amount of transaction in journal's currency", @@ -95,10 +96,35 @@ class AccountStatementImportSheetMapping(models.Model): "transaction amount in original transaction currency from" ), ) + amount_type = fields.Selection( + selection=[ + ("simple_value", "Simple value"), + ("absolute_value", "Absolute value"), + ("distinct_credit_debit", "Distinct Credit/debit Column"), + ], + string="Amount type", + required=True, + default="simple_value", + help=( + "Simple value: use igned amount in ammount comlumn\n" + "Absolute Value: use a same comlumn for debit and credit\n" + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct comlumn for debit and credit" + ), + ) + amount_column = fields.Char( + string="Amount column", + help=( + 'Used if amount type is "Simple value" or "Absolute value"\n' + "Amount of transaction in journal's currency\n" + "Some statement formats use credit/debit columns" + ), + ) debit_credit_column = fields.Char( string="Debit/credit column", help=( - "Some statement formats use absolute amount value and indicate sign" + 'Used if amount type is "Absolute value"\n' + "Some statement formats use absolute amount value and indicate sign\n" "of the transaction by specifying if it was a debit or a credit one" ), ) @@ -123,6 +149,18 @@ class AccountStatementImportSheetMapping(models.Model): bank_account_column = fields.Char( help="Partner's bank account", ) + footer_lines_skip_count = fields.Integer( + string="Footer lines skip count", + help="Set the Footer lines number." + "Used in some csv/xlsx file that integrate meta data in" + "last lines.", + default="0", + ) + header_lines_skip_count = fields.Integer( + string="Header lines skip count", + help="Set the Header lines number.", + default="0", + ) _sql_constraints = [ ( diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py index deee5fd02d..e55046352d 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py @@ -36,21 +36,23 @@ class AccountStatementImportSheetParser(models.TransientModel): _description = "Bank Statement Import Sheet Parser" @api.model - def parse_header(self, data_file, encoding, csv_options): + def parse_header(self, data_file, encoding, csv_options, header_lines_skip_count=1): try: workbook = xlrd.open_workbook( file_contents=data_file, encoding_override=encoding if encoding else None, ) sheet = workbook.sheet_by_index(0) - values = sheet.row_values(0) + values = sheet.row_values(header_lines_skip_count - 1) return [str(value) for value in values] except xlrd.XLRDError: _logger.error("Pass this method") data = StringIO(data_file.decode(encoding or "utf-8")) csv_data = reader(data, **csv_options) - return list(next(csv_data)) + csv_data_lst = list(csv_data) + header = [value.strip() for value in csv_data_lst[header_lines_skip_count - 1]] + return header @api.model def parse(self, data_file, mapping, filename): @@ -62,9 +64,12 @@ def parse(self, data_file, mapping, filename): if not lines: return currency_code, account_number, [{"transactions": []}] - lines = list(sorted(lines, key=lambda line: line["timestamp"])) - first_line = lines[0] - last_line = lines[-1] + if lines[0]["timestamp"] > lines[-1]["timestamp"]: + first_line = lines[-1] + last_line = lines[0] + else: + first_line = lines[0] + last_line = lines[-1] data = { "date": first_line["timestamp"].date(), "name": _("%(code)s: %(filename)s") @@ -171,14 +176,25 @@ def _parse_lines(self, mapping, data_file, currency_code): header = False if not mapping.no_header: if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + header = [ + str(value).strip() + for value in csv_or_xlsx[1].row_values( + mapping.header_lines_skip_count - 1 + ) + ] else: + for _i in range(mapping.header_lines_skip_count - 1): + next(csv_or_xlsx) header = [value.strip() for value in next(csv_or_xlsx)] + + # NOTE no seria necesario debit_column y credit_column ya que tenemos los + # respectivos campos related for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping ) - return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + data = csv_or_xlsx, data_file + return self._parse_rows(mapping, currency_code, data, columns) def _get_values_from_column(self, values, columns, column_name): indexes = columns[column_name] @@ -195,25 +211,38 @@ def _get_values_from_column(self, values, columns, column_name): return " ".join(content_l) return content_l[0] - def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 + def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 + csv_or_xlsx, data_file = data + + # Get the numbers of rows of the file + if isinstance(csv_or_xlsx, tuple): + numrows = csv_or_xlsx[1].nrows + else: + numrows = len(str(data_file.strip()).split("\\n")) + + label_line = mapping.header_lines_skip_count + footer_line = numrows - mapping.footer_lines_skip_count + if isinstance(csv_or_xlsx, tuple): - rows = range(1, csv_or_xlsx[1].nrows) + rows = range(mapping.header_lines_skip_count, footer_line) else: rows = csv_or_xlsx lines = [] - for row in rows: + for index, row in enumerate(rows, label_line): if isinstance(csv_or_xlsx, tuple): book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(sheet.row_len(row)): + for col_index in range(0, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: cell_value = xldate_as_datetime(cell_value, book.datemode) values.append(cell_value) else: + if index >= footer_line: + continue values = list(row) timestamp = self._get_values_from_column( @@ -307,7 +336,7 @@ def _decimal(column_name): else: balance = None - if debit_credit: + if debit_credit is not None: amount = amount.copy_abs() if debit_credit == mapping.debit_value: amount = -amount @@ -429,6 +458,7 @@ def _parse_decimal(self, value, mapping): return value elif isinstance(value, float): return Decimal(value) + value = value or "0" thousands, decimal = mapping._get_float_separators() value = value.replace(thousands, "") value = value.replace(decimal, ".") diff --git a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst index d18dd9a9bb..36fc13e148 100644 --- a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst +++ b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst @@ -1,9 +1,11 @@ * Alexis de Lattre * Sebastien BEAU +* Katherine Zaoral * Tecnativa (https://www.tecnativa.com) * Vicent Cubells * Victor M.M. Torres + * Víctor Martínez * ForgeFlow (https://www.forgeflow.com) diff --git a/account_statement_import_sheet_file/security/ir.model.access.csv b/account_statement_import_sheet_file/security/ir.model.access.csv index aa8c3d9555..be92c3a142 100644 --- a/account_statement_import_sheet_file/security/ir.model.access.csv +++ b/account_statement_import_sheet_file/security/ir.model.access.csv @@ -2,4 +2,3 @@ access_account_statement_import_sheet_mapping_manager,account.statement.import.sheet.mapping:account.group_account_manager,model_account_statement_import_sheet_mapping,account.group_account_manager,1,1,1,1 access_account_statement_import_sheet_mapping_user,account.statement.import.sheet.mapping:account.group_account_user,model_account_statement_import_sheet_mapping,account.group_account_user,1,0,0,0 access_account_statement_import_sheet_parser,account.statement.import.sheet.parser:account.group_account_user,model_account_statement_import_sheet_parser,account.group_account_user,1,1,1,1 -access_account_statement_import_sheet_mapping_wizard,Full access on account.statement.import.sheet.mapping.wizard,model_account_statement_import_sheet_mapping_wizard,account.group_account_user,1,1,1,1 diff --git a/account_statement_import_sheet_file/static/description/index.html b/account_statement_import_sheet_file/static/description/index.html index 1d6a454114..3890b48add 100644 --- a/account_statement_import_sheet_file/static/description/index.html +++ b/account_statement_import_sheet_file/static/description/index.html @@ -438,10 +438,11 @@

Contributors

  • Alexis de Lattre <alexis.delattre@akretion.com>
  • Sebastien BEAU <sebastien.beau@akretion.com>
    • -
  • Mourad EL HADJ MIMOUNE <mourad.elhadj.mimoune@akretion.com>
    • +
  • Katherine Zaoral
  • Tecnativa (https://www.tecnativa.com)
    • Vicent Cubells
    • Victor M.M. Torres
      • +
    • Víctor Martínez
  • ForgeFlow (https://www.forgeflow.com)
      diff --git a/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.csv b/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.csv new file mode 100644 index 0000000000..0b38621283 --- /dev/null +++ b/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.csv @@ -0,0 +1,10 @@ +Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,, +Account Number : 08088804068,Account Name : Account Owner,: EUR,,, +,,,,, +Balance at end of period,,,,"+31070,11", +Date,Operation Number,Label,Debit,Credit,Detail +01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1 +01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X +02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2 +02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3 +Balance at start of period,,,,"+30000,77", diff --git a/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.xlsx b/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.xlsx new file mode 100644 index 0000000000..c7ae2b92e9 Binary files /dev/null and b/account_statement_import_sheet_file/tests/fixtures/meta_data_separated_credit_debit.xlsx differ diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_sheet_file/tests/test_account_statement_import_txt_xlsx.py index 563d9058b6..97bc62c7dc 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_txt_xlsx.py @@ -8,6 +8,7 @@ from odoo import fields from odoo.exceptions import UserError from odoo.tests import common +from odoo.tools import float_round class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): @@ -36,16 +37,12 @@ def setUp(self): self.AccountStatementImportSheetMapping = self.env[ "account.statement.import.sheet.mapping" ] - self.AccountStatementImportSheetMappingWizard = self.env[ - "account.statement.import.sheet.mapping.wizard" - ] + self.AccountStatementImportWizard = self.env["account.statement.import"] self.suspense_account = self.env["account.account"].create( { "code": "987654", "name": "Suspense Account", - "user_type_id": self.env.ref( - "account.data_account_type_current_assets" - ).id, + "account_type": "asset_current", } ) @@ -157,52 +154,6 @@ def test_import_empty_xlsx_file(self): statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 0) - def test_mapping_import_wizard_xlsx(self): - with common.Form(self.AccountStatementImportSheetMappingWizard) as form: - attachment = self.env["ir.attachment"].create( - { - "name": "fixtures/empty_statement_en.xlsx", - "datas": self._data_file("fixtures/empty_statement_en.xlsx"), - } - ) - form.attachment_ids.add(attachment) - self.assertEqual(len(form.header), 90) - self.assertEqual( - len( - self.AccountStatementImportSheetMappingWizard.with_context( - header=form.header, - ).statement_columns() - ), - 7, - ) - form.timestamp_column = "Date" - form.amount_column = "Amount" - wizard = form.save() - wizard.import_mapping() - - def test_mapping_import_wizard_csv(self): - with common.Form(self.AccountStatementImportSheetMappingWizard) as form: - attachment = self.env["ir.attachment"].create( - { - "name": "fixtures/empty_statement_en.csv", - "datas": self._data_file("fixtures/empty_statement_en.csv"), - } - ) - form.attachment_ids.add(attachment) - self.assertEqual(len(form.header), 90) - self.assertEqual( - len( - self.AccountStatementImportSheetMappingWizard.with_context( - header=form.header, - ).statement_columns() - ), - 7, - ) - form.timestamp_column = "Date" - form.amount_column = "Amount" - wizard = form.save() - wizard.import_mapping() - def test_original_currency(self): journal = self.AccountJournal.create( { @@ -232,7 +183,8 @@ def test_original_currency(self): self.assertEqual(line.currency_id, self.currency_usd) self.assertEqual(line.amount, 1525.0) self.assertEqual(line.foreign_currency_id, self.currency_eur) - self.assertEqual(line.amount_currency, 1000.0) + line_amount_currency = float_round(line.amount_currency, precision_digits=1) + self.assertEqual(line_amount_currency, 1000.0) def test_original_currency_no_header(self): no_header_statement_map = self.AccountStatementImportSheetMapping.create( @@ -240,6 +192,7 @@ def test_original_currency_no_header(self): "name": "Sample Statement", "float_thousands_sep": "comma", "float_decimal_sep": "dot", + "header_lines_skip_count": 0, "delimiter": "comma", "quotechar": '"', "timestamp_format": "%m/%d/%Y", @@ -458,3 +411,97 @@ def test_debit_credit_amount(self): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_metadata_separated_debit_credit_csv(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_skip_count": 1, + "header_lines_skip_count": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "amount_debit_column": "Debit", + "amount_credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) + + def test_metadata_separated_debit_credit_xlsx(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_skip_count": 1, + "header_lines_skip_count": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "amount_debit_column": "Debit", + "amount_credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.xlsx", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) diff --git a/account_statement_import_sheet_file/views/account_statement_import.xml b/account_statement_import_sheet_file/views/account_statement_import.xml index 4ef359aa77..f9697c3ca8 100644 --- a/account_statement_import_sheet_file/views/account_statement_import.xml +++ b/account_statement_import_sheet_file/views/account_statement_import.xml @@ -10,7 +10,7 @@ account.statement.import diff --git a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml index 62d617dd0b..3165de06d3 100644 --- a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml @@ -63,10 +63,14 @@ attrs="{'required': [('debit_credit_column', '!=', False)]}" /> + + + + - -