diff --git a/account_statement_import_txt_xlsx/__init__.py b/account_statement_import_txt_xlsx/__init__.py index aee8895e7a..0650744f6b 100644 --- a/account_statement_import_txt_xlsx/__init__.py +++ b/account_statement_import_txt_xlsx/__init__.py @@ -1,2 +1 @@ from . import models -from . import wizards diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index 6eab5d7bdd..34446897ee 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -13,9 +13,7 @@ "license": "AGPL-3", "installable": True, "depends": [ - "account_statement_import", - "multi_step_wizard", - "web_widget_dropdown_dynamic", + "account_statement_import_file", ], "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ @@ -24,6 +22,5 @@ "views/account_statement_import_sheet_mapping.xml", "views/account_statement_import.xml", "views/account_journal_views.xml", - "wizards/account_statement_import_sheet_mapping_wizard.xml", ], } diff --git a/account_statement_import_txt_xlsx/data/map_data.xml b/account_statement_import_txt_xlsx/data/map_data.xml index f874660158..29f48cdd87 100644 --- a/account_statement_import_txt_xlsx/data/map_data.xml +++ b/account_statement_import_txt_xlsx/data/map_data.xml @@ -7,12 +7,15 @@ Sample Statement + 0 + 1 comma dot comma " %m/%d/%Y Date + simple_value Amount Currency Amount Currency diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index caefa7ac19..e118192894 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -70,14 +70,30 @@ class AccountStatementImportSheetMapping(models.Model): amount_column = fields.Char( help="Amount of transaction in journal's currency", ) - amount_debit_column = fields.Char( + + debit_column = fields.Char( string="Debit amount column", help="Debit amount of transaction in journal's currency", ) - amount_credit_column = fields.Char( + credit_column = fields.Char( string="Credit amount column", help="Credit amount of transaction in journal's currency", ) + + # TODO to avoid error in un customer, need to fix using migrate script + amount_debit_column = fields.Char( + related="debit_column", + store=True, + string="OCA Debit Column", + readonly=False, + ) + amount_credit_column = fields.Char( + related="credit_column", + store=True, + string="OCA Credit Column", + readonly=False, + ) + balance_column = fields.Char( help="Balance after transaction in journal's currency", ) @@ -95,10 +111,43 @@ class AccountStatementImportSheetMapping(models.Model): "transaction amount in original transaction currency from" ), ) + amount_type = fields.Selection( + selection=[ + ("simple_value", "Simple value"), + ("absolute_value", "Absolute value"), + ("distinct_credit_debit", "Distinct Credit/debit Column"), + ], + string="Amount type", + required=True, + default="simple_value", + help=( + "Simple value: use igned amount in ammount comlumn\n" + "Absolute Value: use a same comlumn for debit and credit\n" + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct comlumn for debit and credit" + ), + ) + amount_column = fields.Char( + string="Amount column", + help=( + 'Used if amount type is "Simple value" or "Absolute value"\n' + "Amount of transaction in journal's currency\n" + "Some statement formats use credit/debit columns" + ), + ) + debit_column = fields.Char( + string="Debit column", + help='Used if amount type is "Distinct Credit/debit Column"', + ) + credit_column = fields.Char( + string="Credit column", + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) debit_credit_column = fields.Char( string="Debit/credit column", help=( - "Some statement formats use absolute amount value and indicate sign" + 'Used if amount type is "Absolute value"\n' + "Some statement formats use absolute amount value and indicate sign\n" "of the transaction by specifying if it was a debit or a credit one" ), ) @@ -123,6 +172,18 @@ class AccountStatementImportSheetMapping(models.Model): bank_account_column = fields.Char( help="Partner's bank account", ) + footer_lines_count = fields.Integer( + string="Footer lines number", + help="Set the Footer lines number." + "Used in some csv file that integrate meta data in" + "last lines.", + default="0", + ) + column_labels_row = fields.Integer( + string="Row number for column labels", + help="The number of line that contain column names.", + default="1", + ) _sql_constraints = [ ( diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index deee5fd02d..d8dd097f3e 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -36,21 +36,23 @@ class AccountStatementImportSheetParser(models.TransientModel): _description = "Bank Statement Import Sheet Parser" @api.model - def parse_header(self, data_file, encoding, csv_options): + def parse_header(self, data_file, encoding, csv_options, column_labels_row=1): try: workbook = xlrd.open_workbook( file_contents=data_file, encoding_override=encoding if encoding else None, ) sheet = workbook.sheet_by_index(0) - values = sheet.row_values(0) + values = sheet.row_values(column_labels_row - 1) return [str(value) for value in values] except xlrd.XLRDError: _logger.error("Pass this method") data = StringIO(data_file.decode(encoding or "utf-8")) csv_data = reader(data, **csv_options) - return list(next(csv_data)) + csv_data_lst = list(csv_data) + header = [value.strip() for value in csv_data_lst[column_labels_row - 1]] + return header @api.model def parse(self, data_file, mapping, filename): @@ -62,9 +64,12 @@ def parse(self, data_file, mapping, filename): if not lines: return currency_code, account_number, [{"transactions": []}] - lines = list(sorted(lines, key=lambda line: line["timestamp"])) - first_line = lines[0] - last_line = lines[-1] + if lines[0]["timestamp"] > lines[-1]["timestamp"]: + first_line = lines[-1] + last_line = lines[0] + else: + first_line = lines[0] + last_line = lines[-1] data = { "date": first_line["timestamp"].date(), "name": _("%(code)s: %(filename)s") @@ -118,6 +123,8 @@ def _get_column_indexes(self, header, column_name, mapping): return column_indexes def _get_column_names(self): + # NOTE no seria necesario debit_column y credit_column ya que tenemos + # los respectivos campos related return [ "timestamp_column", "currency_column", @@ -171,14 +178,25 @@ def _parse_lines(self, mapping, data_file, currency_code): header = False if not mapping.no_header: if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + header = [ + str(value).strip() + for value in csv_or_xlsx[1].row_values( + mapping.column_labels_row - 1 + ) + ] else: + for _i in range(mapping.column_labels_row - 1): + next(csv_or_xlsx) header = [value.strip() for value in next(csv_or_xlsx)] + + # NOTE no seria necesario debit_column y credit_column ya que tenemos los + # respectivos campos related for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping ) - return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + data = csv_or_xlsx, data_file + return self._parse_rows(mapping, currency_code, data, columns) def _get_values_from_column(self, values, columns, column_name): indexes = columns[column_name] @@ -195,25 +213,38 @@ def _get_values_from_column(self, values, columns, column_name): return " ".join(content_l) return content_l[0] - def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 + def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 + csv_or_xlsx, data_file = data + + # Get the numbers of rows of the file + if isinstance(csv_or_xlsx, tuple): + numrows = csv_or_xlsx[1].nrows + else: + numrows = len(str(data_file.strip()).split("\\n")) + + label_line = mapping.column_labels_row + footer_line = numrows - mapping.footer_lines_count + if isinstance(csv_or_xlsx, tuple): - rows = range(1, csv_or_xlsx[1].nrows) + rows = range(mapping.column_labels_row, footer_line) else: rows = csv_or_xlsx lines = [] - for row in rows: + for index, row in enumerate(rows, label_line): if isinstance(csv_or_xlsx, tuple): book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(sheet.row_len(row)): + for col_index in range(0, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: cell_value = xldate_as_datetime(cell_value, book.datemode) values.append(cell_value) else: + if index >= footer_line: + continue values = list(row) timestamp = self._get_values_from_column( @@ -307,7 +338,7 @@ def _decimal(column_name): else: balance = None - if debit_credit: + if debit_credit is not None: amount = amount.copy_abs() if debit_credit == mapping.debit_value: amount = -amount @@ -429,6 +460,7 @@ def _parse_decimal(self, value, mapping): return value elif isinstance(value, float): return Decimal(value) + value = value or "0" thousands, decimal = mapping._get_float_separators() value = value.replace(thousands, "") value = value.replace(decimal, ".") diff --git a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst index d18dd9a9bb..cd949cea66 100644 --- a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst +++ b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst @@ -1,5 +1,6 @@ * Alexis de Lattre * Sebastien BEAU +* Mourad EL HADJ MIMOUNE * Tecnativa (https://www.tecnativa.com) * Vicent Cubells diff --git a/account_statement_import_txt_xlsx/security/ir.model.access.csv b/account_statement_import_txt_xlsx/security/ir.model.access.csv index aa8c3d9555..be92c3a142 100644 --- a/account_statement_import_txt_xlsx/security/ir.model.access.csv +++ b/account_statement_import_txt_xlsx/security/ir.model.access.csv @@ -2,4 +2,3 @@ access_account_statement_import_sheet_mapping_manager,account.statement.import.sheet.mapping:account.group_account_manager,model_account_statement_import_sheet_mapping,account.group_account_manager,1,1,1,1 access_account_statement_import_sheet_mapping_user,account.statement.import.sheet.mapping:account.group_account_user,model_account_statement_import_sheet_mapping,account.group_account_user,1,0,0,0 access_account_statement_import_sheet_parser,account.statement.import.sheet.parser:account.group_account_user,model_account_statement_import_sheet_parser,account.group_account_user,1,1,1,1 -access_account_statement_import_sheet_mapping_wizard,Full access on account.statement.import.sheet.mapping.wizard,model_account_statement_import_sheet_mapping_wizard,account.group_account_user,1,1,1,1 diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv new file mode 100644 index 0000000000..0b38621283 --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv @@ -0,0 +1,10 @@ +Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,, +Account Number : 08088804068,Account Name : Account Owner,: EUR,,, +,,,,, +Balance at end of period,,,,"+31070,11", +Date,Operation Number,Label,Debit,Credit,Detail +01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1 +01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X +02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2 +02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3 +Balance at start of period,,,,"+30000,77", diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx new file mode 100644 index 0000000000..c7ae2b92e9 Binary files /dev/null and b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx differ diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 563d9058b6..c104168de3 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -8,6 +8,7 @@ from odoo import fields from odoo.exceptions import UserError from odoo.tests import common +from odoo.tools import float_round class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): @@ -36,16 +37,12 @@ def setUp(self): self.AccountStatementImportSheetMapping = self.env[ "account.statement.import.sheet.mapping" ] - self.AccountStatementImportSheetMappingWizard = self.env[ - "account.statement.import.sheet.mapping.wizard" - ] + self.AccountStatementImportWizard = self.env["account.statement.import"] self.suspense_account = self.env["account.account"].create( { "code": "987654", "name": "Suspense Account", - "user_type_id": self.env.ref( - "account.data_account_type_current_assets" - ).id, + "account_type": "asset_current", } ) @@ -157,52 +154,6 @@ def test_import_empty_xlsx_file(self): statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 0) - def test_mapping_import_wizard_xlsx(self): - with common.Form(self.AccountStatementImportSheetMappingWizard) as form: - attachment = self.env["ir.attachment"].create( - { - "name": "fixtures/empty_statement_en.xlsx", - "datas": self._data_file("fixtures/empty_statement_en.xlsx"), - } - ) - form.attachment_ids.add(attachment) - self.assertEqual(len(form.header), 90) - self.assertEqual( - len( - self.AccountStatementImportSheetMappingWizard.with_context( - header=form.header, - ).statement_columns() - ), - 7, - ) - form.timestamp_column = "Date" - form.amount_column = "Amount" - wizard = form.save() - wizard.import_mapping() - - def test_mapping_import_wizard_csv(self): - with common.Form(self.AccountStatementImportSheetMappingWizard) as form: - attachment = self.env["ir.attachment"].create( - { - "name": "fixtures/empty_statement_en.csv", - "datas": self._data_file("fixtures/empty_statement_en.csv"), - } - ) - form.attachment_ids.add(attachment) - self.assertEqual(len(form.header), 90) - self.assertEqual( - len( - self.AccountStatementImportSheetMappingWizard.with_context( - header=form.header, - ).statement_columns() - ), - 7, - ) - form.timestamp_column = "Date" - form.amount_column = "Amount" - wizard = form.save() - wizard.import_mapping() - def test_original_currency(self): journal = self.AccountJournal.create( { @@ -232,7 +183,8 @@ def test_original_currency(self): self.assertEqual(line.currency_id, self.currency_usd) self.assertEqual(line.amount, 1525.0) self.assertEqual(line.foreign_currency_id, self.currency_eur) - self.assertEqual(line.amount_currency, 1000.0) + line_amount_currency = float_round(line.amount_currency, precision_digits=1) + self.assertEqual(line_amount_currency, 1000.0) def test_original_currency_no_header(self): no_header_statement_map = self.AccountStatementImportSheetMapping.create( @@ -240,6 +192,7 @@ def test_original_currency_no_header(self): "name": "Sample Statement", "float_thousands_sep": "comma", "float_decimal_sep": "dot", + "column_labels_row": 0, "delimiter": "comma", "quotechar": '"', "timestamp_format": "%m/%d/%Y", @@ -458,3 +411,97 @@ def test_debit_credit_amount(self): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_metadata_separated_debit_credit_csv(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) + + def test_metadata_separated_debit_credit_xlsx(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.xlsx", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import.xml b/account_statement_import_txt_xlsx/views/account_statement_import.xml index 4ef359aa77..f9697c3ca8 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import.xml @@ -10,7 +10,7 @@ account.statement.import diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 62d617dd0b..a7f4f6fa61 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -63,10 +63,14 @@ attrs="{'required': [('debit_credit_column', '!=', False)]}" /> + + + + - -