diff --git a/account_statement_import_txt_xlsx/__init__.py b/account_statement_import_txt_xlsx/__init__.py index aee8895e7..0650744f6 100644 --- a/account_statement_import_txt_xlsx/__init__.py +++ b/account_statement_import_txt_xlsx/__init__.py @@ -1,2 +1 @@ from . import models -from . import wizards diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index 6eab5d7bd..34446897e 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -13,9 +13,7 @@ "license": "AGPL-3", "installable": True, "depends": [ - "account_statement_import", - "multi_step_wizard", - "web_widget_dropdown_dynamic", + "account_statement_import_file", ], "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ @@ -24,6 +22,5 @@ "views/account_statement_import_sheet_mapping.xml", "views/account_statement_import.xml", "views/account_journal_views.xml", - "wizards/account_statement_import_sheet_mapping_wizard.xml", ], } diff --git a/account_statement_import_txt_xlsx/data/map_data.xml b/account_statement_import_txt_xlsx/data/map_data.xml index f87466015..29f48cdd8 100644 --- a/account_statement_import_txt_xlsx/data/map_data.xml +++ b/account_statement_import_txt_xlsx/data/map_data.xml @@ -7,12 +7,15 @@ Sample Statement + 0 + 1 comma dot comma " %m/%d/%Y Date + simple_value Amount Currency Amount Currency diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index caefa7ac1..e11819289 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -70,14 +70,30 @@ class AccountStatementImportSheetMapping(models.Model): amount_column = fields.Char( help="Amount of transaction in journal's currency", ) - amount_debit_column = fields.Char( + + debit_column = fields.Char( string="Debit amount column", help="Debit amount of transaction in journal's currency", ) - amount_credit_column = fields.Char( + credit_column = fields.Char( string="Credit amount column", help="Credit amount of transaction in journal's currency", ) + + # TODO to avoid error in un customer, need to fix using migrate script + amount_debit_column = fields.Char( + related="debit_column", + store=True, + string="OCA Debit Column", + readonly=False, + ) + amount_credit_column = fields.Char( + related="credit_column", + store=True, + string="OCA Credit Column", + readonly=False, + ) + balance_column = fields.Char( help="Balance after transaction in journal's currency", ) @@ -95,10 +111,43 @@ class AccountStatementImportSheetMapping(models.Model): "transaction amount in original transaction currency from" ), ) + amount_type = fields.Selection( + selection=[ + ("simple_value", "Simple value"), + ("absolute_value", "Absolute value"), + ("distinct_credit_debit", "Distinct Credit/debit Column"), + ], + string="Amount type", + required=True, + default="simple_value", + help=( + "Simple value: use igned amount in ammount comlumn\n" + "Absolute Value: use a same comlumn for debit and credit\n" + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct comlumn for debit and credit" + ), + ) + amount_column = fields.Char( + string="Amount column", + help=( + 'Used if amount type is "Simple value" or "Absolute value"\n' + "Amount of transaction in journal's currency\n" + "Some statement formats use credit/debit columns" + ), + ) + debit_column = fields.Char( + string="Debit column", + help='Used if amount type is "Distinct Credit/debit Column"', + ) + credit_column = fields.Char( + string="Credit column", + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) debit_credit_column = fields.Char( string="Debit/credit column", help=( - "Some statement formats use absolute amount value and indicate sign" + 'Used if amount type is "Absolute value"\n' + "Some statement formats use absolute amount value and indicate sign\n" "of the transaction by specifying if it was a debit or a credit one" ), ) @@ -123,6 +172,18 @@ class AccountStatementImportSheetMapping(models.Model): bank_account_column = fields.Char( help="Partner's bank account", ) + footer_lines_count = fields.Integer( + string="Footer lines number", + help="Set the Footer lines number." + "Used in some csv file that integrate meta data in" + "last lines.", + default="0", + ) + column_labels_row = fields.Integer( + string="Row number for column labels", + help="The number of line that contain column names.", + default="1", + ) _sql_constraints = [ ( diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index deee5fd02..0fbf3dc3e 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -36,21 +36,23 @@ class AccountStatementImportSheetParser(models.TransientModel): _description = "Bank Statement Import Sheet Parser" @api.model - def parse_header(self, data_file, encoding, csv_options): + def parse_header(self, data_file, encoding, csv_options, column_labels_row=1): try: workbook = xlrd.open_workbook( file_contents=data_file, encoding_override=encoding if encoding else None, ) sheet = workbook.sheet_by_index(0) - values = sheet.row_values(0) + values = sheet.row_values(column_labels_row - 1) return [str(value) for value in values] except xlrd.XLRDError: _logger.error("Pass this method") data = StringIO(data_file.decode(encoding or "utf-8")) csv_data = reader(data, **csv_options) - return list(next(csv_data)) + csv_data_lst = list(csv_data) + header = [value.strip() for value in csv_data_lst[column_labels_row - 1]] + return header @api.model def parse(self, data_file, mapping, filename): @@ -62,9 +64,12 @@ def parse(self, data_file, mapping, filename): if not lines: return currency_code, account_number, [{"transactions": []}] - lines = list(sorted(lines, key=lambda line: line["timestamp"])) - first_line = lines[0] - last_line = lines[-1] + if lines[0]["timestamp"] > lines[-1]["timestamp"]: + first_line = lines[-1] + last_line = lines[0] + else: + first_line = lines[0] + last_line = lines[-1] data = { "date": first_line["timestamp"].date(), "name": _("%(code)s: %(filename)s") @@ -118,6 +123,8 @@ def _get_column_indexes(self, header, column_name, mapping): return column_indexes def _get_column_names(self): + # NOTE no seria necesario debit_column y credit_column ya que tenemos + # los respectivos campos related return [ "timestamp_column", "currency_column", @@ -171,14 +178,25 @@ def _parse_lines(self, mapping, data_file, currency_code): header = False if not mapping.no_header: if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + header = [ + str(value).strip() + for value in csv_or_xlsx[1].row_values( + mapping.column_labels_row - 1 + ) + ] else: + for _i in range(mapping.column_labels_row - 1): + next(csv_or_xlsx) header = [value.strip() for value in next(csv_or_xlsx)] + + # NOTE no seria necesario [" y credit_column ya que tenemos los + # respectivos campos related for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping ) - return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + data = csv_or_xlsx, data_file + return self._parse_rows(mapping, currency_code, data, columns) def _get_values_from_column(self, values, columns, column_name): indexes = columns[column_name] @@ -195,25 +213,38 @@ def _get_values_from_column(self, values, columns, column_name): return " ".join(content_l) return content_l[0] - def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 + def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 + csv_or_xlsx, data_file = data + + # Get the numbers of rows of the file + if isinstance(csv_or_xlsx, tuple): + numrows = csv_or_xlsx[1].nrows + else: + numrows = len(str(data_file.strip()).split("\\n")) + + label_line = mapping.column_labels_row + footer_line = numrows - mapping.footer_lines_count + if isinstance(csv_or_xlsx, tuple): - rows = range(1, csv_or_xlsx[1].nrows) + rows = range(mapping.column_labels_row, footer_line) else: rows = csv_or_xlsx lines = [] - for row in rows: + for index, row in enumerate(rows, label_line): if isinstance(csv_or_xlsx, tuple): book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(sheet.row_len(row)): + for col_index in range(0, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: cell_value = xldate_as_datetime(cell_value, book.datemode) values.append(cell_value) else: + if index >= footer_line: + continue values = list(row) timestamp = self._get_values_from_column( @@ -238,6 +269,11 @@ def _decimal(column_name): if not amount: amount = -abs(_decimal("amount_credit_column") or 0) + # amount_column = columns["amount_column"] + # if amount_column and values[columns["amount_column"]]: + # amount = self._parse_decimal( + # values[columns["amount_column"], mapping) + balance = ( self._get_values_from_column(values, columns, "balance_column") if columns["balance_column"] @@ -296,6 +332,17 @@ def _decimal(column_name): else None ) + debit_column = ( + self._get_values_from_column(values, columns, "amount_debit_column") + if columns["amount_debit_column"] + else None + ) + credit_column = ( + self._get_values_from_column(values, columns, "amount_credit_column") + if columns["amount_credit_column"] + else None + ) + if currency != currency_code: continue @@ -307,11 +354,17 @@ def _decimal(column_name): else: balance = None - if debit_credit: + if debit_credit is not None: amount = amount.copy_abs() if debit_credit == mapping.debit_value: amount = -amount + if debit_column and credit_column: + debit_amount = self._parse_decimal(values[debit_column], mapping) + debit_amount = debit_amount.copy_abs() + credit_amount = self._parse_decimal(values[credit_column], mapping) + amount = credit_amount - debit_amount + if original_amount: original_amount = self._parse_decimal( original_amount, mapping @@ -429,6 +482,7 @@ def _parse_decimal(self, value, mapping): return value elif isinstance(value, float): return Decimal(value) + value = value or "0" thousands, decimal = mapping._get_float_separators() value = value.replace(thousands, "") value = value.replace(decimal, ".") diff --git a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst index b5e09af68..6b9085d24 100644 --- a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst +++ b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst @@ -1,5 +1,6 @@ * Alexis de Lattre * Sebastien BEAU +* Mourad EL HADJ MIMOUNE * Tecnativa (https://www.tecnativa.com) * Vicent Cubells diff --git a/account_statement_import_txt_xlsx/security/ir.model.access.csv b/account_statement_import_txt_xlsx/security/ir.model.access.csv index aa8c3d955..be92c3a14 100644 --- a/account_statement_import_txt_xlsx/security/ir.model.access.csv +++ b/account_statement_import_txt_xlsx/security/ir.model.access.csv @@ -2,4 +2,3 @@ access_account_statement_import_sheet_mapping_manager,account.statement.import.sheet.mapping:account.group_account_manager,model_account_statement_import_sheet_mapping,account.group_account_manager,1,1,1,1 access_account_statement_import_sheet_mapping_user,account.statement.import.sheet.mapping:account.group_account_user,model_account_statement_import_sheet_mapping,account.group_account_user,1,0,0,0 access_account_statement_import_sheet_parser,account.statement.import.sheet.parser:account.group_account_user,model_account_statement_import_sheet_parser,account.group_account_user,1,1,1,1 -access_account_statement_import_sheet_mapping_wizard,Full access on account.statement.import.sheet.mapping.wizard,model_account_statement_import_sheet_mapping_wizard,account.group_account_user,1,1,1,1 diff --git a/account_statement_import_txt_xlsx/tests/fixtures/.~lock.meta_data_separated_credit_debit.csv# b/account_statement_import_txt_xlsx/tests/fixtures/.~lock.meta_data_separated_credit_debit.csv# new file mode 100644 index 000000000..c89db3b2c --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/.~lock.meta_data_separated_credit_debit.csv# @@ -0,0 +1 @@ +,pablo,pam-nb2,19.04.2023 15:40,file:///home/pablo/.config/libreoffice/4; diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv new file mode 100644 index 000000000..0b3862128 --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv @@ -0,0 +1,10 @@ +Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,, +Account Number : 08088804068,Account Name : Account Owner,: EUR,,, +,,,,, +Balance at end of period,,,,"+31070,11", +Date,Operation Number,Label,Debit,Credit,Detail +01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1 +01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X +02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2 +02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3 +Balance at start of period,,,,"+30000,77", diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx new file mode 100644 index 000000000..c7ae2b92e Binary files /dev/null and b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx differ diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 71595b872..620429ee4 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -8,6 +8,7 @@ from odoo import fields from odoo.exceptions import UserError from odoo.tests import common +from odoo.tools import float_round class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): @@ -35,9 +36,7 @@ def setUp(self): { "code": "987654", "name": "Suspense Account", - "user_type_id": self.env.ref( - "account.data_account_type_current_assets" - ).id, + "account_type": "asset_current", } ) @@ -224,7 +223,8 @@ def test_original_currency(self): self.assertEqual(line.currency_id, self.currency_usd) self.assertEqual(line.amount, 1525.0) self.assertEqual(line.foreign_currency_id, self.currency_eur) - self.assertEqual(line.amount_currency, 1000.0) + line_amount_currency = float_round(line.amount_currency, precision_digits=1) + self.assertEqual(line_amount_currency, 1000.0) def test_original_currency_no_header(self): no_header_statement_map = self.AccountStatementImportSheetMapping.create( @@ -450,3 +450,97 @@ def test_debit_credit_amount(self): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_metadata_separated_debit_credit_csv(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) + + def test_metadata_separated_debit_credit_xlsx(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.xlsx", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, 50) + self.assertEqual(line4.amount, -1300) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import.xml b/account_statement_import_txt_xlsx/views/account_statement_import.xml index 4ef359aa7..f9697c3ca 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import.xml @@ -10,7 +10,7 @@ account.statement.import diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 62d617dd0..d2fc88968 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -63,9 +63,13 @@ attrs="{'required': [('debit_credit_column', '!=', False)]}" /> + + + + - +