From 240edcef4864dfdf81a8068c14f6f87694a0b6cb Mon Sep 17 00:00:00 2001 From: PicchiSeba Date: Tue, 30 Apr 2024 16:00:12 +0200 Subject: [PATCH] [IMP]account_statement_import_txt_xlsx: additional mapping controls --- .../README.rst | 2 + .../account_statement_import_sheet_mapping.py | 18 ++++ .../account_statement_import_sheet_parser.py | 12 ++- .../readme/CONTRIBUTORS.rst | 2 + .../static/description/index.html | 12 ++- .../tests/fixtures/empty_lines_statement.csv | 5 + .../fixtures/sample_statement_offsets.xlsx | Bin 0 -> 5993 bytes ...est_account_statement_import_sheet_file.py | 93 +++++++++++++++++- ...account_statement_import_sheet_mapping.xml | 9 ++ 9 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv create mode 100644 account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx diff --git a/account_statement_import_sheet_file/README.rst b/account_statement_import_sheet_file/README.rst index b661fdcfcb..864095c0e2 100644 --- a/account_statement_import_sheet_file/README.rst +++ b/account_statement_import_sheet_file/README.rst @@ -103,6 +103,8 @@ Contributors * Alexey Pelykh +* Sebastiano Picchi + Maintainers ~~~~~~~~~~~ diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py index 2db125538f..34f996b450 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py @@ -66,6 +66,18 @@ class AccountStatementImportSheetMapping(models.Model): help="When this occurs please indicate the column number in the Columns section " "instead of the column name, considering that the first column is 0", ) + skip_empty_lines = fields.Boolean( + default=False, + help="Allows to skip empty lines", + ) + offset_column = fields.Integer( + default=0, + help="Horizontal spaces to ignore before starting to parse", + ) + offset_row = fields.Integer( + default=0, + help="Vertical spaces to ignore before starting to parse", + ) timestamp_column = fields.Char(required=True) currency_column = fields.Char( help=( @@ -217,6 +229,12 @@ def onchange_decimal_separator(self): elif "comma" == self.float_thousands_sep == self.float_decimal_sep: self.float_thousands_sep = "dot" + @api.constrains("offset_column", "offset_row") + def _check_columns(self): + for mapping in self: + if mapping.offset_column < 0 or mapping.offset_row < 0: + raise ValidationError(_("Offsets cannot be negative")) + def _get_float_separators(self): self.ensure_one() separators = { diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py index 232008663f..cdb5de91df 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py @@ -179,7 +179,7 @@ def _parse_lines(self, mapping, data_file, currency_code): csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) header = False if not mapping.no_header: - header_line = mapping.header_lines_skip_count - 1 + header_line = mapping.offset_row + mapping.header_lines_skip_count - 1 if isinstance(csv_or_xlsx, tuple): header = [ str(value).strip() @@ -188,6 +188,8 @@ def _parse_lines(self, mapping, data_file, currency_code): else: [next(csv_or_xlsx) for _i in range(header_line)] header = [value.strip() for value in next(csv_or_xlsx)] + if mapping.offset_column: + header = header[mapping.offset_column :] # NOTE no seria necesario debit_column y credit_column ya que tenemos los # respectivos campos related @@ -349,11 +351,11 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 else: numrows = len(str(data_file.strip()).split("\\n")) - label_line = mapping.header_lines_skip_count + label_line = mapping.header_lines_skip_count + mapping.offset_row footer_line = numrows - mapping.footer_lines_skip_count if isinstance(csv_or_xlsx, tuple): - rows = range(mapping.header_lines_skip_count, footer_line) + rows = range(label_line, footer_line) else: rows = csv_or_xlsx @@ -363,7 +365,7 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(0, sheet.row_len(row)): + for col_index in range(mapping.offset_column, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: @@ -373,6 +375,8 @@ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 if index >= footer_line: continue values = list(row) + if mapping.skip_empty_lines and not any(values): + continue line = self._parse_row(mapping, currency_code, values, columns) if line: lines.append(line) diff --git a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst index 36fc13e148..103db48276 100644 --- a/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst +++ b/account_statement_import_sheet_file/readme/CONTRIBUTORS.rst @@ -15,3 +15,5 @@ * `CorporateHub `__ * Alexey Pelykh + +* Sebastiano Picchi diff --git a/account_statement_import_sheet_file/static/description/index.html b/account_statement_import_sheet_file/static/description/index.html index b924b868ee..2b7c93f9f5 100644 --- a/account_statement_import_sheet_file/static/description/index.html +++ b/account_statement_import_sheet_file/static/description/index.html @@ -8,10 +8,11 @@ /* :Author: David Goodger (goodger@python.org) -:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $ +:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $ :Copyright: This stylesheet has been placed in the public domain. Default cascading style sheet for the HTML output of Docutils. +Despite the name, some widely supported CSS2 features are used. See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to customize this style sheet. @@ -274,7 +275,7 @@ margin-left: 2em ; margin-right: 2em } -pre.code .ln { color: grey; } /* line numbers */ +pre.code .ln { color: gray; } /* line numbers */ pre.code, code { background-color: #eeeeee } pre.code .comment, code .comment { color: #5C6576 } pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold } @@ -300,7 +301,7 @@ span.pre { white-space: pre } -span.problematic { +span.problematic, pre.problematic { color: red } span.section-subtitle { @@ -455,12 +456,15 @@

Contributors

  • Alexey Pelykh <alexey.pelykh@corphub.eu>
  • +
  • Sebastiano Picchi <sebastiano.picchi@pytech.it>
  • Maintainers

    This module is maintained by the OCA.

    -Odoo Community Association + +Odoo Community Association +

    OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

    diff --git a/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv b/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv new file mode 100644 index 0000000000..c5b7aded45 --- /dev/null +++ b/account_statement_import_sheet_file/tests/fixtures/empty_lines_statement.csv @@ -0,0 +1,5 @@ +"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account" +"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789" +"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior","" +,,,,,, +"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789" diff --git a/account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx b/account_statement_import_sheet_file/tests/fixtures/sample_statement_offsets.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..2cfc77a8a353e0e76434c0df93455f8ba8c0b17a GIT binary patch literal 5993 zcmaJ_1yq#l)}|W)X$GXbrArzCDd`dzVTNXi0i-(wq>+}EkQPC@r8|chQfW8{QW6)= zx$Bqf{jYo0np$h`@B5zpygMF^XUHf-2pAX`2u$*Y`Uv-o;`aM1J8n~Fkev&+JH*-c zB?My2^~&Ck>+YX2O=+7>9{i>a;+F@XcJqTZIW`_*)xY0D4sNPdu~V`Vvj6_GMAeEC zBL_XJYtY6r6gV*DwAsz9u-&AkVIhJ6{W6njOYFT1e`jK*Gt|#{k~TqF{emXWjgqx; za79P?F$ubAD&FRJ{-M)ohM65`6kK7HcqDGxFFftPt)OLRgX*M3O5;}Oe=~NXIzevi zJO`4D$%~>62`id!%<8?q&7VrrOZB;75=&Es>e82X|#T;i%O3`xrK^p$Dp!@i}RF_#xrz`mQhjiL1YS+)v8O)XL71yZe3ZCoUoj2beNwJLk5csT&225JH3;yBYF%HhMZa&>? zuIBxKpDvF-+Z32KAK)PMJb&{2H^uib)LPy)Y$HT7&LAL}T2LBxw`g$4IW^9Uw{&4| zhVcYgcBOC>qr2d`xm9+bI6{I8%V}eThAd48_kss9P(SF+nZ8Lr^!BsT?2k6+D?qiW z88769x%bA2u4+E6SjHF=r#xl#ld{TJVUr!6K4e(7@ar*}poLr0o7Bi~)zyz_e!&@9 zF?@bm1)ootRWEP=vFwP9N*tWY9di+Hm1 zkZk{YZiNR`VL5ItX{}N`iCLRLbXd;^`lQa4NKvYoYHTlEMfALoABx0L1_LKMI>;pr zp&L3%niE&t0!ne@o5ySUT}?p*ygv_pMKhQ_ZjDlx=fzRW*Ad=yT8SS{%<~ttGp1X- zyJj#DnMA#ck8Y8dUFt(nAByq#$3 zU@B6MH6g^8xn=NiyOY*;=8(B5Iy6S%tOiaw5bIdBceE*z;>-qvq>{M&Y(mhpk889? zX+HQ~L1q5O_?f8WgO?{1ZFv!BQE(|9F#;ywIKlNT(6{y8ezhmsIhjU}8dCuCq~V_#DPr2! z2cj*wP)8vsDKS!V*JcNcUl6gbppr!n3mrlEIS34S0p7s}>K5p8Esy&4vgE5xR+Tq} z47&xKrX94c(ro}T{h4CQ-jz9uM#^yx&3t5tjZyMCXNmM zfO;3X#q}l*rs(n-{iwm_se4CzHzKW99r_!P0W|89kCs=2cl0``JPdqvTaLys{zNa_ zf9T}`^{@lE+!1R}-w2Y{@1=I5{(E^gi5{f^4kHg623ECW6J+C}lBm_kQ z4btd_pMxaJ9|^Rq@+7>pg`i2`hh02aFax^=8a{Ga{aF~{X-@tvQRh*HC=F}nxFEm! z2lPWtF$^>>*$MzDA*eLc8tc}DQ81LHs)lORxT0kI3+Hae_s6b_+$Am{ANrth`pGv` zr^+Rh!z*s{5wtwbEG zDi&VMt`K(2h8W?Se%zZt*M%lXWVGe(&$@nV5=u5H-de7v_$eYebgwHS{Zzhfz zJ33eVWvnFoP^)5tB3gEVBv6LU)3RWZG(~8qhtp4%5dC~3q%HQ%IUVi^+w??P$JhSA z7vb9ZKiW>-aS;*Eu-E4=+5@y;Fa#B*83ub*68=w|BC2jf#fKh}MNbyg zOES^8dj_HWN5QF?Gt*XJe!evFp(wB$FQ3>#q0m>mYCQ+A*H>_Zx8_Pn7d83RYBele!r;35yP<6-Xx zp>CuTjb*hFx-(cNXSIo)o^3x4aZA{vhrTnm3Hecib+w7@w7Kf-^?FUwTQ*)H*&IOIhD} zeYI!p&&^Cq6MnwK>)k}t1kjk5 zHaJ`&xtrXPba^r&U#^L*{RCQic95DBt8I3Y^ksA%rGT>2Fkt$OAQBQ{NUDDP$lnu| z8lAcQZA+MG6we^F+wpM~cXK3&g%D%B<$_yM(1d zY!;ZY(bW`7GTnq?$!_nZl(17@hE8br*OV!K@kP9rHZy`Gf@27j+Zje&jb`e$-3XK1 z;MsbHY(^zI=N4rwK5t{!T|o(nSk{n$buE1zpb^F>=k(NIGY~L(YIqs89d2wV%2YhR zwDXN)@?8K}&%_^ZNr^*{@Mn_{Sv1;4XXPPgLv`1<3(NdLdqo!jny0{aQ!IFa264hV zEO{r0c4?U966-69rd5BzzNh-U9fZt2UbG$v)_Kt&4|v!kESaYJb#N408#9YRFUz>8 zgd_l#_T|Vkhx_Whne~dTz>lFoa`(-(RDZTIOfs{-Qkndb7Wpw+tK?dCzp!v-FEGh@ zHvjkrsyy<;)^@~?X>v?G($|(Y;JvwmJBe16Xq|6m8uX{-bdl`Sn{`^;r=YFe^GznK;cyw!I49-ADPqCn@ws3js}8uV3(^9op_Y~HbyHkuNB<` zV?4E;iw!B_X}yrIhQ^W@3ZY}tPnSn|8URiXWQ)d;-&kIUJ(SNjvX2Egb& zf|7SIylpb!Q@vWvbfH45Nj{EZ07%0^X!Odh=I&LR1sN(;s2S$o&y9d_PV* zW8x!X)Om=VcSx98(9?1*573)J<6^eti8s!-*PM5Etr_yTzRi#oFu%R{$Y<#;yw5w}<1AC2#FY-7_ZY=V0M+Cq_ujk2j+uxsB^qUANiMax^C3-!J zkrUY_3q)mZ{P8@IUy#&?%k`bpV!xYT%V5r4%|BsRMGWZ5tMw495NL_?n6Oa)u}$|H zHy>Zbb19YKn+6fGPL$@8ZiWwi7apnIi;X1R<6jL}(MUVT&YYc-4mx*<c6~s1{1I zzRO%jpsYSXj&(ZL%N-XgYDK&QSWHx0iu!hv8;uGBPiI*nv+( zte1N&v7E)n=J3&UMIZl^!E2yUDi67?eVQ~R^jFRuskdSQJ;zRI{h>WKZ3Xl>* z_NeGfGOxU>2d6uLW?Q`>`e9vV20ZirlF2T+W14se#A*P~tnu>>eG}1DvjWQcFpEVC zuVDKZrpPxIn+vlu5iB6)8qN?$7jAQiGw5y}_0b$qyHgL2h~o}k@6{Xw zKNX`CF|*NrA`VY?sx78<;+Xfjl29AZM`d#m6I+8NGk-LXL-*p*eQOh8C8vR)oLH<4 zBP!E7atub=`K;_=!j_UtsL%gwpGk)AImIj90<<-T*){;{K=8ZVGM6-z@cu^5rm##97n|H!UZQ|^3VnsIm-cU=vz#7B9Yc|hw?jZ zL?_>Z(&{xy#xwaB5^F6ApBTb0@xo-Gn^T%glHHKV4UmKli_RxDM3&SH0I&nWJr)h_ zFYwjy+o#jBnzTFyCMe~&aDEUm72o5R48B>*rU&euOkAp(Riw*nM5R2MVt$>WmvMgy zZ?MguhcIV%{HT5lE#6KPsdHF~X^D${lF`)j#(4g@)9{s;<^o=yk76jhvVPX%xkid& zfg*Om?7;Q`V`qaE`iJtp{WZd1TZL`^m`tb$|CZq85p@FOj>0+p*G;y!6rTTMrtt4) z3h3x~m#ECxc889;jnpfGZFP<*v0R1)caDEAxQ0r!#wdxA%fcq1$mzMqPt;L=z9+EO zvXPUK8LcHobh1FFTDqgCIFwT7x!D);E!(G1rFj2w8<+?-v4{+f?O~TB`UX%6nf5gb zfUWxJF}rfmoGiC2$-JPOq0sR|bCKO5Tcf&eTT+ttTH1+s;=-xyn znWBrNwLiw5zFQVJZb`rhoG>yJOMGGPi4P+9VcTumbPyF!#z_D;;%n>iKp%?a7}2jy zv5%rE{N`mK2#Q-!~bhphL$wC~UAP1)rYk~F;8DX>65^7izWL`Y>tK&sgtzj=_&aK7-`zT8j42dmbe<&eE*wX6g z#3uL2da5dPnmjR}aK!w^rWZTsXK-2Rz5}uF%Hg0fW%q zlrDDbyb@eQSZII8m0MS1z+Nnw)F5H)yfR{6ios2%W6dT zeP{YDYC|f-=r3!>o_=Ziht%6rLmc6jkV7F!FZ${F)ALQVYspZ`%;++~2YPWKYXeu) z*qf6y1p3^kemhBYnpq2`7z6VDXJv~6)(eB3>UW9iH%1h;x=m6q>RqA`k%$m}iJJFi zygO0zKka`@oWIApFYVlke!q<6_R6<$?n{8bJKta2@A&e|jBoqOt@H0%(C-26FF<$u z{V!|0edgN$zqb9~UGI+wcN@|#tGSgC{(a;BwJ-f1<$fQ$8z+9*(Cz))-N5g|#qSaB zH;lV}`OCO&@x0ePe|NlJzwY|UFME1hwEnBR{O*4L6yFuOUv_vK?f=~WUGjeSzE7?H e%pBY8&;RG#Ydk|mySoI|?aTkR;!gaH*1rH(yfWYb literal 0 HcmV?d00001 diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py index 4c767609b6..c0abcd3b10 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py @@ -1,7 +1,6 @@ # Copyright 2019 ForgeFlow, S.L. # Copyright 2020 CorporateHub (https://corporatehub.eu) # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - from base64 import b64encode from decimal import Decimal from os import path @@ -678,3 +677,95 @@ def test_decimal_and_float_inputs(self): self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot), 1234.56, ) + + def test_offsets(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + file_name = "fixtures/sample_statement_offsets.xlsx" + data = self._data_file(file_name) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + with self.assertRaises(UserError): + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement_map_offsets = self.sample_statement_map.copy( + { + "offset_column": 1, + "offset_row": 2, + } + ) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": statement_map_offsets.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 2) + self.assertEqual(statement.balance_start, 0.0) + self.assertEqual(statement.balance_end_real, 1491.5) + self.assertEqual(statement.balance_end, 1491.5) + + def test_skip_empty_lines(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + file_name = "fixtures/empty_lines_statement.csv" + data = self._data_file(file_name, "utf-8") + + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + with self.assertRaises(UserError): + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement_map_empty_line = self.sample_statement_map.copy( + { + "skip_empty_lines": True, + } + ) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": statement_map_empty_line.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 3) + self.assertEqual(statement.balance_start, 0.0) + self.assertEqual(statement.balance_end_real, 2291.5) + self.assertEqual(statement.balance_end, 2291.5) diff --git a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml index 49f34bc9ea..22cee0ec48 100644 --- a/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_sheet_file/views/account_statement_import_sheet_mapping.xml @@ -50,6 +50,15 @@ class="fa fa-info-circle" /> indicate the column number in the Columns section. The first column is 0.
    + + +