From 582db3798fc0657cb0b6d66414fd4153212bada1 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Sat, 30 Nov 2024 01:46:10 +0100 Subject: [PATCH] Upgrade backend --- .mise.toml | 2 +- apps/backend/gleam.toml | 15 +- apps/backend/manifest.toml | 61 +- apps/backend/src/api/github.gleam | 27 +- apps/backend/src/api/hex_repo.gleam | 8 +- apps/backend/src/{ => aws}/s3.gleam | 22 +- apps/backend/src/backend.gleam | 94 +-- apps/backend/src/backend/config.gleam | 76 -- apps/backend/src/backend/context.gleam | 57 ++ apps/backend/src/backend/error.gleam | 34 +- apps/backend/src/backend/gleam/context.gleam | 4 +- .../src/backend/gleam/generate/types.gleam | 107 +-- .../src/backend/gleam/type_search.gleam | 18 +- .../src/backend/gleam/type_search/state.gleam | 16 +- .../src/backend/postgres/postgres.gleam | 27 +- .../src/backend/postgres/queries.gleam | 685 ++++++++++-------- apps/backend/src/backend/router.gleam | 18 +- apps/backend/src/{ => backend}/setup.gleam | 4 +- apps/backend/src/backend/web.gleam | 24 +- apps/backend/src/backend/workers.gleam | 35 + .../{gloogle_hex_ffi.erl => backend_ffi.erl} | 36 +- apps/backend/src/gleam/verl.gleam | 2 +- apps/backend/src/helpers.gleam | 12 +- .../src/{ => processes}/periodic.gleam | 0 apps/backend/src/processes/retrier.gleam | 75 ++ apps/backend/src/retrier.gleam | 84 --- apps/backend/src/tasks/hex.gleam | 168 ++--- apps/backend/src/tasks/popularity.gleam | 6 +- apps/backend/src/tasks/ranking.gleam | 4 +- apps/backend/src/tasks/timeseries.gleam | 15 +- apps/backend/src/wisp/logger.gleam | 12 +- apps/backend/test/backend_test.gleam | 22 +- .../client/src/app.gleam | 2 +- 33 files changed, 935 insertions(+), 837 deletions(-) rename apps/backend/src/{ => aws}/s3.gleam (67%) delete mode 100644 apps/backend/src/backend/config.gleam create mode 100644 apps/backend/src/backend/context.gleam rename apps/backend/src/{ => backend}/setup.gleam (89%) create mode 100644 apps/backend/src/backend/workers.gleam rename apps/backend/src/{gloogle_hex_ffi.erl => backend_ffi.erl} (76%) rename apps/backend/src/{ => processes}/periodic.gleam (100%) create mode 100644 apps/backend/src/processes/retrier.gleam delete mode 100644 apps/backend/src/retrier.gleam diff --git a/.mise.toml b/.mise.toml index a9ab64a..8e48dfe 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,6 +1,6 @@ # Runtimes used by the stack. [tools] -gleam = "1.4.1" +gleam = "1.6.2" erlang = "27" node = "22" deno = "latest" diff --git a/apps/backend/gleam.toml b/apps/backend/gleam.toml index 3a3c8df..3d94365 100644 --- a/apps/backend/gleam.toml +++ b/apps/backend/gleam.toml @@ -2,10 +2,9 @@ name = "backend" version = "1.0.0" [dependencies] -aws4_request = ">= 0.1.1 and < 1.0.0" birl = "~> 1.6" chomp = ">= 0.1.0 and < 1.0.0" -cors_builder = ">= 2.0.0 and < 3.0.0" +cors_builder = ">= 2.0.2 and < 3.0.0" decipher = ">= 1.2.0 and < 2.0.0" gleam_erlang = "~> 0.25" gleam_hexpm = "~> 1.0" @@ -14,10 +13,8 @@ gleam_httpc = ">= 2.2.0 and < 3.0.0" gleam_json = "~> 1.0" gleam_otp = "~> 0.10" gleam_package_interface = ">= 1.0.0 and < 2.0.0" -gleam_pgo = {path = "../../packages/pgo"} -gleam_stdlib = "~> 0.34 or ~> 1.0" glexer = ">= 1.0.1 and < 2.0.0" -mist = ">= 1.2.0 and < 2.0.0" +mist = ">= 3.0.0 and < 4.0.0" pgo = "~> 0.14" prng = ">= 3.0.3 and < 4.0.0" radiate = ">= 0.4.0 and < 1.0.0" @@ -25,7 +22,13 @@ ranger = ">= 1.2.0 and < 2.0.0" simplifile = ">= 2.1.0 and < 3.0.0" tom = { path ="../../packages/tom" } verl = ">= 1.1.1 and < 2.0.0" -wisp = ">= 1.1.0 and < 2.0.0" +wisp = ">= 1.3.0 and < 2.0.0" +aws4_request = ">= 1.2.0 and < 2.0.0" +gleam_stdlib = ">= 0.44.0 and < 1.0.0" +pog = ">= 1.0.1 and < 2.0.0" +envoy = ">= 1.0.2 and < 2.0.0" +gleam_regexp = ">= 1.0.0 and < 2.0.0" +gleam_yielder = ">= 1.1.0 and < 2.0.0" [dev-dependencies] gleeunit = "~> 1.0" diff --git a/apps/backend/manifest.toml b/apps/backend/manifest.toml index fe50b78..9da8fe9 100644 --- a/apps/backend/manifest.toml +++ b/apps/backend/manifest.toml @@ -2,62 +2,65 @@ # You typically do not need to edit this file packages = [ - { name = "aws4_request", version = "0.1.1", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_http", "gleam_stdlib"], otp_app = "aws4_request", source = "hex", outer_checksum = "90B1DB6E2A7F0396CD4713850B14B3A910331B5BA76D051E411D1499AAA2EA9A" }, + { name = "aws4_request", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_http", "gleam_stdlib"], otp_app = "aws4_request", source = "hex", outer_checksum = "A73515D60E2D164B8B546778F9471ADE6089D33B2DC5D362C1B2CAF12FB6471B" }, { name = "backoff", version = "1.1.6", build_tools = ["rebar3"], requirements = [], otp_app = "backoff", source = "hex", outer_checksum = "CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39" }, { name = "birl", version = "1.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "ranger"], otp_app = "birl", source = "hex", outer_checksum = "5C66647D62BCB11FE327E7A6024907C4A17954EF22865FE0940B54A852446D01" }, - { name = "chomp", version = "0.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "chomp", source = "hex", outer_checksum = "C87304897B4D4DEA69420DB2FF88B087673AAE9EC09CA8A0FBF4675F605767C2" }, - { name = "cors_builder", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "E4634D1CAD89BC26B4FE986B45D55268BA5369E9224C2FD7872C0DB5AC028E47" }, - { name = "decipher", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_json", "gleam_stdlib", "stoiridh_version"], otp_app = "decipher", source = "hex", outer_checksum = "9F1B5C6FF0D798046E4E0EF87D09DD729324CB72BD7F0D4152B797324D51223E" }, + { name = "chomp", version = "0.4.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "chomp", source = "hex", outer_checksum = "6DDC9EE76C5364E80F43A90A238351C25F627E59135AEF2506629112A7C4FD4F" }, + { name = "cors_builder", version = "2.0.2", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib", "mist", "wisp"], otp_app = "cors_builder", source = "hex", outer_checksum = "8FB02BDB2F5E20B13C5A85B7B4F2A52E8C50A23F2AE798FA18A9CF2E2197CE7F" }, + { name = "decipher", version = "1.2.1", build_tools = ["gleam"], requirements = ["birl", "gleam_json", "gleam_stdlib", "stoiridh_version"], otp_app = "decipher", source = "hex", outer_checksum = "4F82516A5FF09BD7DF352DE38F1691C2254508066152F5DEA8665B216A9C9909" }, { name = "directories", version = "1.1.0", build_tools = ["gleam"], requirements = ["envoy", "gleam_stdlib", "platform", "simplifile"], otp_app = "directories", source = "hex", outer_checksum = "BDA521A4EB9EE3A7894F0DC863797878E91FF5C7826F7084B2E731E208BDB076" }, - { name = "envoy", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envoy", source = "hex", outer_checksum = "CFAACCCFC47654F7E8B75E614746ED924C65BD08B1DE21101548AC314A8B6A41" }, + { name = "envoy", version = "1.0.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "envoy", source = "hex", outer_checksum = "95FD059345AA982E89A0B6E2A3BF1CF43E17A7048DCD85B5B65D3B9E4E39D359" }, { name = "exception", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "exception", source = "hex", outer_checksum = "F5580D584F16A20B7FCDCABF9E9BE9A2C1F6AC4F9176FA6DD0B63E3B20D450AA" }, - { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" }, + { name = "filepath", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "67A6D15FB39EEB69DD31F8C145BB5A421790581BD6AA14B33D64D5A55DBD6587" }, { name = "filespy", version = "0.5.0", build_tools = ["gleam"], requirements = ["fs", "gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "filespy", source = "hex", outer_checksum = "F8E7A9C9CA86D68CCC25491125BFF36BEF7483892D7BEC24AA30D6B540504F06" }, { name = "fs", version = "8.6.1", build_tools = ["rebar3"], requirements = [], otp_app = "fs", source = "hex", outer_checksum = "61EA2BDAEDAE4E2024D0D25C63E44DCCF65622D4402DB4A2DF12868D1546503F" }, { name = "glam", version = "2.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "glam", source = "hex", outer_checksum = "66EC3BCD632E51EED029678F8DF419659C1E57B1A93D874C5131FE220DFAD2B2" }, { name = "gleam_bitwise", version = "1.3.1", build_tools = ["gleam"], requirements = [], otp_app = "gleam_bitwise", source = "hex", outer_checksum = "B36E1D3188D7F594C7FD4F43D0D2CE17561DE896202017548578B16FE1FE9EFC" }, - { name = "gleam_crypto", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "ADD058DEDE8F0341F1ADE3AAC492A224F15700829D9A3A3F9ADF370F875C51B7" }, - { name = "gleam_erlang", version = "0.25.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "054D571A7092D2A9727B3E5D183B7507DAB0DA41556EC9133606F09C15497373" }, + { name = "gleam_crypto", version = "1.4.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "8AE56026B3E05EBB1F076778478A762E9EB62B31AEEB4285755452F397029D22" }, + { name = "gleam_erlang", version = "0.32.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "B18643083A0117AC5CFD0C1AEEBE5469071895ECFA426DCC26517A07F6AD9948" }, { name = "gleam_hexpm", version = "1.1.0", build_tools = ["gleam"], requirements = ["birl", "gleam_stdlib"], otp_app = "gleam_hexpm", source = "hex", outer_checksum = "D32439FD6AD683FE1094922737904EC2091E2D7B1F236AD23815935694A5221A" }, - { name = "gleam_http", version = "3.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "8C07DF9DF8CC7F054C650839A51C30A7D3C26482AC241C899C1CEA86B22DBE51" }, - { name = "gleam_httpc", version = "2.2.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib"], otp_app = "gleam_httpc", source = "hex", outer_checksum = "CF76C71002DEECF6DC5D9CA83D962728FAE166B57926BE442D827004D3C7DF1B" }, + { name = "gleam_http", version = "3.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "A9EE0722106FCCAB8AD3BF9D0A3EFF92BFE8561D59B83BAE96EB0BE1938D4E0F" }, + { name = "gleam_httpc", version = "2.3.0", build_tools = ["gleam"], requirements = ["gleam_http", "gleam_stdlib"], otp_app = "gleam_httpc", source = "hex", outer_checksum = "CF6CDD88830CC9853F7638ECC0BE7D7CD9522640DA5FAB4C08CFAC8DEBD08028" }, { name = "gleam_json", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "thoas"], otp_app = "gleam_json", source = "hex", outer_checksum = "9063D14D25406326C0255BDA0021541E797D8A7A12573D849462CAFED459F6EB" }, - { name = "gleam_otp", version = "0.10.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "0B04FE915ACECE539B317F9652CAADBBC0F000184D586AAAF2D94C100945D72B" }, + { name = "gleam_otp", version = "0.14.1", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "5A8CE8DBD01C29403390A7BD5C0A63D26F865C83173CF9708E6E827E53159C65" }, { name = "gleam_package_interface", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_json", "gleam_stdlib"], otp_app = "gleam_package_interface", source = "hex", outer_checksum = "CF3BFC5D0997750D9550D8D73A90F4B8D71C6C081B20ED4E70FFBE1E99AFC3C2" }, - { name = "gleam_pgo", version = "0.7.0", build_tools = ["gleam"], requirements = ["gleam_stdlib", "pgo"], source = "local", path = "../../packages/pgo" }, - { name = "gleam_stdlib", version = "0.38.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "663CF11861179AF415A625307447775C09404E752FF99A24E2057C835319F1BE" }, + { name = "gleam_regexp", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_regexp", source = "hex", outer_checksum = "A3655FDD288571E90EE9C4009B719FEF59FA16AFCDF3952A76A125AF23CF1592" }, + { name = "gleam_stdlib", version = "0.44.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "A6E55E309A6778206AAD4038D9C49E15DF71027A1DB13C6ADA06BFDB6CF1260E" }, + { name = "gleam_yielder", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_yielder", source = "hex", outer_checksum = "8E4E4ECFA7982859F430C57F549200C7749823C106759F4A19A78AEA6687717A" }, { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" }, { name = "glexer", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "glexer", source = "hex", outer_checksum = "BD477AD657C2B637FEF75F2405FAEFFA533F277A74EF1A5E17B55B1178C228FB" }, - { name = "glisten", version = "2.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib"], otp_app = "glisten", source = "hex", outer_checksum = "CF3A9383E9BA4A8CBAF2F7B799716290D02F2AC34E7A77556B49376B662B9314" }, + { name = "glisten", version = "6.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib", "logging", "telemetry"], otp_app = "glisten", source = "hex", outer_checksum = "912132751031473CB38F454120124FFC96AF6B0EA33D92C9C90DB16327A2A972" }, { name = "gramps", version = "2.0.3", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_http", "gleam_stdlib"], otp_app = "gramps", source = "hex", outer_checksum = "3CCAA6E081225180D95C79679D383BBF51C8D1FDC1B84DA1DA444F628C373793" }, { name = "hpack_erl", version = "0.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "hpack", source = "hex", outer_checksum = "D6137D7079169D8C485C6962DFE261AF5B9EF60FBC557344511C1E65E3D95FB0" }, { name = "logging", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "1098FBF10B54B44C2C7FDF0B01C1253CAFACDACABEFB4B0D027803246753E06D" }, - { name = "marceau", version = "1.2.0", build_tools = ["gleam"], requirements = [], otp_app = "marceau", source = "hex", outer_checksum = "5188D643C181EE350D8A20A3BDBD63AF7B6C505DE333CFBE05EF642ADD88A59B" }, - { name = "mist", version = "1.2.0", build_tools = ["gleam"], requirements = ["birl", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "gramps", "hpack_erl", "logging"], otp_app = "mist", source = "hex", outer_checksum = "109B4D64E68C104CC23BB3CC5441ECD479DD7444889DA01113B75C6AF0F0E17B" }, - { name = "opentelemetry_api", version = "1.3.0", build_tools = ["rebar3", "mix"], requirements = ["opentelemetry_semantic_conventions"], otp_app = "opentelemetry_api", source = "hex", outer_checksum = "B9E5FF775FD064FA098DBA3C398490B77649A352B40B0B730A6B7DC0BDD68858" }, - { name = "opentelemetry_semantic_conventions", version = "0.2.0", build_tools = ["rebar3", "mix"], requirements = [], otp_app = "opentelemetry_semantic_conventions", source = "hex", outer_checksum = "D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895" }, + { name = "marceau", version = "1.3.0", build_tools = ["gleam"], requirements = [], otp_app = "marceau", source = "hex", outer_checksum = "2D1C27504BEF45005F5DFB18591F8610FB4BFA91744878210BDC464412EC44E9" }, + { name = "mist", version = "3.0.0", build_tools = ["gleam"], requirements = ["birl", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "gramps", "hpack_erl", "logging"], otp_app = "mist", source = "hex", outer_checksum = "CDA1A74E768419235E16886463EC4722EFF4AB3F8D820A76EAD45D7C167D7282" }, + { name = "opentelemetry_api", version = "1.4.0", build_tools = ["rebar3", "mix"], requirements = [], otp_app = "opentelemetry_api", source = "hex", outer_checksum = "3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58" }, { name = "pg_types", version = "0.4.0", build_tools = ["rebar3"], requirements = [], otp_app = "pg_types", source = "hex", outer_checksum = "B02EFA785CAECECF9702C681C80A9CA12A39F9161A846CE17B01FB20AEEED7EB" }, { name = "pgo", version = "0.14.0", build_tools = ["rebar3"], requirements = ["backoff", "opentelemetry_api", "pg_types"], otp_app = "pgo", source = "hex", outer_checksum = "71016C22599936E042DC0012EE4589D24C71427D266292F775EBF201D97DF9C9" }, { name = "platform", version = "1.0.0", build_tools = ["gleam"], requirements = [], otp_app = "platform", source = "hex", outer_checksum = "8339420A95AD89AAC0F82F4C3DB8DD401041742D6C3F46132A8739F6AEB75391" }, - { name = "pprint", version = "1.0.3", build_tools = ["gleam"], requirements = ["glam", "gleam_stdlib"], otp_app = "pprint", source = "hex", outer_checksum = "76BBB92E23D12D954BD452686543F29EDE8EBEBB7FC0ACCBCA66EEF276EC3A06" }, + { name = "pog", version = "1.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib", "pgo"], otp_app = "pog", source = "hex", outer_checksum = "48D6570C1E98F77A3C29E7ED4F1D81205DD0D870DCEC55F536EF3A57A28DF627" }, + { name = "pprint", version = "1.0.4", build_tools = ["gleam"], requirements = ["glam", "gleam_stdlib"], otp_app = "pprint", source = "hex", outer_checksum = "C310A98BDC0995644847C3C8702DE19656D6BCD638B2A8A358B97824379ECAA1" }, { name = "prng", version = "3.0.3", build_tools = ["gleam"], requirements = ["gleam_bitwise", "gleam_stdlib"], otp_app = "prng", source = "hex", outer_checksum = "53006736FE23A0F61828C95B505193E10905D8DB76E128F1642D3E571E08F589" }, { name = "radiate", version = "0.4.0", build_tools = ["gleam"], requirements = ["filespy", "gleam_otp", "gleam_stdlib", "shellout"], otp_app = "radiate", source = "hex", outer_checksum = "93A76A66EE4741DBFD3E79E27CBD11FE58EC3CB1C58F017FC165944E339D6293" }, - { name = "ranger", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "1566C272B1D141B3BBA38B25CB761EF56E312E79EC0E2DFD4D3C19FB0CC1F98C" }, + { name = "ranger", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "ranger", source = "hex", outer_checksum = "B8F3AFF23A3A5B5D9526B8D18E7C43A7DFD3902B151B97EC65397FE29192B695" }, { name = "shellout", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "shellout", source = "hex", outer_checksum = "E2FCD18957F0E9F67E1F497FC9FF57393392F8A9BAEAEA4779541DE7A68DD7E0" }, - { name = "simplifile", version = "2.1.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "BDD04F5D31D6D34E2EDFAEF0B68A6297AEC939888C3BFCE61133DE13857F6DA2" }, + { name = "simplifile", version = "2.2.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "0DFABEF7DC7A9E2FF4BB27B108034E60C81BEBFCB7AB816B9E7E18ED4503ACD8" }, { name = "stoiridh_version", version = "0.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "stoiridh_version", source = "hex", outer_checksum = "EEF8ADAB9755BD33EB202F169376F1A7797AEF90823FDCA671D8590D04FBF56B" }, + { name = "telemetry", version = "1.3.0", build_tools = ["rebar3"], requirements = [], otp_app = "telemetry", source = "hex", outer_checksum = "7015FC8919DBE63764F4B4B87A95B7C0996BD539E0D499BE6EC9D7F3875B79E6" }, { name = "thoas", version = "1.2.1", build_tools = ["rebar3"], requirements = [], otp_app = "thoas", source = "hex", outer_checksum = "E38697EDFFD6E91BD12CEA41B155115282630075C2A727E7A6B2947F5408B86A" }, { name = "tom", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], source = "local", path = "../../packages/tom" }, { name = "verl", version = "1.1.1", build_tools = ["rebar3"], requirements = [], otp_app = "verl", source = "hex", outer_checksum = "0925E51CD92A0A8BE271765B02430B2E2CFF8AC30EF24D123BD0D58511E8FB18" }, - { name = "wisp", version = "1.1.0", build_tools = ["gleam"], requirements = ["directories", "exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "5B25E37E08A94039668D2ACF7B2D2A665BEAF4B48EFA8613010B4E3164137664" }, + { name = "wisp", version = "1.3.0", build_tools = ["gleam"], requirements = ["directories", "exception", "gleam_crypto", "gleam_erlang", "gleam_http", "gleam_json", "gleam_stdlib", "logging", "marceau", "mist", "simplifile"], otp_app = "wisp", source = "hex", outer_checksum = "D12384EE63ADEE833B40A6D26EF8014A6E4BFC10EC2CDC8B57D325DD4B52740E" }, ] [requirements] -aws4_request = { version = ">= 0.1.1 and < 1.0.0" } +aws4_request = { version = ">= 1.2.0 and < 2.0.0" } birl = { version = "~> 1.6" } chomp = { version = ">= 0.1.0 and < 1.0.0" } -cors_builder = { version = ">= 2.0.0 and < 3.0.0" } +cors_builder = { version = ">= 2.0.2 and < 3.0.0" } decipher = { version = ">= 1.2.0 and < 2.0.0" } +envoy = { version = ">= 1.0.2 and < 2.0.0" } gleam_erlang = { version = "~> 0.25" } gleam_hexpm = { version = "~> 1.0" } gleam_http = { version = "~> 3.6" } @@ -65,12 +68,14 @@ gleam_httpc = { version = ">= 2.2.0 and < 3.0.0" } gleam_json = { version = "~> 1.0" } gleam_otp = { version = "~> 0.10" } gleam_package_interface = { version = ">= 1.0.0 and < 2.0.0" } -gleam_pgo = { path = "../../packages/pgo" } -gleam_stdlib = { version = "~> 0.34 or ~> 1.0" } +gleam_regexp = { version = ">= 1.0.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 0.44.0 and < 1.0.0" } +gleam_yielder = { version = ">= 1.1.0 and < 2.0.0" } gleeunit = { version = "~> 1.0" } glexer = { version = ">= 1.0.1 and < 2.0.0" } -mist = { version = ">= 1.2.0 and < 2.0.0" } +mist = { version = ">= 3.0.0 and < 4.0.0" } pgo = { version = "~> 0.14" } +pog = { version = ">= 1.0.1 and < 2.0.0" } pprint = { version = ">= 1.0.3 and < 2.0.0" } prng = { version = ">= 3.0.3 and < 4.0.0" } radiate = { version = ">= 0.4.0 and < 1.0.0" } @@ -78,4 +83,4 @@ ranger = { version = ">= 1.2.0 and < 2.0.0" } simplifile = { version = ">= 2.1.0 and < 3.0.0" } tom = { path = "../../packages/tom" } verl = { version = ">= 1.1.1 and < 2.0.0" } -wisp = { version = ">= 1.1.0 and < 2.0.0" } +wisp = { version = ">= 1.3.0 and < 2.0.0" } diff --git a/apps/backend/src/api/github.gleam b/apps/backend/src/api/github.gleam index a00f0d8..6f01afa 100644 --- a/apps/backend/src/api/github.gleam +++ b/apps/backend/src/api/github.gleam @@ -8,7 +8,7 @@ import gleam/httpc import gleam/json import gleam/list import gleam/option.{type Option, Some} -import gleam/regex +import gleam/regexp import gleam/result fn query( @@ -17,12 +17,7 @@ fn query( variables: Option(json.Json), decoder: dynamic.Decoder(a), ) { - let body = - json.object([ - #("query", json.string(query)), - #("variables", json.nullable(variables, function.identity)), - ]) - use response <- result.try( + use response <- result.try({ request.new() |> request.set_header("authorization", "Bearer " <> token) |> request.set_header("user-agent", "gloogle / 1.0.0") @@ -30,19 +25,27 @@ fn query( |> request.set_scheme(http.Https) |> request.set_host("api.github.com") |> request.set_path("/graphql") - |> request.set_body(json.to_string(body)) + |> request.set_body(encode_body(query, variables)) |> httpc.send() - |> result.map_error(error.FetchError), - ) + |> result.map_error(error.FetchError) + }) response.body |> json.decode(using: decoder) |> result.map_error(error.JsonError) } +fn encode_body(query: String, variables: Option(json.Json)) -> String { + json.object([ + #("query", json.string(query)), + #("variables", json.nullable(variables, function.identity)), + ]) + |> json.to_string +} + fn match_repository_name(repo_url: String) { - let assert Ok(owner_name) = regex.from_string("https://github.com/(.+)/(.+)") - regex.scan(with: owner_name, content: repo_url) + let assert Ok(owner_name) = regexp.from_string("https://github.com/(.+)/(.+)") + regexp.scan(with: owner_name, content: repo_url) |> list.first() |> result.replace_error(error.UnknownError( "No repository match for " <> repo_url, diff --git a/apps/backend/src/api/hex_repo.gleam b/apps/backend/src/api/hex_repo.gleam index eea473f..25982b7 100644 --- a/apps/backend/src/api/hex_repo.gleam +++ b/apps/backend/src/api/hex_repo.gleam @@ -1,3 +1,4 @@ +import aws/s3 import backend/error import gleam/bit_array import gleam/erlang/process @@ -7,15 +8,14 @@ import gleam/httpc import gleam/json import gleam/package_interface import gleam/result -import s3 import simplifile import tom import wisp -@external(erlang, "gloogle_hex_ffi", "get_home") +@external(erlang, "backend_ffi", "get_home") pub fn get_home() -> Result(String, Nil) -@external(erlang, "gloogle_hex_ffi", "extract_tar") +@external(erlang, "backend_ffi", "extract_tar") fn extract_tar( tarbin: BitArray, base_name: String, @@ -23,7 +23,7 @@ fn extract_tar( slug: String, ) -> Result(#(String, String, String), Nil) -@external(erlang, "gloogle_hex_ffi", "remove_tar") +@external(erlang, "backend_ffi", "remove_tar") fn remove_tar(slug: String) -> Nil fn package_slug(name: String, version: String) { diff --git a/apps/backend/src/s3.gleam b/apps/backend/src/aws/s3.gleam similarity index 67% rename from apps/backend/src/s3.gleam rename to apps/backend/src/aws/s3.gleam index c8725a2..2d2405b 100644 --- a/apps/backend/src/s3.gleam +++ b/apps/backend/src/aws/s3.gleam @@ -1,6 +1,5 @@ import aws4_request -import backend/config -import birl +import backend/context import gleam/http import gleam/http/request import gleam/httpc @@ -8,9 +7,8 @@ import gleam/option.{type Option, None, Some} import gleam/result fn request(url: String, method: http.Method, body: Option(BitArray)) { - let date = birl.to_erlang_universal_datetime(birl.now()) - use bucket_uri <- result.try(config.bucket_uri()) - use #(access_key, secret_key) <- result.try(config.scaleway_keys()) + use bucket_uri <- result.try(context.bucket_uri()) + use #(access_key, secret_key) <- result.try(context.scaleway_keys()) request.new() |> request.set_method(method) |> request.set_path(url) @@ -18,9 +16,19 @@ fn request(url: String, method: http.Method, body: Option(BitArray)) { |> request.set_host(bucket_uri) |> request.set_scheme(http.Https) |> request.set_header("content-type", "application/octet-stream") - |> aws4_request.sign(date, access_key, secret_key, "fr-par", "s3") + |> sign(access_key, secret_key) |> httpc.send_bits() - |> result.nil_error() + |> result.replace_error(Nil) +} + +fn sign(request, access_key_id, secret_access_key) { + aws4_request.signer( + access_key_id:, + secret_access_key:, + region: "fr-par", + service: "s3", + ) + |> aws4_request.sign_bits(request) } pub fn get(name: String) { diff --git a/apps/backend/src/backend.gleam b/apps/backend/src/backend.gleam index a9f76dc..03f82a4 100644 --- a/apps/backend/src/backend.gleam +++ b/apps/backend/src/backend.gleam @@ -1,73 +1,57 @@ -import backend/config +import backend/context.{type Context, Context} import backend/gleam/type_search/state as type_search -import backend/postgres/postgres import backend/router +import backend/setup +import backend/workers +import envoy import gleam/erlang/process import gleam/function +import gleam/int +import gleam/option.{Some} import gleam/otp/supervisor +import gleam/result import mist -import periodic -import setup -import tasks/hex -import tasks/popularity -import tasks/ranking -import tasks/timeseries import wisp import wisp/logger import wisp/wisp_mist pub fn main() { - wisp.configure_logger() - - let secret_key_base = config.get_secret_key_base() - let cnf = config.read_config() - let ctx = postgres.connect(cnf) + configure_logger() + let assert Ok(ctx) = context.init() + let assert Ok(ctx) = start_type_search_worker(ctx) + let assert Ok(_) = start_http_server(ctx) + let assert Ok(_) = start_periodic_workers(ctx) + process.sleep_forever() +} - logger.set_level(cnf.level) +fn configure_logger() { + let level = logger.read_level() + wisp.configure_logger() + logger.set_level(level) setup.radiate() +} - let assert Ok(subject) = type_search.init(ctx.db) - - let ctx = ctx |> config.add_type_search_subject(subject) - - let assert Ok(_) = - router.handle_request(_, ctx) - |> wisp_mist.handler(secret_key_base) - |> mist.new() - |> mist.port(cnf.port) - |> mist.start_http() - - let assert Ok(_) = { - use periodic_children <- supervisor.start() - use periodic_children <- function.tap(periodic_children) - let assert Ok(_) = { - use children <- supervisor.start() - // Every 10 seconds - add_periodic_worker(periodic_children, waiting: 10 * 1000, do: fn() { - hex.sync_new_gleam_releases(ctx, children) - }) - // Every day - add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { - ranking.compute_ranking(ctx) - }) - // Every day - add_periodic_worker(periodic_children, waiting: 86_400_000, do: fn() { - popularity.compute_popularity(ctx) - }) - // Every hour - add_periodic_worker(periodic_children, waiting: 3600 * 1000, do: fn() { - timeseries.store_timeseries(ctx) - }) - } - } +fn start_type_search_worker(ctx: Context) { + use subject <- result.map(type_search.init(ctx.db)) + Context(..ctx, type_search_subject: Some(subject)) +} - process.sleep_forever() +fn start_http_server(ctx) { + use port <- result.try(envoy.get("PORT")) + use port <- result.map(int.parse(port)) + let secret_key_base = context.get_secret_key_base() + router.handle_request(_, ctx) + |> wisp_mist.handler(secret_key_base) + |> mist.new() + |> mist.port(port) + |> mist.start_http() } -fn add_periodic_worker(children, waiting delay, do work) { - use _ <- function.tap(children) - supervisor.add(children, { - use _ <- supervisor.worker() - periodic.periodically(do: work, waiting: delay) - }) +fn start_periodic_workers(ctx) { + use children <- supervisor.start() + use children <- function.tap(children) + workers.sync_new_gleam_releases_ten_secondly(ctx, children) + workers.compute_ranking_daily(ctx, children) + workers.compute_popularity_daily(ctx, children) + workers.store_timeseries_hourly(ctx, children) } diff --git a/apps/backend/src/backend/config.gleam b/apps/backend/src/backend/config.gleam deleted file mode 100644 index 2a6cef2..0000000 --- a/apps/backend/src/backend/config.gleam +++ /dev/null @@ -1,76 +0,0 @@ -import backend/gleam/type_search/msg as type_search -import gleam/erlang/os -import gleam/erlang/process.{type Subject} -import gleam/int -import gleam/option.{type Option} -import gleam/pgo -import gleam/result -import gleam/string -import wisp -import wisp/logger - -pub type Environment { - Development - Production -} - -pub type Context { - Context( - db: pgo.Connection, - hex_api_key: String, - github_token: String, - env: Environment, - type_search_subject: Option(Subject(type_search.Msg)), - ) -} - -pub type Config { - Config( - database_url: String, - hex_api_key: String, - port: Int, - level: logger.Level, - github_token: String, - env: Environment, - ) -} - -pub fn read_config() { - let assert Ok(database_url) = os.get_env("DATABASE_URL") - let assert Ok(hex_api_key) = os.get_env("HEX_API_KEY") - let assert Ok(github_token) = os.get_env("GITHUB_TOKEN") - let env = case result.unwrap(os.get_env("GLEAM_ENV"), "") { - "development" -> Development - _ -> Production - } - let assert Ok(port) = - os.get_env("PORT") - |> result.try(int.parse) - let level = - os.get_env("LOG_LEVEL") - |> result.try(logger.parse) - |> result.unwrap(logger.Info) - Config(database_url, hex_api_key, port, level, github_token, env) -} - -pub fn get_secret_key_base() { - wisp.random_string(64) -} - -pub fn is_dev() { - os.get_env("GLEAM_ENV") == Ok("development") -} - -pub fn bucket_uri() { - os.get_env("BUCKET_URI") -} - -pub fn scaleway_keys() { - use access_key <- result.try(os.get_env("SCALEWAY_ACCESS_KEY")) - use secret_key <- result.map(os.get_env("SCALEWAY_SECRET_KEY")) - #(access_key, secret_key) -} - -pub fn add_type_search_subject(context, subject) { - Context(..context, type_search_subject: option.Some(subject)) -} diff --git a/apps/backend/src/backend/context.gleam b/apps/backend/src/backend/context.gleam new file mode 100644 index 0000000..43836f2 --- /dev/null +++ b/apps/backend/src/backend/context.gleam @@ -0,0 +1,57 @@ +import backend/gleam/type_search/msg as type_search +import backend/postgres/postgres +import envoy +import gleam/erlang/process.{type Subject} +import gleam/option.{type Option, None} +import gleam/result +import pog +import wisp + +pub type Environment { + Development + Production +} + +pub type Context { + Context( + db: pog.Connection, + hex_api_key: String, + github_token: String, + env: Environment, + type_search_subject: Option(Subject(type_search.Msg)), + ) +} + +pub fn init() { + let env = read_environment() + use database_url <- result.try(envoy.get("DATABASE_URL")) + use hex_api_key <- result.try(envoy.get("HEX_API_KEY")) + use github_token <- result.map(envoy.get("GITHUB_TOKEN")) + let db = postgres.connect(database_url) + Context(db:, hex_api_key:, github_token:, env:, type_search_subject: None) +} + +pub fn read_environment() { + case envoy.get("GLEAM_ENV") { + Ok("development") -> Development + _ -> Production + } +} + +pub fn get_secret_key_base() { + wisp.random_string(64) +} + +pub fn is_dev() { + envoy.get("GLEAM_ENV") == Ok("development") +} + +pub fn bucket_uri() { + envoy.get("BUCKET_URI") +} + +pub fn scaleway_keys() { + use access_key <- result.try(envoy.get("SCALEWAY_ACCESS_KEY")) + use secret_key <- result.map(envoy.get("SCALEWAY_SECRET_KEY")) + #(access_key, secret_key) +} diff --git a/apps/backend/src/backend/error.gleam b/apps/backend/src/backend/error.gleam index 2f436af..f4bb5c0 100644 --- a/apps/backend/src/backend/error.gleam +++ b/apps/backend/src/backend/error.gleam @@ -2,15 +2,15 @@ import gleam/dynamic import gleam/int import gleam/json import gleam/list -import gleam/pgo import gleam/result import gleam/string +import pog import simplifile import tom import wisp pub type Error { - DatabaseError(pgo.QueryError) + DatabaseError(pog.QueryError) FetchError(dynamic.Dynamic) JsonError(json.DecodeError) SimplifileError(simplifile.FileError, String) @@ -20,6 +20,18 @@ pub type Error { EmptyError } +pub fn empty() { + Error(EmptyError) +} + +pub fn new(message: String) { + Error(UnknownError(message)) +} + +pub fn replace_nil(res, message: String) { + result.replace_error(res, UnknownError(message)) +} + pub fn log_dynamic_error(error: dynamic.DecodeError) { wisp.log_warning("Dynamic Decode Error") wisp.log_warning(" expected: " <> error.expected) @@ -54,7 +66,7 @@ pub fn log_error(error: Error) { FetchError(_dyn) -> wisp.log_warning("Fetch error") DatabaseError(error) -> { wisp.log_warning("Query error") - log_pgo_error(error) + log_pog_error(error) } JsonError(error) -> { wisp.log_warning("JSON error") @@ -168,36 +180,36 @@ pub fn log_simplifile(error: simplifile.FileError) { } } -pub fn log_pgo_error(error: pgo.QueryError) { +pub fn log_pog_error(error: pog.QueryError) { case error { - pgo.ConstraintViolated(message, constraint, details) -> { + pog.ConstraintViolated(message, constraint, details) -> { wisp.log_warning("Constraint violated") wisp.log_warning(" message: " <> message) wisp.log_warning(" constraint: " <> constraint) wisp.log_warning(" details: " <> details) } - pgo.PostgresqlError(code, name, message) -> { - let code = result.unwrap(pgo.error_code_name(code), code) + pog.PostgresqlError(code, name, message) -> { + let code = result.unwrap(pog.error_code_name(code), code) wisp.log_warning("PostgreSQL error") wisp.log_warning(" error: " <> code) wisp.log_warning(" name: " <> name) wisp.log_warning(" message: " <> message) } - pgo.UnexpectedArgumentCount(expected, got) -> { + pog.UnexpectedArgumentCount(expected, got) -> { wisp.log_warning("Unexpected argument count") wisp.log_warning(" expected: " <> int.to_string(expected)) wisp.log_warning(" got: " <> int.to_string(got)) } - pgo.UnexpectedArgumentType(expected, got) -> { + pog.UnexpectedArgumentType(expected, got) -> { wisp.log_warning("Unexpected argument type") wisp.log_warning(" expected: " <> expected) wisp.log_warning(" got: " <> got) } - pgo.UnexpectedResultType(error) -> { + pog.UnexpectedResultType(error) -> { wisp.log_warning("Unexpected result type") list.map(error, log_dynamic_error) Nil } - pgo.ConnectionUnavailable -> wisp.log_warning("Connection unavailable") + pog.ConnectionUnavailable -> wisp.log_warning("Connection unavailable") } } diff --git a/apps/backend/src/backend/gleam/context.gleam b/apps/backend/src/backend/gleam/context.gleam index 0171d39..2ef8586 100644 --- a/apps/backend/src/backend/gleam/context.gleam +++ b/apps/backend/src/backend/gleam/context.gleam @@ -3,12 +3,12 @@ import gleam/dict.{type Dict} import gleam/erlang/process.{type Subject} import gleam/option.{type Option} import gleam/package_interface -import gleam/pgo +import pog import tom pub type Context { Context( - db: pgo.Connection, + db: pog.Connection, package_interface: package_interface.Package, gleam_toml: Dict(String, tom.Toml), /// Allow to bypass parameters relations if activated. diff --git a/apps/backend/src/backend/gleam/generate/types.gleam b/apps/backend/src/backend/gleam/generate/types.gleam index 35c4ff9..4c4f7d4 100644 --- a/apps/backend/src/backend/gleam/generate/types.gleam +++ b/apps/backend/src/backend/gleam/generate/types.gleam @@ -7,17 +7,17 @@ import gleam/dict import gleam/dynamic import gleam/json.{type Json} import gleam/list -import gleam/option +import gleam/option.{type Option, None, Some} import gleam/order import gleam/package_interface.{ type Constant, type Function, type Implementations, type Parameter, type Type, type TypeAlias, type TypeConstructor, type TypeDefinition, } import gleam/pair -import gleam/pgo import gleam/result import gleam/set.{type Set} import gleam/verl +import pog fn reduce_components( components: List(a), @@ -105,8 +105,8 @@ fn type_to_json(ctx: Context, type_: Type) { let res = extract_parameters_relation(ctx, name, package, module) use ref <- result.map(res) let new_ids = case ref { - option.None -> gen.1 - option.Some(ref) -> set.insert(gen.1, ref.1) + None -> gen.1 + Some(ref) -> set.insert(gen.1, ref.1) } json.object([ #("kind", json.string("named")), @@ -122,38 +122,47 @@ fn type_to_json(ctx: Context, type_: Type) { } fn find_package_release(ctx: Context, package: String, requirement: String) { - let decoder = dynamic.tuple2(dynamic.int, dynamic.string) - use response <- result.try({ - "SELECT package_release.id, package_release.version + "SELECT package_release.id, package_release.version FROM package JOIN package_release ON package.id = package_release.package_id WHERE package.name = $1" - |> pgo.execute(ctx.db, [pgo.text(package)], decoder) - |> result.map_error(error.DatabaseError) - }) - response.rows - |> keep_matching_releases(requirement) + |> pog.query + |> pog.parameter(pog.text(package)) + |> pog.returning(dynamic.tuple2(dynamic.int, dynamic.string)) + |> pog.execute(ctx.db) + |> result.map_error(error.DatabaseError) + |> result.map(fn(response) { response.rows }) + |> result.map(keep_matching_releases(_, requirement)) } fn keep_matching_releases(rows: List(#(Int, String)), requirement: String) { - let requirement = bit_array.from_string(requirement) rows - |> list.filter(fn(r) { - let version = bit_array.from_string(r.1) - let is_matching = verl.is_match(version: version, requirement: requirement) - result.unwrap(is_matching, False) - }) - |> list.sort(fn(a, b) { - let a = bit_array.from_string(a.1) - let b = bit_array.from_string(b.1) - case verl.gte(version: a, with: b) { - True -> order.Lt - False -> order.Gt - } - }) - |> list.map(fn(a) { a.0 }) - |> Ok() + |> list.filter(keep_matching_requirement(_, requirement)) + |> list.sort(by_decreasing_version) + |> list.map(pair.first) +} + +fn keep_matching_requirement(release: #(Int, String), requirement: String) { + let #(_release_id, release_version) = release + let requirement = bit_array.from_string(requirement) + let version = bit_array.from_string(release_version) + let is_matching = verl.is_match(version:, requirement:) + result.unwrap(is_matching, False) +} + +fn by_decreasing_version( + release_1: #(Int, String), + release_2: #(Int, String), +) -> order.Order { + let #(_, release_1_version) = release_1 + let #(_, release_2_version) = release_2 + let release_1_version = bit_array.from_string(release_1_version) + let release_2_version = bit_array.from_string(release_2_version) + case verl.gte(version: release_1_version, with: release_2_version) { + True -> order.Lt + False -> order.Gt + } } fn find_signature_from_release( @@ -162,11 +171,9 @@ fn find_signature_from_release( module: String, releases: List(Int), ) { - use acc, release <- list.fold(releases, Error(Nil)) + use acc, release <- list.fold(releases, error.empty()) use <- bool.guard(when: result.is_ok(acc), return: acc) - let args = [pgo.text(name), pgo.text(module), pgo.int(release)] - use t <- result.try({ - "SELECT release.version, signature.id + "SELECT release.version, signature.id FROM package_release release JOIN package_module module ON module.package_release_id = release.id @@ -175,11 +182,17 @@ fn find_signature_from_release( WHERE signature.name = $1 AND module.name = $2 AND module.package_release_id = $3" - |> pgo.execute(ctx.db, args, dynamic.tuple2(dynamic.string, dynamic.int)) - |> result.nil_error() + |> pog.query + |> pog.parameter(pog.text(name)) + |> pog.parameter(pog.text(module)) + |> pog.parameter(pog.int(release)) + |> pog.returning(dynamic.tuple2(dynamic.string, dynamic.int)) + |> pog.execute(ctx.db) + |> result.map_error(error.DatabaseError) + |> result.try(fn(response) { + list.first(response.rows) + |> error.replace_nil("[find_signature_from_release] No row") }) - list.first(t.rows) - |> result.nil_error() } fn find_type_signature( @@ -188,9 +201,9 @@ fn find_type_signature( package: String, module: String, releases: List(Int), -) -> Result(option.Option(#(String, Int)), error.Error) { +) -> Result(Option(#(String, Int)), error.Error) { case find_signature_from_release(ctx, name, module, releases) { - Ok(value) -> Ok(option.Some(value)) + Ok(value) -> Ok(Some(value)) Error(_) -> { let slug = package <> "/" <> module let package_name = ctx.package_interface.name @@ -205,7 +218,7 @@ fn find_type_signature( True -> case dict.get(ctx.package_interface.modules, module) { // Module is hidden, everything is correct, type is hidden. - Error(_) -> Ok(option.None) + Error(_) -> Ok(None) // Module is not hidden, checking if type is hidden by itself. Ok(mod) -> { let slug = slug <> "." <> name @@ -220,7 +233,7 @@ fn find_type_signature( Error(_) -> case dict.get(mod.types, name) { // Type is hidden, returning None because it can't be extracted. - Error(_) -> Ok(option.None) + Error(_) -> Ok(None) // Type is not hidden, returning an error to restart the extraction. Ok(_) -> { let id = package_name <> ", looking for " <> slug @@ -241,17 +254,15 @@ fn extract_parameters_relation( name: String, package: String, module: String, -) -> Result(option.Option(#(String, Int)), error.Error) { - use <- bool.guard(when: is_prelude(package, module), return: Ok(option.None)) +) -> Result(Option(#(String, Int)), error.Error) { + use <- bool.guard(when: is_prelude(package, module), return: Ok(None)) use requirement <- result.try(toml.find_package_requirement(ctx, package)) use releases <- result.try(find_package_release(ctx, package, requirement)) - use error <- result.try_recover({ - find_type_signature(ctx, name, package, module, releases) + find_type_signature(ctx, name, package, module, releases) + |> result.try_recover(fn(error) { + use <- bool.guard(when: ctx.ignore_parameters_errors, return: Ok(None)) + Error(error) }) - case ctx.ignore_parameters_errors { - False -> Error(error) - True -> Ok(option.None) - } } fn is_prelude(package: String, module: String) { diff --git a/apps/backend/src/backend/gleam/type_search.gleam b/apps/backend/src/backend/gleam/type_search.gleam index 2373e44..e4ddec2 100644 --- a/apps/backend/src/backend/gleam/type_search.gleam +++ b/apps/backend/src/backend/gleam/type_search.gleam @@ -5,8 +5,8 @@ import gleam/int import gleam/list import gleam/option.{type Option} import gleam/pair -import gleam/pgo import gleam/result +import pog pub type TypeSearch { TypeSearch(keys: Keys, rows: List(Int)) @@ -48,25 +48,25 @@ fn update_keys( parse.DiscardName -> panic as "No Discard name in add" parse.Index(_value, index) -> { let value = int.to_string(index) - dict.update(keys.keys, value, fn(k) { + dict.upsert(keys.keys, value, fn(k) { let k = option.unwrap(k, Keys(keys: dict.new(), next: next)) update_keys(k, rest, updater) }) } parse.Custom(value, kinds) -> - dict.update(keys.keys, value, fn(k) { + dict.upsert(keys.keys, value, fn(k) { let k = option.unwrap(k, Keys(keys: dict.new(), next: next)) update_keys(k, list.append(kinds, rest), updater) }) parse.Function(kinds, return) -> { let kinds = postpend(kinds, return) - dict.update(keys.keys, "fn", fn(k) { + dict.upsert(keys.keys, "fn", fn(k) { let k = option.unwrap(k, Keys(keys: dict.new(), next: next)) update_keys(k, list.append(kinds, rest), updater) }) } parse.Tuple(kinds) -> { - dict.update(keys.keys, "#()", fn(k) { + dict.upsert(keys.keys, "#()", fn(k) { let k = option.unwrap(k, Keys(keys: dict.new(), next: next)) update_keys(k, list.append(kinds, rest), updater) }) @@ -112,7 +112,7 @@ fn get_next_tree( keys: Keys, kind: Kind, env: Dict(Int, String), - db: pgo.Connection, + db: pog.Connection, ) -> List(#(Keys, Dict(Int, String))) { case kind { parse.DiscardName -> { @@ -182,7 +182,7 @@ fn find_next_tree( kind: Kind, kinds: List(Kind), env: Dict(Int, String), - db: pgo.Connection, + db: pog.Connection, ) -> List(Int) { case kind { parse.DiscardName -> { @@ -252,7 +252,7 @@ fn do_find( searches: TypeSearch, kinds: List(Kind), env: Dict(Int, String), - db: pgo.Connection, + db: pog.Connection, ) { case kinds { [] -> searches.rows @@ -260,7 +260,7 @@ fn do_find( } } -pub fn find(searches: TypeSearch, kind: Kind, db: pgo.Connection) { +pub fn find(searches: TypeSearch, kind: Kind, db: pog.Connection) { case kind { Function(kinds, return_value) -> kinds diff --git a/apps/backend/src/backend/gleam/type_search/state.gleam b/apps/backend/src/backend/gleam/type_search/state.gleam index 9930818..e8c057c 100644 --- a/apps/backend/src/backend/gleam/type_search/state.gleam +++ b/apps/backend/src/backend/gleam/type_search/state.gleam @@ -8,14 +8,14 @@ import gleam/function import gleam/list import gleam/option import gleam/otp/actor -import gleam/pgo import gleam/result +import pog pub type State { - State(db: pgo.Connection, search: TypeSearch) + State(db: pog.Connection, search: TypeSearch) } -pub fn init(db: pgo.Connection) { +pub fn init(db: pog.Connection) { let init = fn() { let search = compute_rows(0, db, #(0, type_search.empty()), { @@ -41,7 +41,7 @@ fn loop(msg: msg.Msg, state: State) -> actor.Next(msg.Msg, State) { msg.Find(subject, signature) -> { signature |> parse.parse_function - |> result.nil_error + |> result.replace_error(Nil) |> result.then(permutation_search(state, _)) |> option.from_result |> function.tap(fn(res) { process.send(subject, res) }) @@ -88,11 +88,10 @@ fn permutation_search(state: State, kind: parse.Kind) { fn compute_rows( offset: Int, - db: pgo.Connection, + db: pog.Connection, default: a, next: fn(a, #(String, Int)) -> a, ) { - let decoder = dynamic.tuple2(dynamic.string, dynamic.int) let rows = "SELECT signature_, id FROM package_type_fun_signature @@ -100,7 +99,10 @@ fn compute_rows( ORDER BY id ASC LIMIT 1000 OFFSET $1" - |> pgo.execute(db, [pgo.int(offset)], decoder) + |> pog.query + |> pog.parameter(pog.int(offset)) + |> pog.returning(dynamic.tuple2(dynamic.string, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.unwrap([]) use <- bool.guard(when: list.is_empty(rows), return: default) diff --git a/apps/backend/src/backend/postgres/postgres.gleam b/apps/backend/src/backend/postgres/postgres.gleam index 65a31b5..45170e1 100644 --- a/apps/backend/src/backend/postgres/postgres.gleam +++ b/apps/backend/src/backend/postgres/postgres.gleam @@ -1,26 +1,15 @@ -import backend/config.{type Config, type Context, Context} import backend/error import gleam/bool import gleam/list import gleam/option.{type Option, Some} -import gleam/pgo.{Config} import gleam/result import gleam/string import gleam/uri +import pog.{Config} -pub fn connect(cnf: Config) { - let assert Ok(config) = parse_database_url(cnf.database_url) - config - |> pgo.connect() - |> fn(db) { - Context( - db: db, - hex_api_key: cnf.hex_api_key, - github_token: cnf.github_token, - env: cnf.env, - type_search_subject: option.None, - ) - } +pub fn connect(database_url: String) { + let assert Ok(config) = parse_database_url(database_url) + pog.connect(config) } fn parse_database_url(database_url: String) { @@ -33,7 +22,7 @@ fn parse_database_url(database_url: String) { return: Error(error.UnknownError("No postgres protocol")), ) - pgo.default_config() + pog.default_config() |> fn(cnf) { Config(..cnf, database: string.replace(db_uri.path, "/", "")) } |> update_config(db_uri.userinfo, add_user_info) |> update_config(db_uri.host, fn(cnf, u) { Config(..cnf, host: u) }) @@ -51,15 +40,15 @@ fn parse_database_url(database_url: String) { } fn update_config( - cnf: pgo.Config, + cnf: pog.Config, field: Option(a), - mapper: fn(pgo.Config, a) -> pgo.Config, + mapper: fn(pog.Config, a) -> pog.Config, ) { option.map(field, fn(u) { mapper(cnf, u) }) |> option.unwrap(cnf) } -fn add_user_info(c: pgo.Config, u: String) { +fn add_user_info(c: pog.Config, u: String) { case string.split(u, ":") { [user, password] -> Config(..c, user: user, password: Some(password)) [user] -> Config(..c, user: user) diff --git a/apps/backend/src/backend/postgres/queries.gleam b/apps/backend/src/backend/postgres/queries.gleam index 1411fdb..edb7cab 100644 --- a/apps/backend/src/backend/postgres/queries.gleam +++ b/apps/backend/src/backend/postgres/queries.gleam @@ -13,10 +13,13 @@ import gleam/list import gleam/option.{type Option, None, Some} import gleam/package_interface import gleam/pair -import gleam/pgo import gleam/result import gleam/string import helpers +import pog + +@external(erlang, "backend_ffi", "coerce") +fn coerce(a: a) -> b pub type SignatureKind { TypeAlias @@ -25,9 +28,11 @@ pub type SignatureKind { Function } -pub fn get_last_hex_date(db: pgo.Connection) { +pub fn get_last_hex_date(db: pog.Connection) { "SELECT id, last_check FROM hex_read ORDER BY last_check DESC LIMIT 1" - |> pgo.execute(db, [], hex_read.decode) + |> pog.query + |> pog.returning(hex_read.decode) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(response) { response.rows @@ -37,15 +42,17 @@ pub fn get_last_hex_date(db: pgo.Connection) { }) } -pub fn upsert_most_recent_hex_timestamp(db: pgo.Connection, latest: Time) { - let timestamp = helpers.convert_time(latest) +pub fn upsert_most_recent_hex_timestamp(db: pog.Connection, latest: Time) { "INSERT INTO hex_read OVERRIDING SYSTEM VALUE VALUES (1, $1) ON CONFLICT (id) DO UPDATE SET last_check = $1 RETURNING *" - |> pgo.execute(db, [timestamp], hex_read.decode) + |> pog.query + |> pog.parameter(helpers.convert_time(latest)) + |> pog.returning(hex_read.decode) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.try(fn(response) { let err = "Upsert most recent hex timestamp failed" @@ -55,13 +62,16 @@ pub fn upsert_most_recent_hex_timestamp(db: pgo.Connection, latest: Time) { }) } -pub fn upsert_search_analytics(db: pgo.Connection, query: String) { +pub fn upsert_search_analytics(db: pog.Connection, query: String) { "INSERT INTO search_analytics (query) VALUES ($1) ON CONFLICT (query) DO UPDATE SET occurences = search_analytics.occurences + 1 RETURNING *" - |> pgo.execute(db, [pgo.text(query)], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.text(query)) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.try(fn(response) { let err = "Upsert search analytics failed" @@ -71,20 +81,15 @@ pub fn upsert_search_analytics(db: pgo.Connection, query: String) { }) } -pub fn select_more_popular_packages(db: pgo.Connection) { - let decoder = - dynamic.tuple4( - dynamic.string, - dynamic.string, - dynamic.int, - dynamic.optional(dynamic.int), - ) +pub fn select_more_popular_packages(db: pog.Connection) { use ranked <- result.try({ "SELECT name, repository, rank, (popularity -> 'github')::int FROM package ORDER BY rank DESC LIMIT 22" - |> pgo.execute(db, [], decoder) + |> pog.query + |> pog.returning(decode_popular_package) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) }) @@ -96,49 +101,59 @@ pub fn select_more_popular_packages(db: pgo.Connection) { AND name != 'dew' ORDER BY popularity -> 'github' DESC LIMIT 23" - |> pgo.execute(db, [], decoder) + |> pog.query + |> pog.returning(decode_popular_package) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) }) Ok(#(ranked, popular)) } -pub fn select_last_day_search_analytics(db: pgo.Connection) { +fn decode_popular_package(dyn) { + dynamic.tuple4( + dynamic.string, + dynamic.string, + dynamic.int, + dynamic.optional(dynamic.int), + )(dyn) +} + +pub fn select_last_day_search_analytics(db: pog.Connection) { let #(date, _) = birl.to_erlang_universal_datetime(birl.now()) let now = birl.from_erlang_universal_datetime(#(date, #(0, 0, 0))) "SELECT query, occurences FROM search_analytics WHERE updated_at >= $1" - |> pgo.execute( - db, - [helpers.convert_time(now)], - dynamic.tuple2(dynamic.string, dynamic.int), - ) + |> pog.query + |> pog.parameter(helpers.convert_time(now)) + |> pog.returning(dynamic.tuple2(dynamic.string, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } pub fn upsert_search_analytics_timeseries( - db: pgo.Connection, + db: pog.Connection, analytic: #(String, Int), ) { let #(date, _) = birl.to_erlang_universal_datetime(birl.now()) let now = birl.from_erlang_universal_datetime(#(date, #(0, 0, 0))) - let timestamp = helpers.convert_time(now) let #(query, occurences) = analytic "INSERT INTO analytics_timeseries (query, occurences, date) VALUES ($1, $2, $3) ON CONFLICT (query, date) DO UPDATE SET occurences = $2" - |> pgo.execute( - db, - [pgo.text(query), pgo.int(occurences), timestamp], - dynamic.dynamic, - ) + |> pog.query + |> pog.parameter(pog.text(query)) + |> pog.parameter(pog.int(occurences)) + |> pog.parameter(helpers.convert_time(now)) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) |> result.map_error(error.DatabaseError) } -pub fn get_timeseries_count(db: pgo.Connection) { +pub fn get_timeseries_count(db: pog.Connection) { "SELECT SUM(at.occurences - COALESCE( (SELECT att.occurences @@ -154,65 +169,77 @@ pub fn get_timeseries_count(db: pgo.Connection) { WHERE at.date >= now() - INTERVAL '30 day' GROUP BY at.date ORDER BY date DESC" - |> pgo.execute(db, [], dynamic.tuple2(dynamic.int, helpers.decode_time)) + |> pog.query + |> pog.returning(dynamic.tuple2(dynamic.int, helpers.decode_time)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn upsert_hex_user(db: pgo.Connection, owner: hexpm.PackageOwner) { - let username = pgo.text(owner.username) - let email = pgo.nullable(pgo.text, owner.email) - let url = pgo.text(owner.url) +pub fn upsert_hex_user(db: pog.Connection, owner: hexpm.PackageOwner) { "INSERT INTO hex_user (username, email, url) VALUES ($1, $2, $3) ON CONFLICT (username) DO UPDATE SET email = $2, url = $3 RETURNING id, username, email, url, created_at, updated_at" - |> pgo.execute(db, [username, email, url], hex_user.decode) + |> pog.query + |> pog.parameter(pog.text(owner.username)) + |> pog.parameter(pog.nullable(pog.text, owner.email)) + |> pog.parameter(pog.text(owner.url)) + |> pog.returning(hex_user.decode) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } -fn upsert_package_owners(db: pgo.Connection, owners: List(hexpm.PackageOwner)) { +fn upsert_package_owners(db: pog.Connection, owners: List(hexpm.PackageOwner)) { owners |> list.map(upsert_hex_user(db, _)) |> result.all() |> result.map(list.flatten) } -fn get_current_package_owners(db: pgo.Connection, package_id: Int) { - let pid = pgo.int(package_id) +fn get_current_package_owners(db: pog.Connection, package_id: Int) { "SELECT package_owner.hex_user_id FROM package_owner WHERE package_owner.package_id = $1" - |> pgo.execute(db, [pid], dynamic.element(0, dynamic.int)) + |> pog.query + |> pog.parameter(pog.int(package_id)) + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } -pub fn get_total_searches(db: pgo.Connection) { +pub fn get_total_searches(db: pog.Connection) { "SELECT SUM(occurences) FROM search_analytics" - |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> pog.query + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } -pub fn get_total_signatures(db: pgo.Connection) { +pub fn get_total_signatures(db: pog.Connection) { "SELECT COUNT(*) FROM package_type_fun_signature" - |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> pog.query + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } -pub fn get_total_packages(db: pgo.Connection) { +pub fn get_total_packages(db: pog.Connection) { "SELECT COUNT(*) FROM package" - |> pgo.execute(db, [], dynamic.element(0, dynamic.int)) + |> pog.query + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } fn add_new_package_owners( - db: pgo.Connection, + db: pog.Connection, owners: List(HexUser), current_owners: List(Int), package_id: Int, @@ -220,18 +247,20 @@ fn add_new_package_owners( owners |> list.filter(fn(o) { bool.negate(list.contains(current_owners, o.id)) }) |> list.map(fn(u) { - let hex_user_id = pgo.int(u.id) - let pid = pgo.int(package_id) "INSERT INTO package_owner (hex_user_id, package_id) VALUES ($1, $2)" - |> pgo.execute(db, [hex_user_id, pid], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.int(u.id)) + |> pog.parameter(pog.int(package_id)) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) }) |> result.all() |> result.map_error(error.DatabaseError) } fn remove_old_package_owners( - db: pgo.Connection, + db: pog.Connection, owners: List(HexUser), current_owners: List(Int), package_id: Int, @@ -240,19 +269,21 @@ fn remove_old_package_owners( current_owners |> list.filter(fn(id) { list.contains(curr, id) }) |> list.map(fn(u) { - let hex_user_id = pgo.int(u) - let pid = pgo.int(package_id) "DELETE FROM package_owner WHERE package_owner.hex_user_id = $1 AND package_owner.package_id = $2" - |> pgo.execute(db, [hex_user_id, pid], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.int(u)) + |> pog.parameter(pog.int(package_id)) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) }) |> result.all() |> result.map_error(error.DatabaseError) } pub fn sync_package_owners( - db: pgo.Connection, + db: pog.Connection, package_id: Int, owners: List(hexpm.PackageOwner), ) { @@ -263,24 +294,7 @@ pub fn sync_package_owners( Ok(Nil) } -pub fn upsert_package(db: pgo.Connection, package: hexpm.Package) { - let name = pgo.text(package.name) - let hex_url = pgo.nullable(pgo.text, package.html_url) - let docs = pgo.nullable(pgo.text, package.docs_html_url) - let repo = - package.meta.links - |> dict.get("Repository") - |> option.from_result() - |> pgo.nullable(pgo.text, _) - let links = - package.meta.links - |> helpers.json_dict() - |> pgo.text() - let licenses = - package.meta.licenses - |> helpers.json_list() - |> pgo.text() - let description = pgo.nullable(pgo.text, package.meta.description) +pub fn upsert_package(db: pog.Connection, package: hexpm.Package) { "INSERT INTO package (name, repository, documentation, hex_url, links, licenses, description) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -293,11 +307,21 @@ pub fn upsert_package(db: pgo.Connection, package: hexpm.Package) { licenses = $6, description = $7 RETURNING id" - |> pgo.execute( - db, - [name, repo, docs, hex_url, links, licenses, description], - dynamic.element(0, dynamic.int), - ) + |> pog.query + |> pog.parameter(pog.text(package.name)) + |> pog.parameter(pog.nullable(pog.text, package.html_url)) + |> pog.parameter(pog.nullable(pog.text, package.docs_html_url)) + |> pog.parameter({ + package.meta.links + |> dict.get("Repository") + |> option.from_result() + |> pog.nullable(pog.text, _) + }) + |> pog.parameter(package.meta.links |> helpers.json_dict() |> pog.text()) + |> pog.parameter(package.meta.licenses |> helpers.json_list() |> pog.text()) + |> pog.parameter(pog.nullable(pog.text, package.meta.description)) + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.try(fn(response) { response.rows @@ -307,30 +331,12 @@ pub fn upsert_package(db: pgo.Connection, package: hexpm.Package) { } pub fn upsert_release( - db: pgo.Connection, + db: pog.Connection, package_id: Int, release: hexpm.Release, package_interface: Option(String), gleam_toml: Option(String), ) { - let package_id = pgo.int(package_id) - let version = pgo.text(release.version) - let url = pgo.text(release.url) - let package_interface = pgo.nullable(pgo.text, package_interface) - let gleam_toml = pgo.nullable(pgo.text, gleam_toml) - let inserted_at = - release.inserted_at - |> birl.to_erlang_universal_datetime - |> dynamic.from - |> dynamic.unsafe_coerce - let args = [ - package_id, - version, - url, - package_interface, - gleam_toml, - inserted_at, - ] "INSERT INTO package_release ( package_id, version, @@ -346,38 +352,43 @@ pub fn upsert_release( gleam_toml = $5, inserted_at = $6 RETURNING id, package_interface, gleam_toml" - |> pgo.execute( - db, - args, - dynamic.tuple3( - dynamic.int, - dynamic.optional(dynamic.string), - dynamic.optional(dynamic.string), - ), - ) + |> pog.query + |> pog.parameter(pog.int(package_id)) + |> pog.parameter(pog.text(release.version)) + |> pog.parameter(pog.text(release.url)) + |> pog.parameter(pog.nullable(pog.text, package_interface)) + |> pog.parameter(pog.nullable(pog.text, gleam_toml)) + |> pog.parameter({ + release.inserted_at + |> birl.to_erlang_universal_datetime + |> coerce + }) + |> pog.returning(dynamic.tuple3( + dynamic.int, + dynamic.optional(dynamic.string), + dynamic.optional(dynamic.string), + )) + |> pog.execute(db) |> result.map_error(error.DatabaseError) } pub fn lookup_release( - db: pgo.Connection, + db: pog.Connection, package_id: Int, release: hexpm.Release, ) { - let package_id = pgo.int(package_id) - let version = pgo.text(release.version) - let args = [package_id, version] "SELECT id, package_interface, gleam_toml - FROM package_release - WHERE package_id = $1 AND version = $2" - |> pgo.execute( - db, - args, - dynamic.tuple3( - dynamic.int, - dynamic.optional(dynamic.string), - dynamic.optional(dynamic.string), - ), - ) + FROM package_release + WHERE package_id = $1 AND version = $2" + |> pog.query + |> pog.parameter(pog.int(package_id)) + |> pog.parameter(pog.text(release.version)) + |> pog.returning(dynamic.tuple3( + dynamic.int, + dynamic.optional(dynamic.string), + dynamic.optional(dynamic.string), + )) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.try(fn(res) { res.rows @@ -387,84 +398,92 @@ pub fn lookup_release( } pub fn add_package_gleam_constraint( - db: pgo.Connection, + db: pog.Connection, constraint: String, release_id: Int, ) { - let constraint = pgo.text(constraint) - let release_id = pgo.int(release_id) "UPDATE package_release SET gleam_constraint = $1 WHERE id = $2" - |> pgo.execute(db, [constraint, release_id], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.text(constraint)) + |> pog.parameter(pog.int(release_id)) + |> pog.execute(db) |> result.replace(Nil) |> result.map_error(error.DatabaseError) } -pub fn add_package_gleam_retirement( - db: pgo.Connection, +pub fn add_package_retirement( + db: pog.Connection, retirement: hexpm.ReleaseRetirement, release_id: Int, ) { - let retirement = - json.object([ - #("message", json.nullable(retirement.message, json.string)), - #("reason", { - json.string(case retirement.reason { - hexpm.OtherReason -> "other-reason" - hexpm.Invalid -> "invalid" - hexpm.Security -> "security" - hexpm.Deprecated -> "deprecated" - hexpm.Renamed -> "renamed" - }) - }), - ]) - |> json.to_string() - |> pgo.text() - let release_id = pgo.int(release_id) "UPDATE package_release SET retirement = $1 WHERE id = $2" - |> pgo.execute(db, [retirement, release_id], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.text(encode_retirement(retirement))) + |> pog.parameter(pog.int(release_id)) + |> pog.execute(db) |> result.replace(Nil) |> result.map_error(error.DatabaseError) } +fn encode_retirement(retirement: hexpm.ReleaseRetirement) { + json.object([ + #("message", json.nullable(retirement.message, json.string)), + #("reason", { + json.string(case retirement.reason { + hexpm.OtherReason -> "other-reason" + hexpm.Invalid -> "invalid" + hexpm.Security -> "security" + hexpm.Deprecated -> "deprecated" + hexpm.Renamed -> "renamed" + }) + }), + ]) + |> json.to_string() +} + pub fn get_package_release_ids( - db: pgo.Connection, + db: pog.Connection, package: package_interface.Package, ) { - use response <- result.try({ - let args = [pgo.text(package.name), pgo.text(package.version)] - "SELECT - package.id package_id, - package_release.id package_release_id - FROM package - JOIN package_release - ON package_release.package_id = package.id - WHERE package.name = $1 - AND package_release.version = $2" - |> pgo.execute(db, args, dynamic.tuple2(dynamic.int, dynamic.int)) - |> result.map_error(error.DatabaseError) + "SELECT + package.id package_id, + package_release.id package_release_id + FROM package + JOIN package_release + ON package_release.package_id = package.id + WHERE package.name = $1 + AND package_release.version = $2" + |> pog.query + |> pog.parameter(pog.text(package.name)) + |> pog.parameter(pog.text(package.version)) + |> pog.returning(dynamic.tuple2(dynamic.int, dynamic.int)) + |> pog.execute(db) + |> result.map_error(error.DatabaseError) + |> result.try(fn(response) { + list.first(response.rows) + |> result.replace_error(error.UnknownError( + "No release found for " <> package.name <> "@" <> package.version, + )) }) - response.rows - |> list.first() - |> result.replace_error(error.UnknownError( - "No release found for " <> package.name <> "@" <> package.version, - )) } -pub fn upsert_package_module(db: pgo.Connection, module: context.Module) { +pub fn upsert_package_module(db: pog.Connection, module: context.Module) { use response <- result.try({ - let args = [ - pgo.text(module.name), - module.module.documentation - |> string.join("\n") - |> pgo.text(), - pgo.int(module.release_id), - ] "INSERT INTO package_module (name, documentation, package_release_id) VALUES ($1, $2, $3) ON CONFLICT (name, package_release_id) DO UPDATE SET documentation = $2 RETURNING id" - |> pgo.execute(db, args, dynamic.element(0, dynamic.int)) + |> pog.query + |> pog.parameter(pog.text(module.name)) + |> pog.parameter({ + module.module.documentation + |> string.join("\n") + |> pog.text() + }) + |> pog.parameter(pog.int(module.release_id)) + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) }) response.rows @@ -486,7 +505,7 @@ fn implementations_pgo(implementations: package_interface.Implementations) { } pub fn upsert_package_type_fun_signature( - db db: pgo.Connection, + db db: pog.Connection, kind kind: SignatureKind, name name: String, documentation documentation: Option(String), @@ -498,12 +517,6 @@ pub fn upsert_package_type_fun_signature( deprecation deprecation: Option(package_interface.Deprecation), implementations implementations: Option(package_interface.Implementations), ) { - let kind = case kind { - Function -> "function" - TypeAlias -> "type_alias" - TypeDefinition -> "type_definition" - Constant -> "constant" - } "INSERT INTO package_type_fun_signature ( name, documentation, @@ -527,39 +540,47 @@ pub fn upsert_package_type_fun_signature( deprecation = $9, implementations = $10 RETURNING id" - |> pgo.execute( - db, - [ - pgo.text(name), - documentation - |> option.unwrap("") - |> string.trim() - |> pgo.text(), - pgo.text(signature), - json_signature - |> json.to_string() - |> pgo.text(), - pgo.text(kind), - dynamic.unsafe_coerce(dynamic.from(parameters)), - metadata - |> json.to_string() - |> pgo.text(), - pgo.int(module_id), - deprecation - |> option.map(fn(d) { d.message }) - |> pgo.nullable(pgo.text, _), - implementations - |> option.map(implementations_pgo) - |> option.map(pgo.text) - |> option.unwrap(pgo.null()), - ], - dynamic.element(0, dynamic.int), - ) + |> pog.query + |> pog.parameter(pog.text(name)) + |> pog.parameter({ + documentation + |> option.unwrap("") + |> string.trim() + |> pog.text() + }) + |> pog.parameter(pog.text(signature)) + |> pog.parameter(json_signature |> json.to_string() |> pog.text()) + |> pog.parameter(pog.text(kind_to_string(kind))) + |> pog.parameter(coerce(parameters)) + |> pog.parameter(metadata |> json.to_string() |> pog.text()) + |> pog.parameter(pog.int(module_id)) + |> pog.parameter({ + deprecation + |> option.map(fn(d) { d.message }) + |> pog.nullable(pog.text, _) + }) + |> pog.parameter({ + implementations + |> option.map(implementations_pgo) + |> option.map(pog.text) + |> option.unwrap(pog.null()) + }) + |> pog.returning(dynamic.element(0, dynamic.int)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn find_similar_type_names(db: pgo.Connection, name: String) { +fn kind_to_string(kind: SignatureKind) -> String { + case kind { + Function -> "function" + TypeAlias -> "type_alias" + TypeDefinition -> "type_definition" + Constant -> "constant" + } +} + +pub fn find_similar_type_names(db: pog.Connection, name: String) { "SELECT * FROM (VALUES ('Int'), @@ -577,13 +598,15 @@ pub fn find_similar_type_names(db: pgo.Connection, name: String) { FROM package_type_fun_signature WHERE (kind = 'type_definition' OR kind = 'type_alias') AND levenshtein_less_equal(name, $1, 2) <= 2" - |> pgo.execute(db, [pgo.text(name)], dynamic.element(0, dynamic.string)) + |> pog.query + |> pog.parameter(pog.text(name)) + |> pog.returning(dynamic.element(0, dynamic.string)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn name_search(db: pgo.Connection, query: String) { - let query = pgo.text(query) +pub fn name_search(db: pog.Connection, query: String) { "SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, s.documentation, @@ -605,13 +628,15 @@ pub fn name_search(db: pgo.Connection, query: String) { WHERE s.name = $1 ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name LIMIT 100" - |> pgo.execute(db, [query], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(query)) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn module_and_name_search(db: pgo.Connection, query: String) { - let query = pgo.text(query) +pub fn module_and_name_search(db: pog.Connection, query: String) { "WITH splitted_name AS (SELECT string_to_array($1, '.') AS full_name) SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, @@ -637,7 +662,10 @@ pub fn module_and_name_search(db: pgo.Connection, query: String) { AND m.name LIKE '%' || lower(s_n.full_name[1]) || '%' ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name LIMIT 100" - |> pgo.execute(db, [query], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(query)) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } @@ -653,9 +681,7 @@ fn transform_query(q: String) { |> string.join(" ") } -pub fn content_search(db: pgo.Connection, query: String) { - let pattern = pgo.text(transform_query(query)) - let query = pgo.text(query) +pub fn content_search(db: pog.Connection, query: String) { "SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, s.documentation, @@ -683,7 +709,11 @@ pub fn content_search(db: pgo.Connection, query: String) { ) ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name LIMIT 100" - |> pgo.execute(db, [query, pattern], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(query)) + |> pog.parameter(pog.text(transform_query(query))) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } @@ -708,16 +738,15 @@ pub fn type_search_to_json(item) { #("name", json.string(a)), #("documentation", json.string(b)), #("kind", json.string(c)), - #("metadata", dynamic.unsafe_coerce(d)), - #("json_signature", dynamic.unsafe_coerce(e)), + #("metadata", coerce(d)), + #("json_signature", coerce(e)), #("module_name", json.string(f)), #("package_name", json.string(g)), #("version", json.string(h)), ]) } -pub fn signature_search(db: pgo.Connection, q: String) { - let query = pgo.text(q) +pub fn signature_search(db: pog.Connection, q: String) { "SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, s.documentation, @@ -739,13 +768,15 @@ pub fn signature_search(db: pgo.Connection, q: String) { WHERE to_tsvector('english', s.signature_) @@ websearch_to_tsquery($1) ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name LIMIT 100" - |> pgo.execute(db, [query], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(q)) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn documentation_search(db: pgo.Connection, q: String) { - let query = pgo.text(q) +pub fn documentation_search(db: pog.Connection, q: String) { "SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, s.documentation, @@ -767,13 +798,15 @@ pub fn documentation_search(db: pgo.Connection, q: String) { WHERE to_tsvector('english', s.documentation) @@ websearch_to_tsquery($1) ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name LIMIT 100" - |> pgo.execute(db, [query], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(q)) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn module_search(db: pgo.Connection, q: String) { - let query = pgo.text(q) +pub fn module_search(db: pog.Connection, q: String) { "SELECT DISTINCT ON (package_rank, ordering, type_name, signature_kind, module_name) s.name type_name, s.documentation, @@ -804,12 +837,15 @@ pub fn module_search(db: pgo.Connection, q: String) { AND r.id = m.package_release_id ) ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name" - |> pgo.execute(db, [query], decode_type_search) + |> pog.query + |> pog.parameter(pog.text(q)) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn exact_type_search(db: pgo.Connection, q: List(Int)) { +pub fn exact_type_search(db: pog.Connection, q: List(Int)) { use <- bool.guard(when: list.is_empty(q), return: Ok([])) let ids = list.index_map(q, fn(_, idx) { "$" <> int.to_string(idx + 1) }) @@ -838,31 +874,44 @@ pub fn exact_type_search(db: pgo.Connection, q: List(Int)) { <> ") ORDER BY package_rank DESC, ordering DESC, type_name, signature_kind, module_name" } - |> pgo.execute(db, list.map(q, pgo.int), decode_type_search) + |> pog.query + |> list.fold(q, _, fn(query, q) { pog.parameter(query, pog.int(q)) }) + |> pog.returning(decode_type_search) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn select_gleam_toml(db: pgo.Connection, offset: Int) { +pub fn select_gleam_toml(db: pog.Connection, offset: Int) { "SELECT gleam_toml FROM package_release WHERE gleam_toml IS NOT NULL ORDER BY id LIMIT 100 OFFSET $1" - |> pgo.execute(db, [pgo.int(offset)], dynamic.element(0, dynamic.string)) + |> pog.query + |> pog.parameter(pog.int(offset)) + |> pog.returning(dynamic.element(0, dynamic.string)) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } -pub fn update_package_rank(db: pgo.Connection, package: String, rank: Int) { +pub fn update_package_rank(db: pog.Connection, package: String, rank: Int) { "UPDATE package SET rank = $2 WHERE name = $1" - |> pgo.execute(db, [pgo.text(package), pgo.int(rank)], dynamic.dynamic) + |> pog.query + |> pog.parameter(pog.text(package)) + |> pog.parameter(pog.int(rank)) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) |> result.map_error(error.DatabaseError) } -pub fn select_package_repository_address(db: pgo.Connection, offset: Int) { - let decoder = fn(dyn) { +pub fn select_package_repository_address(db: pog.Connection, offset: Int) { + "SELECT id, repository FROM package LIMIT 100 OFFSET $1" + |> pog.query + |> pog.parameter(pog.int(offset)) + |> pog.returning(fn(dyn) { dynamic.tuple2(dynamic.int, dynamic.optional(dynamic.string))(dyn) |> result.map(fn(content) { case content { @@ -870,30 +919,33 @@ pub fn select_package_repository_address(db: pgo.Connection, offset: Int) { #(_, None) -> None } }) - } - "SELECT id, repository FROM package LIMIT 100 OFFSET $1" - |> pgo.execute(db, [pgo.int(offset)], decoder) + }) + |> pog.execute(db) |> result.map_error(error.DatabaseError) |> result.map(fn(r) { r.rows }) } pub fn update_package_popularity( - db: pgo.Connection, + db: pog.Connection, url: String, popularity: Dict(String, Int), ) { - let popularity = + "UPDATE package SET popularity = $2 WHERE repository = $1" + |> pog.query + |> pog.parameter(pog.text(url)) + |> pog.parameter({ dict.to_list(popularity) |> list.map(pair.map_second(_, json.int)) |> json.object() |> json.to_string() - |> pgo.text() - "UPDATE package SET popularity = $2 WHERE repository = $1" - |> pgo.execute(db, [pgo.text(url), popularity], dynamic.dynamic) + |> pog.text() + }) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) |> result.map_error(error.DatabaseError) } -pub fn select_package_by_popularity(db: pgo.Connection, page: Int) { +pub fn select_package_by_popularity(db: pog.Connection, page: Int) { let offset = 40 * page "SELECT name, @@ -909,37 +961,36 @@ pub fn select_package_by_popularity(db: pgo.Connection, page: Int) { ORDER BY popularity -> 'github' DESC LIMIT 40 OFFSET $1" - |> pgo.execute( - db, - [pgo.int(offset)], - dynamic.decode8( - fn(a, b, c, d, e, f, g, h) { - json.object([ - #("name", json.string(a)), - #("repository", json.nullable(b, json.string)), - #("documentation", json.nullable(c, json.string)), - #("hex-url", json.nullable(d, json.string)), - #("licenses", json.string(e)), - #("description", json.nullable(f, json.string)), - #("rank", json.int(g)), - #("popularity", json.nullable(h, json.string)), - ]) - }, - dynamic.element(0, dynamic.string), - dynamic.element(1, dynamic.optional(dynamic.string)), - dynamic.element(2, dynamic.optional(dynamic.string)), - dynamic.element(3, dynamic.optional(dynamic.string)), - dynamic.element(4, dynamic.string), - dynamic.element(5, dynamic.optional(dynamic.string)), - dynamic.element(6, dynamic.int), - dynamic.element(7, dynamic.optional(dynamic.string)), - ), - ) + |> pog.query + |> pog.parameter(pog.int(offset)) + |> pog.returning(dynamic.decode8( + fn(a, b, c, d, e, f, g, h) { + json.object([ + #("name", json.string(a)), + #("repository", json.nullable(b, json.string)), + #("documentation", json.nullable(c, json.string)), + #("hex-url", json.nullable(d, json.string)), + #("licenses", json.string(e)), + #("description", json.nullable(f, json.string)), + #("rank", json.int(g)), + #("popularity", json.nullable(h, json.string)), + ]) + }, + dynamic.element(0, dynamic.string), + dynamic.element(1, dynamic.optional(dynamic.string)), + dynamic.element(2, dynamic.optional(dynamic.string)), + dynamic.element(3, dynamic.optional(dynamic.string)), + dynamic.element(4, dynamic.string), + dynamic.element(5, dynamic.optional(dynamic.string)), + dynamic.element(6, dynamic.int), + dynamic.element(7, dynamic.optional(dynamic.string)), + )) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } -pub fn select_package_by_updated_at(db: pgo.Connection) { +pub fn select_package_by_updated_at(db: pog.Connection) { "SELECT name, repository, @@ -951,60 +1002,62 @@ pub fn select_package_by_updated_at(db: pgo.Connection) { popularity FROM package ORDER BY updated_at DESC" - |> pgo.execute( - db, - [], - dynamic.decode8( - fn(a, b, c, d, e, f, g, h) { - json.object([ - #("name", json.string(a)), - #("repository", json.nullable(b, json.string)), - #("documentation", json.nullable(c, json.string)), - #("hex-url", json.nullable(d, json.string)), - #("licenses", json.string(e)), - #("description", json.nullable(f, json.string)), - #("rank", json.int(g)), - #("popularity", json.nullable(h, json.string)), - ]) - }, - dynamic.element(0, dynamic.string), - dynamic.element(1, dynamic.optional(dynamic.string)), - dynamic.element(2, dynamic.optional(dynamic.string)), - dynamic.element(3, dynamic.optional(dynamic.string)), - dynamic.element(4, dynamic.string), - dynamic.element(5, dynamic.optional(dynamic.string)), - dynamic.element(6, dynamic.int), - dynamic.element(7, dynamic.optional(dynamic.string)), - ), - ) + |> pog.query + |> pog.returning(dynamic.decode8( + fn(a, b, c, d, e, f, g, h) { + json.object([ + #("name", json.string(a)), + #("repository", json.nullable(b, json.string)), + #("documentation", json.nullable(c, json.string)), + #("hex-url", json.nullable(d, json.string)), + #("licenses", json.string(e)), + #("description", json.nullable(f, json.string)), + #("rank", json.int(g)), + #("popularity", json.nullable(h, json.string)), + ]) + }, + dynamic.element(0, dynamic.string), + dynamic.element(1, dynamic.optional(dynamic.string)), + dynamic.element(2, dynamic.optional(dynamic.string)), + dynamic.element(3, dynamic.optional(dynamic.string)), + dynamic.element(4, dynamic.string), + dynamic.element(5, dynamic.optional(dynamic.string)), + dynamic.element(6, dynamic.int), + dynamic.element(7, dynamic.optional(dynamic.string)), + )) + |> pog.execute(db) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } pub fn insert_analytics( - db: pgo.Connection, + db: pog.Connection, id: Int, table_name: String, content: Dict(String, Int), ) { - let day = - birl.now() - |> birl.to_erlang_universal_datetime() - |> pair.map_second(fn(_) { #(0, 0, 0) }) - |> dynamic.from() - |> dynamic.unsafe_coerce() - let content = + "INSERT INTO analytics (foreign_id, table_name, content, day) + VALUES ($1, $2, $3, $4) + ON CONFLICT (foreign_id, table_name, day) DO UPDATE + SET content = $3" + |> pog.query + |> pog.parameter(pog.int(id)) + |> pog.parameter(pog.text(table_name)) + |> pog.parameter({ content |> dict.to_list() |> list.map(pair.map_second(_, json.int)) |> json.object() |> json.to_string() - |> pgo.text() - let parameters = [pgo.int(id), pgo.text(table_name), content, day] - "INSERT INTO analytics (foreign_id, table_name, content, day) - VALUES ($1, $2, $3, $4) - ON CONFLICT (foreign_id, table_name, day) DO UPDATE - SET content = $3" - |> pgo.execute(db, parameters, dynamic.dynamic) + |> pog.text() + }) + |> pog.parameter({ + birl.now() + |> birl.to_erlang_universal_datetime() + |> pair.map_second(fn(_) { #(0, 0, 0) }) + |> coerce + }) + |> pog.returning(dynamic.dynamic) + |> pog.execute(db) |> result.map_error(error.DatabaseError) } diff --git a/apps/backend/src/backend/router.gleam b/apps/backend/src/backend/router.gleam index e3fdc0b..5c57d6e 100644 --- a/apps/backend/src/backend/router.gleam +++ b/apps/backend/src/backend/router.gleam @@ -1,5 +1,5 @@ import api/hex -import backend/config.{type Context} +import backend/context.{type Context} import backend/error import backend/gleam/type_search/msg as type_search import backend/postgres/queries @@ -13,14 +13,14 @@ import gleam/json import gleam/list import gleam/option import gleam/result -import gleam/string_builder +import gleam/string_tree import tasks/hex as syncing import wisp.{type Request, type Response} fn empty_json() { let content = "{}" content - |> string_builder.from_string() + |> string_tree.from_string() |> wisp.json_response(200) } @@ -50,18 +50,18 @@ fn search(query: String, ctx: Context) { |> result.map_error(error.debug_log) |> result.unwrap([]) |> list.filter(fn(i) { - !list.contains(list.concat([exact_type_searches, exact_name_matches]), i) + !list.contains(list.flatten([exact_type_searches, exact_name_matches]), i) }) let exact_matches = - list.concat([exact_name_matches, exact_module_and_name_matches]) + list.flatten([exact_name_matches, exact_module_and_name_matches]) let matches = queries.content_search(ctx.db, query) |> result.map_error(error.debug_log) |> result.unwrap([]) |> list.filter(fn(i) { - !list.contains(list.concat([exact_matches, exact_type_searches]), i) + !list.contains(list.flatten([exact_matches, exact_type_searches]), i) }) let signature_searches = @@ -70,7 +70,7 @@ fn search(query: String, ctx: Context) { |> result.unwrap([]) |> list.filter(fn(i) { !list.contains( - list.concat([exact_matches, exact_type_searches, matches]), + list.flatten([exact_matches, exact_type_searches, matches]), i, ) }) @@ -81,7 +81,7 @@ fn search(query: String, ctx: Context) { |> result.unwrap([]) |> list.filter(fn(i) { !list.contains( - list.concat([ + list.flatten([ exact_matches, exact_type_searches, matches, @@ -97,7 +97,7 @@ fn search(query: String, ctx: Context) { |> result.unwrap([]) |> list.filter(fn(i) { !list.contains( - list.concat([ + list.flatten([ exact_matches, exact_type_searches, matches, diff --git a/apps/backend/src/setup.gleam b/apps/backend/src/backend/setup.gleam similarity index 89% rename from apps/backend/src/setup.gleam rename to apps/backend/src/backend/setup.gleam index 661df93..2e94433 100644 --- a/apps/backend/src/setup.gleam +++ b/apps/backend/src/backend/setup.gleam @@ -1,4 +1,4 @@ -import backend/config +import backend/context import radiate import wisp @@ -7,7 +7,7 @@ fn print_radiate_update(_state: state, path: String) { } pub fn radiate() { - case config.is_dev() { + case context.is_dev() { False -> Nil True -> { let assert Ok(_) = diff --git a/apps/backend/src/backend/web.gleam b/apps/backend/src/backend/web.gleam index e7b117e..29aa89a 100644 --- a/apps/backend/src/backend/web.gleam +++ b/apps/backend/src/backend/web.gleam @@ -1,4 +1,4 @@ -import backend/config +import backend/context import cors_builder as cors_ import gleam/http import wisp.{type Request, type Response} @@ -14,14 +14,7 @@ pub fn foundations(req: Request, handler: Handler) -> Response { } pub fn cors() { - let origin = case config.is_dev() { - True -> cors_.allow_origin(_, "http://localhost:5173") - False -> fn(cors) { - cors - |> cors_.allow_origin("https://gloogle.run") - |> cors_.allow_origin("https://www.gloogle.run") - } - } + let origin = select_origin() cors_.new() |> origin() |> cors_.allow_method(http.Get) @@ -32,3 +25,16 @@ pub fn cors() { |> cors_.allow_header("sentry-trace") |> cors_.max_age(86_400) } + +fn select_origin() { + case context.read_environment() { + context.Development -> cors_.allow_origin(_, "http://localhost:5173") + context.Production -> allow_production + } +} + +fn allow_production(cors: cors_.Cors) -> cors_.Cors { + cors + |> cors_.allow_origin("https://gloogle.run") + |> cors_.allow_origin("https://www.gloogle.run") +} diff --git a/apps/backend/src/backend/workers.gleam b/apps/backend/src/backend/workers.gleam new file mode 100644 index 0000000..439a4f4 --- /dev/null +++ b/apps/backend/src/backend/workers.gleam @@ -0,0 +1,35 @@ +import gleam/function +import gleam/otp/supervisor +import processes/periodic +import tasks/hex +import tasks/popularity +import tasks/ranking +import tasks/timeseries + +pub fn sync_new_gleam_releases_ten_secondly(ctx, children) { + use <- add_periodic_worker(children, waiting: 10_000) + hex.sync_new_gleam_releases(ctx) +} + +pub fn compute_ranking_daily(ctx, children) { + use <- add_periodic_worker(children, waiting: 86_400_000) + ranking.compute_ranking(ctx) +} + +pub fn compute_popularity_daily(ctx, children) { + use <- add_periodic_worker(children, waiting: 86_400_000) + popularity.compute_popularity(ctx) +} + +pub fn store_timeseries_hourly(ctx, children) { + use <- add_periodic_worker(children, waiting: 3_600_000) + timeseries.store_timeseries(ctx) +} + +fn add_periodic_worker(children, waiting delay, do work) { + use _ <- function.tap(children) + supervisor.add(children, { + use _ <- supervisor.worker() + periodic.periodically(do: work, waiting: delay) + }) +} diff --git a/apps/backend/src/gloogle_hex_ffi.erl b/apps/backend/src/backend_ffi.erl similarity index 76% rename from apps/backend/src/gloogle_hex_ffi.erl rename to apps/backend/src/backend_ffi.erl index aed54c9..6c46943 100644 --- a/apps/backend/src/gloogle_hex_ffi.erl +++ b/apps/backend/src/backend_ffi.erl @@ -1,5 +1,6 @@ --module(gloogle_hex_ffi). --export([extract_tar/4, remove_tar/1, is_match/2, get_home/0, set_level/1]). +-module(backend_ffi). + +-export([extract_tar/4, remove_tar/1, is_match/2, get_home/0, set_level/1, coerce/1]). package_interface_path(ContentDest, BaseName) -> BuildFolder = <<"/build/dev/docs/">>, @@ -20,14 +21,22 @@ extract_tar(Binary, BaseName, Version, Slug) -> ContentDest = <>, Content = <>, case erl_tar:extract({binary, Binary}, [{cwd, PackagePath}]) of - {error, _} -> {error, nil}; + {error, _} -> + {error, nil}; _ -> - Url = <<"https://hexdocs.pm/", BaseName/binary, "/", Version/binary, "/package-interface.json">>, + Url = + <<"https://hexdocs.pm/", + BaseName/binary, + "/", + Version/binary, + "/package-interface.json">>, erl_tar:extract(Content, [{cwd, ContentDest}, compressed]), {PackageInterface, Result} = case httpc:request(Url) of - {ok, {{_, 200, _}, _, HttpBody}} -> save_file(ContentDest, HttpBody); - {ok, {200, HttpBody}} -> save_file(ContentDest, HttpBody); + {ok, {{_, 200, _}, _, HttpBody}} -> + save_file(ContentDest, HttpBody); + {ok, {200, HttpBody}} -> + save_file(ContentDest, HttpBody); {error, _} -> BuildCmd = <<"cd ", ContentDest/binary, " && gleam docs build">>, Res = os:cmd(binary_to_list(BuildCmd)), @@ -45,15 +54,22 @@ remove_tar(Slug) -> is_match(Version, Requirement) -> case verl:is_match(Version, Requirement) of - {error, _} -> {error, nil}; - Bool -> {ok, Bool} + {error, _} -> + {error, nil}; + Bool -> + {ok, Bool} end. get_home() -> case init:get_argument(home) of - {ok, Content} -> {ok, unicode:characters_to_binary(Content)}; - error -> {error, nil} + {ok, Content} -> + {ok, unicode:characters_to_binary(Content)}; + error -> + {error, nil} end. set_level(Level) -> logger:set_primary_config(level, Level). + +coerce(A) -> + A. diff --git a/apps/backend/src/gleam/verl.gleam b/apps/backend/src/gleam/verl.gleam index ca41040..250edda 100644 --- a/apps/backend/src/gleam/verl.gleam +++ b/apps/backend/src/gleam/verl.gleam @@ -1,4 +1,4 @@ -@external(erlang, "gloogle_hex_ffi", "is_match") +@external(erlang, "backend_ffi", "is_match") pub fn is_match( version version: BitArray, requirement requirement: BitArray, diff --git a/apps/backend/src/helpers.gleam b/apps/backend/src/helpers.gleam index ed4b920..a790c60 100644 --- a/apps/backend/src/helpers.gleam +++ b/apps/backend/src/helpers.gleam @@ -5,14 +5,16 @@ import gleam/float import gleam/json import gleam/list import gleam/pair -import gleam/pgo import gleam/result +import pog -pub fn convert_time(time: Time) -> pgo.Value { +@external(erlang, "backend_ffi", "coerce") +fn coerce(a: a) -> b + +pub fn convert_time(time: Time) -> pog.Value { time - |> birl.to_erlang_universal_datetime() - |> dynamic.from() - |> dynamic.unsafe_coerce() + |> birl.to_erlang_universal_datetime + |> coerce } pub fn decode_time(data: Dynamic) { diff --git a/apps/backend/src/periodic.gleam b/apps/backend/src/processes/periodic.gleam similarity index 100% rename from apps/backend/src/periodic.gleam rename to apps/backend/src/processes/periodic.gleam diff --git a/apps/backend/src/processes/retrier.gleam b/apps/backend/src/processes/retrier.gleam new file mode 100644 index 0000000..2ee1c0f --- /dev/null +++ b/apps/backend/src/processes/retrier.gleam @@ -0,0 +1,75 @@ +import backend/error.{type Error} +import gleam/bool +import gleam/erlang/process.{type Subject} +import gleam/function +import gleam/iterator.{type Iterator} +import gleam/otp/actor +import prng/random +import wisp + +pub opaque type Message { + Rerun +} + +type State(a) { + State( + self: Subject(Message), + work: fn(Int) -> Result(a, Error), + random_ints: Iterator(Int), + interval: Int, + iterations: Int, + ) +} + +pub const one_minute: Int = 60_000 + +fn enqueue_next_rerun(state: State(a)) { + let assert iterator.Next(cooldown, acc) = iterator.step(state.random_ints) + process.send_after(state.self, state.interval + cooldown, Rerun) + State(..state, random_ints: acc) +} + +/// Repeatedly call a function, leaving `interval` milliseconds between each call. +/// When the `work` function returns an error, it is printed. +pub fn retry(do work: fn(Int) -> Result(a, Error)) { + let _ = start_retrier(work) + Nil +} + +fn start_retrier(work: fn(Int) -> Result(a, Error)) { + fn() { init(every: one_minute, do: work) } + |> actor.Spec(loop:, init_timeout: 100) + |> actor.start_spec +} + +fn init( + every interval: Int, + do work: fn(Int) -> Result(a, Error), +) -> actor.InitResult(State(a), Message) { + let self = process.new_subject() + let random_ints = random.to_random_iterator(random.int(1000, 5000)) + let state = State(self:, work:, interval:, iterations: 10, random_ints:) + process.new_selector() + |> process.selecting(self, function.identity) + |> actor.Ready(state, _) + |> function.tap(fn(_) { process.send(state.self, Rerun) }) +} + +fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) { + case message, state.work(state.iterations) { + Rerun, Ok(_) -> actor.Stop(process.Normal) + Rerun, Error(error) -> { + wisp.log_notice("Process on error") + error.log_error(error) + use <- bool.lazy_guard(when: state.iterations == 0, return: stop_process) + State(..state, iterations: state.iterations - 1) + |> enqueue_next_rerun() + |> actor.continue() + } + } +} + +fn stop_process() { + wisp.log_notice("Stopping process after 10 iterations") + actor.Stop(process.Normal) +} diff --git a/apps/backend/src/retrier.gleam b/apps/backend/src/retrier.gleam deleted file mode 100644 index 1453bd6..0000000 --- a/apps/backend/src/retrier.gleam +++ /dev/null @@ -1,84 +0,0 @@ -import backend/error.{type Error} -import gleam/erlang/process.{type Subject} -import gleam/function -import gleam/iterator.{type Iterator} -import gleam/otp/actor -import prng/random -import wisp - -pub opaque type Message { - Rerun -} - -type State(a) { - State( - self: Subject(Message), - work: fn(Int) -> Result(a, Error), - random_ints: Iterator(Int), - interval: Int, - iterations: Int, - ) -} - -pub const one_minute: Int = 60_000 - -fn enqueue_next_rerun(state: State(a)) { - let assert iterator.Next(cooldown, acc) = iterator.step(state.random_ints) - process.send_after(state.self, state.interval + cooldown, Rerun) - State(..state, random_ints: acc) -} - -/// Repeatedly call a function, leaving `interval` milliseconds between each call. -/// When the `work` function returns an error it is printed. -pub fn retry( - do work: fn(Int) -> Result(a, Error), -) -> Result(Subject(Message), actor.StartError) { - fn() { init(one_minute, work) } - |> actor.Spec(loop: loop, init_timeout: 100) - |> actor.start_spec() -} - -fn init( - interval: Int, - work: fn(Int) -> Result(a, Error), -) -> actor.InitResult(State(a), Message) { - let subject = process.new_subject() - let random_ints = random.to_random_iterator(random.int(1000, 5000)) - let state = - State( - self: subject, - work: work, - interval: interval, - iterations: 10, - random_ints: random_ints, - ) - process.new_selector() - |> process.selecting(subject, function.identity) - |> actor.Ready(state, _) - |> function.tap(fn(_) { process.send(state.self, Rerun) }) -} - -fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) { - case message { - Rerun -> { - case state.work(state.iterations) { - Ok(_) -> actor.Stop(process.Normal) - Error(error) -> { - wisp.log_notice("Process on error") - error.log_error(error) - case state.iterations == 0 { - True -> { - wisp.log_notice("Stopping process after 10 iterations") - actor.Stop(process.Normal) - } - False -> { - State(..state, iterations: state.iterations - 1) - |> enqueue_next_rerun() - |> actor.continue() - } - } - } - } - } - } -} diff --git a/apps/backend/src/tasks/hex.gleam b/apps/backend/src/tasks/hex.gleam index 94ae811..5990189 100644 --- a/apps/backend/src/tasks/hex.gleam +++ b/apps/backend/src/tasks/hex.gleam @@ -1,14 +1,15 @@ import api/hex as api import api/hex_repo import api/signatures -import backend/config.{type Context} +import backend/context.{type Context} import backend/data/hex_read.{type HexRead} import backend/error.{type Error} -import backend/gleam/context +import backend/gleam/context as gcontext import backend/gleam/type_search/msg as type_search import backend/postgres/queries import birl.{type Time} import birl/duration +import gleam/bool import gleam/erlang/process.{type Subject} import gleam/function import gleam/hexpm.{type Package} @@ -16,11 +17,10 @@ import gleam/int import gleam/list import gleam/option.{type Option, None, Some} import gleam/order -import gleam/otp/supervisor -import gleam/pgo import gleam/result import gleam/string -import retrier +import pog +import processes/retrier import wisp type State { @@ -30,29 +30,30 @@ type State { newest: Time, hex_api_key: String, last_logged: Time, - db: pgo.Connection, + db: pog.Connection, type_search_subject: Option(Subject(type_search.Msg)), ) } -pub fn sync_new_gleam_releases( - ctx: Context, - children: supervisor.Children(Nil), -) -> Result(HexRead, Error) { +type WorkMode { + WorkAsync + WorkSync +} + +pub fn sync_new_gleam_releases(ctx: Context) -> Result(HexRead, Error) { wisp.log_info("Syncing new releases from Hex") use limit <- result.try(queries.get_last_hex_date(ctx.db)) - use latest <- result.try(sync_packages( - State( + use latest <- result.try({ + sync_packages(State( page: 1, - limit: limit, + limit:, newest: limit, hex_api_key: ctx.hex_api_key, last_logged: birl.now(), db: ctx.db, type_search_subject: ctx.type_search_subject, - ), - children, - )) + )) + }) let latest = queries.upsert_most_recent_hex_timestamp(ctx.db, latest) wisp.log_info("") wisp.log_info("Up to date!") @@ -73,10 +74,7 @@ fn first_timestamp(packages: List(hexpm.Package), state: State) -> Time { |> result.unwrap(state.newest) } -fn sync_packages( - state: State, - children: supervisor.Children(Nil), -) -> Result(Time, Error) { +fn sync_packages(state: State) -> Result(Time, Error) { let page = state.page let api_key = state.hex_api_key use all_packages <- result.try(api.get_api_packages_page(page, api_key)) @@ -84,56 +82,52 @@ fn sync_packages( let new_packages = take_fresh_packages(all_packages, state.limit) use state <- result.try({ list.try_fold(new_packages, state, { - do_sync_package(Some(children), force: False) + do_sync_package(WorkAsync, force: False) }) }) case list.length(all_packages) == list.length(new_packages) { _ if all_packages == [] -> Ok(state.newest) False -> Ok(state.newest) - True -> sync_packages(State(..state, page: state.page + 1), children) + True -> sync_packages(State(..state, page: state.page + 1)) } } +pub fn sync_package(ctx: Context, package: hexpm.Package) { + State( + page: -1, + limit: birl.now(), + newest: birl.now(), + hex_api_key: ctx.hex_api_key, + last_logged: birl.now(), + db: ctx.db, + type_search_subject: ctx.type_search_subject, + ) + |> do_sync_package(WorkSync, force: True)(package) + |> result.replace_error(error.EmptyError) + |> result.replace(Nil) +} + fn do_sync_package( - children: Option(supervisor.Children(Nil)), + work_mode work_mode: WorkMode, force force_old_release_update: Bool, ) { fn(state: State, package: hexpm.Package) -> Result(State, Error) { let secret = state.hex_api_key - use releases <- result.try(lookup_gleam_releases(package, secret: secret)) - case releases { - [] -> Ok(log_if_needed(state, package.updated_at)) - _ -> { - use _ <- result.map(insert_package_and_releases( - package, - releases, - state, - children, - force_old_release_update, - )) - State(..state, last_logged: birl.now()) - } - } + use releases <- result.try(lookup_gleam_releases(package, secret:)) + use <- bool.lazy_guard(when: list.is_empty(releases), return: fn() { + Ok(log_if_needed(state, package.updated_at)) + }) + use _ <- result.map(insert_package_and_releases( + package, + releases, + state, + work_mode, + force_old_release_update, + )) + State(..state, last_logged: birl.now()) } } -pub fn sync_package(ctx: Context, package: hexpm.Package) { - do_sync_package(None, force: True)( - State( - page: -1, - limit: birl.now(), - newest: birl.now(), - hex_api_key: ctx.hex_api_key, - last_logged: birl.now(), - db: ctx.db, - type_search_subject: ctx.type_search_subject, - ), - package, - ) - |> result.replace_error(error.EmptyError) - |> result.replace(Nil) -} - fn log_retirement_data(release: String, retirement: hexpm.ReleaseRetirement) { wisp.log_info("Release " <> release <> " is retired.") case retirement.message { @@ -169,9 +163,8 @@ fn extract_release_interfaces_from_hex( use data <- result.map({ hex_repo.get_package_infos(package.name, release.version) }) - let interface = Some(data.2) - let toml = Some(data.3) - let _ = queries.upsert_release(state.db, id, release, interface, toml) + let _ = + queries.upsert_release(state.db, id, release, Some(data.2), Some(data.3)) #(data.0, data.1) } @@ -204,8 +197,7 @@ fn save_retirement_data( option.Some(retirement) -> { let release = package.name <> " v" <> release.version log_retirement_data(release, retirement) - let _ = - queries.add_package_gleam_retirement(state.db, retirement, release_id) + let _ = queries.add_package_retirement(state.db, retirement, release_id) Nil } } @@ -215,7 +207,7 @@ fn insert_package_and_releases( package: hexpm.Package, releases: List(hexpm.Release), state: State, - children: Option(supervisor.Children(Nil)), + work_async: WorkMode, force_old_release_update: Bool, ) { let secret = state.hex_api_key @@ -246,23 +238,18 @@ fn insert_package_and_releases( wisp.log_debug("Handling release " <> r.version) use interfaces <- result.map(extract_release_interfaces_from_db(state, id, r)) save_retirement_data(state, interfaces.0, package, r) - let _ = case children { - None -> { + case work_async { + WorkSync -> { let _ = do_extract_package(state, id, r, package, interfaces, False) Nil } - Some(children) -> { - supervisor.add(children, { - use _ <- supervisor.worker() - use iterations <- retrier.retry() - let it = int.to_string(iterations) - wisp.log_notice("Trying iteration " <> it <> " for " <> release) - do_extract_package(state, id, r, package, interfaces, iterations == 0) - }) - Nil + WorkAsync -> { + use iterations <- retrier.retry + let it = int.to_string(iterations) + wisp.log_notice("Trying iteration " <> it <> " for " <> release) + do_extract_package(state, id, r, package, interfaces, iterations == 0) } } - Nil } fn do_extract_package( @@ -271,17 +258,17 @@ fn do_extract_package( release: hexpm.Release, package: hexpm.Package, interfaces: #(Int, Option(String), Option(String)), - ignore_errors: Bool, + ignore_parameters_errors: Bool, ) { - use #(package, gleam_toml) <- result.try({ + use #(package_interface, gleam_toml) <- result.try({ extract_release_interfaces(state, id, package, release, interfaces) }) - context.Context( - state.db, - package, - gleam_toml, - ignore_errors, - state.type_search_subject, + gcontext.Context( + db: state.db, + package_interface:, + gleam_toml:, + ignore_parameters_errors:, + type_search_subject: state.type_search_subject, ) |> signatures.extract_signatures() |> result.map(fn(content) { @@ -295,22 +282,21 @@ fn lookup_gleam_releases( package: hexpm.Package, secret hex_api_key: String, ) -> Result(List(hexpm.Release), Error) { - let lookup = - list.try_map(package.releases, api.lookup_release(_, hex_api_key)) - use releases <- result.map(lookup) - list.filter(releases, fn(r) { list.contains(r.meta.build_tools, "gleam") }) + package.releases + |> list.try_map(api.lookup_release(_, hex_api_key)) + |> result.map(fn(releases) { + use release <- list.filter(releases) + list.contains(release.meta.build_tools, "gleam") + }) } fn log_if_needed(state: State, time: Time) -> State { let interval = duration.new([#(5, duration.Second)]) let print_deadline = birl.add(state.last_logged, interval) - case birl.compare(print_deadline, birl.now()) == order.Lt { - False -> state - True -> { - wisp.log_info("Still syncing, up to " <> birl.to_iso8601(time)) - State(..state, last_logged: birl.now()) - } - } + let should_log = birl.compare(print_deadline, birl.now()) == order.Lt + use <- bool.guard(when: !should_log, return: state) + wisp.log_info("Still syncing, up to " <> birl.to_iso8601(time)) + State(..state, last_logged: birl.now()) } pub fn take_fresh_packages(packages: List(Package), limit: Time) { diff --git a/apps/backend/src/tasks/popularity.gleam b/apps/backend/src/tasks/popularity.gleam index 86fdd49..c2d3bb1 100644 --- a/apps/backend/src/tasks/popularity.gleam +++ b/apps/backend/src/tasks/popularity.gleam @@ -1,5 +1,5 @@ import api/github -import backend/config.{type Context} +import backend/context.{type Context} import backend/postgres/queries import gleam/bool import gleam/dict @@ -11,8 +11,8 @@ import wisp pub fn compute_popularity(ctx: Context) { case ctx.env { - config.Development -> Ok(Nil) - config.Production -> { + context.Development -> Ok(Nil) + context.Production -> { wisp.log_info("Syncing popularity") do_compute_popularity(ctx, offset: 0) |> function.tap(fn(_) { wisp.log_info("Syncing popularity finished!") }) diff --git a/apps/backend/src/tasks/ranking.gleam b/apps/backend/src/tasks/ranking.gleam index a06f06e..22fbe40 100644 --- a/apps/backend/src/tasks/ranking.gleam +++ b/apps/backend/src/tasks/ranking.gleam @@ -1,4 +1,4 @@ -import backend/config.{type Context} +import backend/context.{type Context} import backend/error.{type Error} import backend/postgres/queries import gleam/dict.{type Dict} @@ -52,7 +52,7 @@ fn get_dependencies(toml: Dict(String, tom.Toml)) { fn add_dependencies(in usages: Usages, from toml: Dict(String, tom.Toml)) { use usages, dep <- list.fold(from: usages, over: get_dependencies(toml)) - dict.update(usages, dep, fn(value) { option.unwrap(value, 0) + 1 }) + dict.upsert(usages, dep, fn(value) { option.unwrap(value, 0) + 1 }) } fn save_packages_rank(ctx: Context, usages: Usages) { diff --git a/apps/backend/src/tasks/timeseries.gleam b/apps/backend/src/tasks/timeseries.gleam index 252bed4..aaf2c5f 100644 --- a/apps/backend/src/tasks/timeseries.gleam +++ b/apps/backend/src/tasks/timeseries.gleam @@ -1,4 +1,4 @@ -import backend/config.{type Context} +import backend/context.{type Context} import backend/error import backend/postgres/queries import gleam/function @@ -10,12 +10,9 @@ pub fn store_timeseries(ctx: Context) { wisp.log_info("Storing analytics timeseries") let query = queries.select_last_day_search_analytics(ctx.db) use analytics <- result.try(query) - use _ <- function.tap({ - result.all({ - use analytic <- list.map(analytics) - queries.upsert_search_analytics_timeseries(ctx.db, analytic) - }) - |> result.map_error(error.debug_log) - }) - wisp.log_info("Storing analytics finished!") + analytics + |> list.map(queries.upsert_search_analytics_timeseries(ctx.db, _)) + |> result.all + |> result.map_error(error.debug_log) + |> function.tap(fn(_) { wisp.log_info("Storing analytics finished!") }) } diff --git a/apps/backend/src/wisp/logger.gleam b/apps/backend/src/wisp/logger.gleam index c9325a1..3208016 100644 --- a/apps/backend/src/wisp/logger.gleam +++ b/apps/backend/src/wisp/logger.gleam @@ -1,4 +1,6 @@ +import envoy import gleam +import gleam/result import gleam/string pub type Level { @@ -12,10 +14,16 @@ pub type Level { Debug } -@external(erlang, "gloogle_hex_ffi", "set_level") +@external(erlang, "backend_ffi", "set_level") pub fn set_level(level: Level) -> Nil -pub fn parse(level: String) -> Result(Level, Nil) { +pub fn read_level() { + envoy.get("LOG_LEVEL") + |> result.try(parse) + |> result.unwrap(Info) +} + +fn parse(level: String) -> Result(Level, Nil) { case string.lowercase(level) { "emergency" -> Ok(Emergency) "alert" -> Ok(Alert) diff --git a/apps/backend/test/backend_test.gleam b/apps/backend/test/backend_test.gleam index 4ad1a9e..4f2f121 100644 --- a/apps/backend/test/backend_test.gleam +++ b/apps/backend/test/backend_test.gleam @@ -1,13 +1,13 @@ import backend/gleam/parse import backend/gleam/type_search -import gleam/erlang/os +import envoy import gleam/function import gleam/int import gleam/option -import gleam/pgo import gleam/result import gleeunit import gleeunit/should +import pog const signature = "fn use_callback(a, b) -> c" @@ -18,24 +18,30 @@ pub fn main() { } fn postgres_connect() { - let host = os.get_env("POSTGRES_HOST") |> result.unwrap("localhost") - pgo.Config( - ..pgo.default_config(), + let host = envoy.get("POSTGRES_HOST") |> result.unwrap("localhost") + pog.Config( + ..pog.default_config(), host: host, database: "gloogle", user: "gloogle", password: option.Some("gloogle"), ssl: False, ) - |> pgo.connect + |> pog.connect } // gleeunit test functions end in `_test` pub fn type_search_test() { let db = postgres_connect() let index = int.random(1000) - use kind <- result.try(parse.parse_function(signature) |> result.nil_error) - use skind <- result.try(parse.parse_function(search_test) |> result.nil_error) + use kind <- result.try({ + parse.parse_function(signature) + |> result.replace_error(Nil) + }) + use skind <- result.try({ + parse.parse_function(search_test) + |> result.replace_error(Nil) + }) let search = type_search.add(type_search.empty(), kind, index) type_search.find(search, skind, db) |> function.tap(should.equal(_, Ok([index]))) diff --git a/packages/lustre/examples/99-full-stack-applications/client/src/app.gleam b/packages/lustre/examples/99-full-stack-applications/client/src/app.gleam index aae4530..9f1dc27 100644 --- a/packages/lustre/examples/99-full-stack-applications/client/src/app.gleam +++ b/packages/lustre/examples/99-full-stack-applications/client/src/app.gleam @@ -108,7 +108,7 @@ fn view_grocery_list(model: Model) -> Element(Msg) { fn view_grocery_item(name: String, quantity: Int) -> Element(Msg) { let handle_input = fn(e) { event.value(e) - |> result.nil_error + |> result.replace_error(Nil) |> result.then(int.parse) |> result.map(UserUpdatedQuantity(name, _)) |> result.replace_error([])