From eea6841a52360234f9673c0764a803f5783cb58c Mon Sep 17 00:00:00 2001
From: Guillaume Hivert <hivert.is.coming@gmail.com>
Date: Mon, 6 May 2024 10:35:22 +0200
Subject: [PATCH] feat: cap retrier to 20

Signed-off-by: Guillaume Hivert <hivert.is.coming@gmail.com>
---
 apps/backend/src/api/hex_repo.gleam           | 58 ++++++++++---------
 apps/backend/src/api/signatures.gleam         | 14 ++---
 apps/backend/src/backend.gleam                |  2 +
 apps/backend/src/backend/config.gleam         |  2 +-
 apps/backend/src/backend/error.gleam          |  2 +-
 apps/backend/src/backend/gleam/context.gleam  |  3 +
 .../src/backend/gleam/generate/types.gleam    |  8 ++-
 apps/backend/src/gling_hex_ffi.erl            | 22 ++++---
 apps/backend/src/periodic.gleam               |  2 +-
 apps/backend/src/retrier.gleam                | 38 ++++++++----
 apps/backend/src/tasks/hex.gleam              | 58 +++++++++----------
 apps/backend/src/wisp/logger.gleam            | 13 +++++
 12 files changed, 136 insertions(+), 86 deletions(-)
 create mode 100644 apps/backend/src/wisp/logger.gleam

diff --git a/apps/backend/src/api/hex_repo.gleam b/apps/backend/src/api/hex_repo.gleam
index a6bae72..95ae9d4 100644
--- a/apps/backend/src/api/hex_repo.gleam
+++ b/apps/backend/src/api/hex_repo.gleam
@@ -4,7 +4,6 @@ import gleam/http
 import gleam/http/request
 import gleam/httpc
 import gleam/json
-import gleam/option
 import gleam/package_interface
 import gleam/result
 import simplifile
@@ -19,7 +18,7 @@ fn extract_tar(
   tarbin: BitArray,
   base_name: String,
   slug: String,
-) -> #(String, String)
+) -> Result(#(String, String, String), Nil)
 
 @external(erlang, "gling_hex_ffi", "remove_tar")
 fn remove_tar(slug: String) -> Nil
@@ -41,7 +40,7 @@ fn read_archive(archives_path: String, name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   let filepath = archives_path <> "/" <> name <> "/" <> slug
   use content <- result.map(simplifile.read_bits(filepath))
-  wisp.log_info("Using filesystem for " <> slug)
+  wisp.log_debug("Using filesystem for " <> slug)
   content
 }
 
@@ -63,7 +62,7 @@ fn get_tarball(name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   use archives_path <- result.try(create_archives_directory())
   use _ <- result.try_recover(read_archive(archives_path, name, version))
-  wisp.log_info("Querying hex for " <> slug)
+  wisp.log_debug("Querying hex for " <> slug)
   request.new()
   |> request.set_host("repo.hex.pm")
   |> request.set_path("/tarballs/" <> slug)
@@ -77,45 +76,52 @@ fn get_tarball(name: String, version: String) {
   })
 }
 
-fn read_interface(filepath: String) {
-  case simplifile.read(filepath) {
-    Ok(interface) -> Ok(option.Some(interface))
-    Error(_) -> Ok(option.None)
-  }
+fn read_interface(filepath: String, artifacts: String) {
+  filepath
+  |> simplifile.read()
+  |> result.map_error(fn(error) {
+    wisp.log_warning("Unable to read " <> filepath)
+    wisp.log_warning("Compilation artifacts:")
+    wisp.log_warning(artifacts)
+    error.SimplifileError(error, filepath)
+  })
 }
 
-fn read_file(filepath: String) {
+fn read_toml_file(filepath: String) {
   filepath
   |> simplifile.read()
   |> result.map_error(error.SimplifileError(_, filepath))
 }
 
-fn read_package_interface(blob: option.Option(String)) {
-  case blob {
-    option.None -> Ok(option.None)
-    option.Some(blob) ->
-      blob
-      |> json.decode(using: package_interface.decoder)
-      |> result.map_error(error.JsonError)
-      |> result.map(option.Some)
-  }
+fn read_package_interface(blob: String) {
+  blob
+  |> json.decode(using: package_interface.decoder)
+  |> result.map_error(error.JsonError)
 }
 
-fn read_gleam_toml(blob: String) {
-  blob
-  |> tom.parse()
+fn parse_toml(toml_blob: String) {
+  tom.parse(toml_blob)
   |> result.map_error(error.ParseTomlError)
 }
 
 fn extract_package_infos(name: String, version: String) {
+  let package_name = name <> "@" <> version
   let slug = package_slug(name, version)
   let req = get_tarball(name, version)
   use body <- result.try(req)
-  let #(interface, toml) = extract_tar(body, name, slug)
-  use interface_blob <- result.try(read_interface(interface))
-  use toml_blob <- result.try(read_file(toml))
+  use #(interface, toml, res) <- result.try({
+    body
+    |> extract_tar(name, slug)
+    |> result.map_error(fn(_) {
+      let content = "Impossible to extract tar for " <> package_name
+      wisp.log_warning(content)
+      error.UnknownError(content)
+    })
+  })
+  use interface_blob <- result.try(read_interface(interface, res))
+  use toml_blob <- result.try(read_toml_file(toml))
   use interface <- result.try(read_package_interface(interface_blob))
-  use toml <- result.map(read_gleam_toml(toml_blob))
+  use toml <- result.map(parse_toml(toml_blob))
   #(interface, toml)
 }
 
diff --git a/apps/backend/src/api/signatures.gleam b/apps/backend/src/api/signatures.gleam
index 7e9cedd..64a1fbb 100644
--- a/apps/backend/src/api/signatures.gleam
+++ b/apps/backend/src/api/signatures.gleam
@@ -27,7 +27,7 @@ fn add_gleam_constraint(ctx: Context, release_id: Int) {
 
 fn upsert_type_definitions(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " type definitions")
+  wisp.log_debug("Extracting " <> name <> " type definitions")
   let all_types = dict.to_list(module.module.types)
   result.all({
     use #(type_name, type_def) <- list.map(all_types)
@@ -65,7 +65,7 @@ fn upsert_type_definitions(ctx: Context, module: context.Module) {
 
 fn upsert_type_aliases(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " type aliases")
+  wisp.log_debug("Extracting " <> name <> " type aliases")
   let all_types = dict.to_list(module.module.type_aliases)
   result.all({
     use #(type_name, type_alias) <- list.map(all_types)
@@ -103,7 +103,7 @@ fn upsert_type_aliases(ctx: Context, module: context.Module) {
 
 fn upsert_constants(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " constants")
+  wisp.log_debug("Extracting " <> name <> " constants")
   let all_constants = dict.to_list(module.module.constants)
   result.all({
     use #(constant_name, constant) <- list.map(all_constants)
@@ -127,7 +127,7 @@ fn upsert_constants(ctx: Context, module: context.Module) {
 
 fn upsert_functions(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " functions")
+  wisp.log_debug("Extracting " <> name <> " functions")
   let all_functions = dict.to_list(module.module.functions)
   result.all({
     use #(function_name, function) <- list.map(all_functions)
@@ -156,7 +156,7 @@ fn extract_module_signatures(
 ) {
   let module = context.Module(module.1, -1, module.0, release_id)
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " signatures")
+  wisp.log_debug("Extracting " <> name <> " signatures")
   use module_id <- result.try(queries.upsert_package_module(ctx.db, module))
   let module = context.Module(..module, id: module_id)
   use _ <- result.try(upsert_type_definitions(ctx, module))
@@ -164,14 +164,14 @@ fn extract_module_signatures(
   use _ <- result.try(upsert_constants(ctx, module))
   let res = upsert_functions(ctx, module)
   use <- bool.guard(when: result.is_error(res), return: res)
-  wisp.log_info("Extracting " <> name <> " finished")
+  wisp.log_debug("Extracting " <> name <> " finished")
   res
 }
 
 pub fn extract_signatures(ctx: Context) {
   let package = ctx.package_interface
   let package_slug = package.name <> "@" <> package.version
-  wisp.log_info("Extracting signatures for " <> package_slug)
+  wisp.log_debug("Extracting signatures for " <> package_slug)
   let res = queries.get_package_release_ids(ctx.db, ctx.package_interface)
   use #(_pid, release_id) <- result.try(res)
   use _ <- result.try(add_gleam_constraint(ctx, release_id))
diff --git a/apps/backend/src/backend.gleam b/apps/backend/src/backend.gleam
index f66b265..681bbde 100644
--- a/apps/backend/src/backend.gleam
+++ b/apps/backend/src/backend.gleam
@@ -8,11 +8,13 @@ import periodic
 import setup
 import tasks/hex
 import wisp
+import wisp/logger
 
 pub fn main() {
   dot_env.load()
   setup.radiate()
   wisp.configure_logger()
+  logger.set_level(logger.Debug)
 
   let secret_key_base = config.get_secret_key_base()
   let cnf = config.read_config()
diff --git a/apps/backend/src/backend/config.gleam b/apps/backend/src/backend/config.gleam
index 51c38ac..138f00b 100644
--- a/apps/backend/src/backend/config.gleam
+++ b/apps/backend/src/backend/config.gleam
@@ -3,7 +3,7 @@ import gleam/pgo
 import wisp
 
 pub type Context {
-  Context(connection: pgo.Connection)
+  Context(db: pgo.Connection)
 }
 
 pub type Config {
diff --git a/apps/backend/src/backend/error.gleam b/apps/backend/src/backend/error.gleam
index bccb7b5..3790ce6 100644
--- a/apps/backend/src/backend/error.gleam
+++ b/apps/backend/src/backend/error.gleam
@@ -47,7 +47,7 @@ pub fn log_decode_error(error: json.DecodeError) {
   }
 }
 
-pub fn log(error: Error) {
+pub fn log_error(error: Error) {
   case error {
     FetchError(_dyn) -> wisp.log_warning("Fetch error")
     DatabaseError(error) -> {
diff --git a/apps/backend/src/backend/gleam/context.gleam b/apps/backend/src/backend/gleam/context.gleam
index 80534df..1c1238e 100644
--- a/apps/backend/src/backend/gleam/context.gleam
+++ b/apps/backend/src/backend/gleam/context.gleam
@@ -8,6 +8,9 @@ pub type Context {
     db: pgo.Connection,
     package_interface: package_interface.Package,
     gleam_toml: Dict(String, tom.Toml),
+    /// Allow to bypass parameters relations if activated.
+    /// This allows to ignore internals for example.
+    ignore_parameters_errors: Bool,
   )
 }
 
diff --git a/apps/backend/src/backend/gleam/generate/types.gleam b/apps/backend/src/backend/gleam/generate/types.gleam
index 8396c55..9f7dcf6 100644
--- a/apps/backend/src/backend/gleam/generate/types.gleam
+++ b/apps/backend/src/backend/gleam/generate/types.gleam
@@ -255,7 +255,13 @@ fn extract_parameters_relation(
   use <- bool.guard(when: is_prelude(package, module), return: Ok(option.None))
   use requirement <- result.try(toml.find_package_requirement(ctx, package))
   use releases <- result.try(find_package_release(ctx, package, requirement))
-  find_type_signature(ctx, name, package, module, releases)
+  use error <- result.try_recover({
+    find_type_signature(ctx, name, package, module, releases)
+  })
+  case ctx.ignore_parameters_errors {
+    False -> Error(error)
+    True -> Ok(option.None)
+  }
 }
 
 fn is_prelude(package: String, module: String) {
diff --git a/apps/backend/src/gling_hex_ffi.erl b/apps/backend/src/gling_hex_ffi.erl
index d3247ff..73f43b6 100644
--- a/apps/backend/src/gling_hex_ffi.erl
+++ b/apps/backend/src/gling_hex_ffi.erl
@@ -1,5 +1,5 @@
 -module(gling_hex_ffi).
--export([extract_tar/3, remove_tar/1, is_match/2, get_home/0]).
+-export([extract_tar/3, remove_tar/1, is_match/2, get_home/0, set_level/1]).
 
 package_interface_path(ContentDest, BaseName) ->
   BuildFolder = <<"/build/dev/docs/">>,
@@ -13,13 +13,16 @@ extract_tar(Binary, BaseName, Slug) ->
   PackagePath = <<"/tmp/", Slug/binary>>,
   ContentDest = <<PackagePath/binary, "/contents">>,
   Content = <<PackagePath/binary, "/contents.tar.gz">>,
-  erl_tar:extract({binary, Binary}, [{cwd, PackagePath}]),
-  erl_tar:extract(Content, [{cwd, ContentDest}, compressed]),
-  BuildCmd = <<"cd ", ContentDest/binary, " && gleam docs build">>,
-  os:cmd(binary_to_list(BuildCmd)),
-  PackageInterface = package_interface_path(ContentDest, BaseName),
-  GleamToml = unicode:characters_to_binary(<<ContentDest/binary, "/gleam.toml">>),
-  {PackageInterface, GleamToml}.
+  case erl_tar:extract({binary, Binary}, [{cwd, PackagePath}]) of
+    {error, _} -> {error, nil};
+    _ ->
+      erl_tar:extract(Content, [{cwd, ContentDest}, compressed]),
+      BuildCmd = <<"cd ", ContentDest/binary, " && gleam docs build">>,
+      Result = os:cmd(binary_to_list(BuildCmd)),
+      PackageInterface = package_interface_path(ContentDest, BaseName),
+      GleamToml = unicode:characters_to_binary(<<ContentDest/binary, "/gleam.toml">>),
+      {ok, {PackageInterface, GleamToml, unicode:characters_to_binary(Result)}}
+  end.
 
 % Suppress the tarball.
 remove_tar(Slug) ->
@@ -38,3 +41,6 @@ get_home() ->
     {ok, Content} -> {ok, unicode:characters_to_binary(Content)};
     error -> {error, nil}
   end.
+
+set_level(Level) ->
+  logger:set_primary_config(level, Level).
diff --git a/apps/backend/src/periodic.gleam b/apps/backend/src/periodic.gleam
index fd44392..b7a3bc1 100644
--- a/apps/backend/src/periodic.gleam
+++ b/apps/backend/src/periodic.gleam
@@ -37,7 +37,7 @@ fn init(
   process.new_selector()
   |> process.selecting(subject, function.identity)
   |> actor.Ready(state, _)
-  |> function.tap(fn(_) { enqueue_next_rerun(state) })
+  |> function.tap(fn(_) { process.send(state.self, Rerun) })
 }
 
 fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) {
diff --git a/apps/backend/src/retrier.gleam b/apps/backend/src/retrier.gleam
index 24370bb..58ccb62 100644
--- a/apps/backend/src/retrier.gleam
+++ b/apps/backend/src/retrier.gleam
@@ -2,15 +2,23 @@ import backend/error.{type Error}
 import gleam/erlang/process.{type Subject}
 import gleam/function
 import gleam/otp/actor
+import wisp
 
 pub opaque type Message {
   Rerun
 }
 
 type State(a) {
-  State(self: Subject(Message), work: fn() -> Result(a, Error), interval: Int)
+  State(
+    self: Subject(Message),
+    work: fn(Int) -> Result(a, Error),
+    interval: Int,
+    iterations: Int,
+  )
 }
 
+pub const ten_minutes: Int = 600_000
+
 fn enqueue_next_rerun(state: State(a)) {
   process.send_after(state.self, state.interval, Rerun)
 }
@@ -18,19 +26,19 @@ fn enqueue_next_rerun(state: State(a)) {
 /// Repeatedly call a function, leaving `interval` milliseconds between each call.
 /// When the `work` function returns an error it is printed.
 pub fn retry(
-  do work: fn() -> Result(a, Error),
+  do work: fn(Int) -> Result(a, Error),
 ) -> Result(Subject(Message), actor.StartError) {
-  fn() { init(120_000, work) }
+  fn() { init(ten_minutes, work) }
   |> actor.Spec(loop: loop, init_timeout: 100)
   |> actor.start_spec()
 }
 
 fn init(
   interval: Int,
-  work: fn() -> Result(a, Error),
+  work: fn(Int) -> Result(a, Error),
 ) -> actor.InitResult(State(a), Message) {
   let subject = process.new_subject()
-  let state = State(subject, work, interval)
+  let state = State(subject, work, interval, 20)
   process.new_selector()
   |> process.selecting(subject, function.identity)
   |> actor.Ready(state, _)
@@ -40,12 +48,22 @@ fn init(
 fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) {
   case message {
     Rerun -> {
-      case state.work() {
+      case state.work(state.iterations) {
         Ok(_) -> actor.Stop(process.Normal)
-        Error(e) -> {
-          error.log(e)
-          enqueue_next_rerun(state)
-          actor.continue(state)
+        Error(error) -> {
+          wisp.log_notice("Process on error")
+          error.log_error(error)
+          case state.iterations == 0 {
+            True -> {
+              wisp.log_notice("Stopping process after 20 iterations")
+              actor.Stop(process.Normal)
+            }
+            False -> {
+              let new_state = State(..state, iterations: state.iterations - 1)
+              enqueue_next_rerun(new_state)
+              actor.continue(new_state)
+            }
+          }
         }
       }
     }
diff --git a/apps/backend/src/tasks/hex.gleam b/apps/backend/src/tasks/hex.gleam
index ba6ae4a..680d83a 100644
--- a/apps/backend/src/tasks/hex.gleam
+++ b/apps/backend/src/tasks/hex.gleam
@@ -37,7 +37,7 @@ pub fn sync_new_gleam_releases(
 ) -> Result(HexRead, Error) {
   let ctx = postgres.connect(cnf)
   wisp.log_info("Syncing new releases from Hex")
-  use limit <- result.try(queries.get_last_hex_date(ctx.connection))
+  use limit <- result.try(queries.get_last_hex_date(ctx.db))
   use latest <- result.try(sync_packages(
     State(
       page: 1,
@@ -45,11 +45,11 @@ pub fn sync_new_gleam_releases(
       newest: limit,
       hex_api_key: cnf.hex_api_key,
       last_logged: birl.now(),
-      db: ctx.connection,
+      db: ctx.db,
     ),
     children,
   ))
-  let latest = queries.upsert_most_recent_hex_timestamp(ctx.connection, latest)
+  let latest = queries.upsert_most_recent_hex_timestamp(ctx.db, latest)
   wisp.log_info("\nUp to date!")
   latest
 }
@@ -108,6 +108,21 @@ fn sync_package(children: supervisor.Children(Nil)) {
   }
 }
 
+fn log_retirement_data(release: String, retirement: hexpm.ReleaseRetirement) {
+  wisp.log_info("Release " <> release <> " is retired. Skipping.")
+  case retirement.message {
+    option.Some(m) -> wisp.log_debug("  Retired because " <> m)
+    option.None -> Nil
+  }
+  case retirement.reason {
+    hexpm.OtherReason -> wisp.log_debug("  Retired for an other reason")
+    hexpm.Invalid -> wisp.log_debug("  Retired because it was invalid")
+    hexpm.Security -> wisp.log_debug("  Retired for security reasons")
+    hexpm.Deprecated -> wisp.log_debug("  Retired because it's deprecated")
+    hexpm.Renamed -> wisp.log_debug("  Retired because it's renamed")
+  }
+}
+
 fn insert_package_and_releases(
   package: hexpm.Package,
   releases: List(hexpm.Release),
@@ -121,42 +136,23 @@ fn insert_package_and_releases(
     |> string.join(", v")
   wisp.log_info("Saving " <> package.name <> " v" <> versions)
   use id <- result.try(queries.upsert_package(state.db, package))
-  wisp.log_info("Saving owners for " <> package.name)
+  wisp.log_debug("Saving owners for " <> package.name)
   use owners <- result.try(api.get_package_owners(package.name, secret: secret))
   use _ <- result.try(queries.sync_package_owners(state.db, id, owners))
-  wisp.log_info("Saving releases for " <> package.name)
+  wisp.log_debug("Saving releases for " <> package.name)
   list.try_each(releases, fn(r) {
+    let release = package.name <> " v" <> r.version
     use _ <- result.map(queries.upsert_release(state.db, id, r))
     case r.retirement {
-      option.Some(retirement) -> {
-        let release = package.name <> " v" <> r.version
-        wisp.log_info("Release " <> release <> " is retired. Skipping.")
-        case retirement.message {
-          option.None -> Nil
-          option.Some(m) -> wisp.log_info("  Retired because " <> m)
-        }
-        case retirement.reason {
-          hexpm.OtherReason -> wisp.log_info("  Retired for an other reason")
-          hexpm.Invalid -> wisp.log_info("  Retired because it was invalid")
-          hexpm.Security -> wisp.log_info("  Retired for security reasons")
-          hexpm.Deprecated -> wisp.log_info("  Retired because it's deprecated")
-          hexpm.Renamed -> wisp.log_info("  Retired because it's renamed")
-        }
-      }
+      option.Some(retirement) -> log_retirement_data(release, retirement)
       option.None -> {
         supervisor.add(children, {
           use _ <- supervisor.worker()
-          retrier.retry(fn() {
-            let infos = hex_repo.get_package_infos(package.name, r.version)
-            use #(package, gleam_toml) <- result.try(infos)
-            case package {
-              option.None -> Ok([])
-              option.Some(package) -> {
-                let ctx = context.Context(state.db, package, gleam_toml)
-                signatures.extract_signatures(ctx)
-              }
-            }
-          })
+          use iterations <- retrier.retry()
+          let infos = hex_repo.get_package_infos(package.name, r.version)
+          use #(package, gleam_toml) <- result.try(infos)
+          context.Context(state.db, package, gleam_toml, iterations == 0)
+          |> signatures.extract_signatures()
         })
         Nil
       }
diff --git a/apps/backend/src/wisp/logger.gleam b/apps/backend/src/wisp/logger.gleam
new file mode 100644
index 0000000..008ad98
--- /dev/null
+++ b/apps/backend/src/wisp/logger.gleam
@@ -0,0 +1,13 @@
+pub type Level {
+  Emergency
+  Alert
+  Critical
+  Error
+  Warning
+  Notice
+  Info
+  Debug
+}
+
+@external(erlang, "gling_hex_ffi", "set_level")
+pub fn set_level(level: Level) -> Nil