feat: cap retrier to 20

Signed-off-by: Guillaume Hivert <hivert.is.coming@gmail.com>

apps/backend/src/api/hex_repo.gleam

@@ -4,7 +4,6 @@ import gleam/http
 import gleam/http/request
 import gleam/httpc
 import gleam/json
-import gleam/option
 import gleam/package_interface
 import gleam/result
 import simplifile
@@ -19,7 +18,7 @@ fn extract_tar(
   tarbin: BitArray,
   base_name: String,
   slug: String,
-) -> #(String, String)
+) -> Result(#(String, String, String), Nil)
 
 @external(erlang, "gling_hex_ffi", "remove_tar")
 fn remove_tar(slug: String) -> Nil
@@ -41,7 +40,7 @@ fn read_archive(archives_path: String, name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   let filepath = archives_path <> "/" <> name <> "/" <> slug
   use content <- result.map(simplifile.read_bits(filepath))
-  wisp.log_info("Using filesystem for " <> slug)
+  wisp.log_debug("Using filesystem for " <> slug)
   content
 }
 
@@ -63,7 +62,7 @@ fn get_tarball(name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   use archives_path <- result.try(create_archives_directory())
   use _ <- result.try_recover(read_archive(archives_path, name, version))
-  wisp.log_info("Querying hex for " <> slug)
+  wisp.log_debug("Querying hex for " <> slug)
   request.new()
   |> request.set_host("repo.hex.pm")
   |> request.set_path("/tarballs/" <> slug)
@@ -77,45 +76,52 @@ fn get_tarball(name: String, version: String) {
   })
 }
 
-fn read_interface(filepath: String) {
-  case simplifile.read(filepath) {
-    Ok(interface) -> Ok(option.Some(interface))
-    Error(_) -> Ok(option.None)
-  }
+fn read_interface(filepath: String, artifacts: String) {
+  filepath
+  |> simplifile.read()
+  |> result.map_error(fn(error) {
+    wisp.log_warning("Unable to read " <> filepath)
+    wisp.log_warning("Compilation artifacts:")
+    wisp.log_warning(artifacts)
+    error.SimplifileError(error, filepath)
+  })
 }
 
-fn read_file(filepath: String) {
+fn read_toml_file(filepath: String) {
   filepath
   |> simplifile.read()
   |> result.map_error(error.SimplifileError(_, filepath))
 }
 
-fn read_package_interface(blob: option.Option(String)) {
-  case blob {
-    option.None -> Ok(option.None)
-    option.Some(blob) ->
-      blob
-      |> json.decode(using: package_interface.decoder)
-      |> result.map_error(error.JsonError)
-      |> result.map(option.Some)
-  }
+fn read_package_interface(blob: String) {
+  blob
+  |> json.decode(using: package_interface.decoder)
+  |> result.map_error(error.JsonError)
 }
 
-fn read_gleam_toml(blob: String) {
-  blob
-  |> tom.parse()
+fn parse_toml(toml_blob: String) {
+  tom.parse(toml_blob)
   |> result.map_error(error.ParseTomlError)
 }
 
 fn extract_package_infos(name: String, version: String) {
+  let package_name = name <> "@" <> version
   let slug = package_slug(name, version)
   let req = get_tarball(name, version)
   use body <- result.try(req)
-  let #(interface, toml) = extract_tar(body, name, slug)
-  use interface_blob <- result.try(read_interface(interface))
-  use toml_blob <- result.try(read_file(toml))
+  use #(interface, toml, res) <- result.try({
+    body
+    |> extract_tar(name, slug)
+    |> result.map_error(fn(_) {
+      let content = "Impossible to extract tar for " <> package_name
+      wisp.log_warning(content)
+      error.UnknownError(content)
+    })
+  })
+  use interface_blob <- result.try(read_interface(interface, res))
+  use toml_blob <- result.try(read_toml_file(toml))
   use interface <- result.try(read_package_interface(interface_blob))
-  use toml <- result.map(read_gleam_toml(toml_blob))
+  use toml <- result.map(parse_toml(toml_blob))
   #(interface, toml)
 }
 

apps/backend/src/api/signatures.gleam

@@ -27,7 +27,7 @@ fn add_gleam_constraint(ctx: Context, release_id: Int) {
 
 fn upsert_type_definitions(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " type definitions")
+  wisp.log_debug("Extracting " <> name <> " type definitions")
   let all_types = dict.to_list(module.module.types)
   result.all({
     use #(type_name, type_def) <- list.map(all_types)
@@ -65,7 +65,7 @@ fn upsert_type_definitions(ctx: Context, module: context.Module) {
 
 fn upsert_type_aliases(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " type aliases")
+  wisp.log_debug("Extracting " <> name <> " type aliases")
   let all_types = dict.to_list(module.module.type_aliases)
   result.all({
     use #(type_name, type_alias) <- list.map(all_types)
@@ -103,7 +103,7 @@ fn upsert_type_aliases(ctx: Context, module: context.Module) {
 
 fn upsert_constants(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " constants")
+  wisp.log_debug("Extracting " <> name <> " constants")
   let all_constants = dict.to_list(module.module.constants)
   result.all({
     use #(constant_name, constant) <- list.map(all_constants)
@@ -127,7 +127,7 @@ fn upsert_constants(ctx: Context, module: context.Module) {
 
 fn upsert_functions(ctx: Context, module: context.Module) {
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " functions")
+  wisp.log_debug("Extracting " <> name <> " functions")
   let all_functions = dict.to_list(module.module.functions)
   result.all({
     use #(function_name, function) <- list.map(all_functions)
@@ -156,22 +156,22 @@ fn extract_module_signatures(
 ) {
   let module = context.Module(module.1, -1, module.0, release_id)
   let name = context.qualified_name(ctx, module)
-  wisp.log_info("Extracting " <> name <> " signatures")
+  wisp.log_debug("Extracting " <> name <> " signatures")
   use module_id <- result.try(queries.upsert_package_module(ctx.db, module))
   let module = context.Module(..module, id: module_id)
   use _ <- result.try(upsert_type_definitions(ctx, module))
   use _ <- result.try(upsert_type_aliases(ctx, module))
   use _ <- result.try(upsert_constants(ctx, module))
   let res = upsert_functions(ctx, module)
   use <- bool.guard(when: result.is_error(res), return: res)
-  wisp.log_info("Extracting " <> name <> " finished")
+  wisp.log_debug("Extracting " <> name <> " finished")
   res
 }
 
 pub fn extract_signatures(ctx: Context) {
   let package = ctx.package_interface
   let package_slug = package.name <> "@" <> package.version
-  wisp.log_info("Extracting signatures for " <> package_slug)
+  wisp.log_debug("Extracting signatures for " <> package_slug)
   let res = queries.get_package_release_ids(ctx.db, ctx.package_interface)
   use #(_pid, release_id) <- result.try(res)
   use _ <- result.try(add_gleam_constraint(ctx, release_id))

apps/backend/src/backend.gleam

@@ -8,11 +8,13 @@ import periodic
 import setup
 import tasks/hex
 import wisp
+import wisp/logger
 
 pub fn main() {
   dot_env.load()
   setup.radiate()
   wisp.configure_logger()
+  logger.set_level(logger.Debug)
 
   let secret_key_base = config.get_secret_key_base()
   let cnf = config.read_config()

apps/backend/src/backend/config.gleam

@@ -3,7 +3,7 @@ import gleam/pgo
 import wisp
 
 pub type Context {
-  Context(connection: pgo.Connection)
+  Context(db: pgo.Connection)
 }
 
 pub type Config {

apps/backend/src/backend/error.gleam

@@ -47,7 +47,7 @@ pub fn log_decode_error(error: json.DecodeError) {
   }
 }
 
-pub fn log(error: Error) {
+pub fn log_error(error: Error) {
   case error {
     FetchError(_dyn) -> wisp.log_warning("Fetch error")
     DatabaseError(error) -> {

apps/backend/src/backend/gleam/context.gleam

@@ -8,6 +8,9 @@ pub type Context {
     db: pgo.Connection,
     package_interface: package_interface.Package,
     gleam_toml: Dict(String, tom.Toml),
+    /// Allow to bypass parameters relations if activated.
+    /// This allows to ignore internals for example.
+    ignore_parameters_errors: Bool,
   )
 }
 

apps/backend/src/backend/gleam/generate/types.gleam

@@ -255,7 +255,13 @@ fn extract_parameters_relation(
   use <- bool.guard(when: is_prelude(package, module), return: Ok(option.None))
   use requirement <- result.try(toml.find_package_requirement(ctx, package))
   use releases <- result.try(find_package_release(ctx, package, requirement))
-  find_type_signature(ctx, name, package, module, releases)
+  use error <- result.try_recover({
+    find_type_signature(ctx, name, package, module, releases)
+  })
+  case ctx.ignore_parameters_errors {
+    False -> Error(error)
+    True -> Ok(option.None)
+  }
 }
 
 fn is_prelude(package: String, module: String) {

apps/backend/src/gling_hex_ffi.erl

@@ -1,5 +1,5 @@
 -module(gling_hex_ffi).
--export([extract_tar/3, remove_tar/1, is_match/2, get_home/0]).
+-export([extract_tar/3, remove_tar/1, is_match/2, get_home/0, set_level/1]).
 
 package_interface_path(ContentDest, BaseName) ->
   BuildFolder = <<"/build/dev/docs/">>,
@@ -13,13 +13,16 @@ extract_tar(Binary, BaseName, Slug) ->
   PackagePath = <<"/tmp/", Slug/binary>>,
   ContentDest = <<PackagePath/binary, "/contents">>,
   Content = <<PackagePath/binary, "/contents.tar.gz">>,
-  erl_tar:extract({binary, Binary}, [{cwd, PackagePath}]),
-  erl_tar:extract(Content, [{cwd, ContentDest}, compressed]),
-  BuildCmd = <<"cd ", ContentDest/binary, " && gleam docs build">>,
-  os:cmd(binary_to_list(BuildCmd)),
-  PackageInterface = package_interface_path(ContentDest, BaseName),
-  GleamToml = unicode:characters_to_binary(<<ContentDest/binary, "/gleam.toml">>),
-  {PackageInterface, GleamToml}.
+  case erl_tar:extract({binary, Binary}, [{cwd, PackagePath}]) of
+    {error, _} -> {error, nil};
+    _ ->
+      erl_tar:extract(Content, [{cwd, ContentDest}, compressed]),
+      BuildCmd = <<"cd ", ContentDest/binary, " && gleam docs build">>,
+      Result = os:cmd(binary_to_list(BuildCmd)),
+      PackageInterface = package_interface_path(ContentDest, BaseName),
+      GleamToml = unicode:characters_to_binary(<<ContentDest/binary, "/gleam.toml">>),
+      {ok, {PackageInterface, GleamToml, unicode:characters_to_binary(Result)}}
+  end.
 
 % Suppress the tarball.
 remove_tar(Slug) ->
@@ -38,3 +41,6 @@ get_home() ->
     {ok, Content} -> {ok, unicode:characters_to_binary(Content)};
     error -> {error, nil}
   end.
+
+set_level(Level) ->
+  logger:set_primary_config(level, Level).

apps/backend/src/periodic.gleam

@@ -37,7 +37,7 @@ fn init(
   process.new_selector()
   |> process.selecting(subject, function.identity)
   |> actor.Ready(state, _)
-  |> function.tap(fn(_) { enqueue_next_rerun(state) })
+  |> function.tap(fn(_) { process.send(state.self, Rerun) })
 }
 
 fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) {

apps/backend/src/retrier.gleam

@@ -2,35 +2,43 @@ import backend/error.{type Error}
 import gleam/erlang/process.{type Subject}
 import gleam/function
 import gleam/otp/actor
+import wisp
 
 pub opaque type Message {
   Rerun
 }
 
 type State(a) {
-  State(self: Subject(Message), work: fn() -> Result(a, Error), interval: Int)
+  State(
+    self: Subject(Message),
+    work: fn(Int) -> Result(a, Error),
+    interval: Int,
+    iterations: Int,
+  )
 }
 
+pub const ten_minutes: Int = 600_000
+
 fn enqueue_next_rerun(state: State(a)) {
   process.send_after(state.self, state.interval, Rerun)
 }
 
 /// Repeatedly call a function, leaving `interval` milliseconds between each call.
 /// When the `work` function returns an error it is printed.
 pub fn retry(
-  do work: fn() -> Result(a, Error),
+  do work: fn(Int) -> Result(a, Error),
 ) -> Result(Subject(Message), actor.StartError) {
-  fn() { init(120_000, work) }
+  fn() { init(ten_minutes, work) }
   |> actor.Spec(loop: loop, init_timeout: 100)
   |> actor.start_spec()
 }
 
 fn init(
   interval: Int,
-  work: fn() -> Result(a, Error),
+  work: fn(Int) -> Result(a, Error),
 ) -> actor.InitResult(State(a), Message) {
   let subject = process.new_subject()
-  let state = State(subject, work, interval)
+  let state = State(subject, work, interval, 20)
   process.new_selector()
   |> process.selecting(subject, function.identity)
   |> actor.Ready(state, _)
@@ -40,12 +48,22 @@ fn init(
 fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) {
   case message {
     Rerun -> {
-      case state.work() {
+      case state.work(state.iterations) {
         Ok(_) -> actor.Stop(process.Normal)
-        Error(e) -> {
-          error.log(e)
-          enqueue_next_rerun(state)
-          actor.continue(state)
+        Error(error) -> {
+          wisp.log_notice("Process on error")
+          error.log_error(error)
+          case state.iterations == 0 {
+            True -> {
+              wisp.log_notice("Stopping process after 20 iterations")
+              actor.Stop(process.Normal)
+            }
+            False -> {
+              let new_state = State(..state, iterations: state.iterations - 1)
+              enqueue_next_rerun(new_state)
+              actor.continue(new_state)
+            }
+          }
         }
       }
     }