From 58a8d4c55992c103569d65d23729096f1b092745 Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Sun, 12 May 2024 23:47:14 +0200 Subject: [PATCH] feat: rename nature to kind Signed-off-by: Guillaume Hivert --- ...r_table_package_release_add_retirement.sql | 7 ++ ..._package_release_rename_nature_to_kind.sql | 7 ++ apps/backend/db/schema.sql | 9 +- apps/backend/src/api/signatures.gleam | 12 +- .../src/backend/gleam/generate/types.gleam | 22 ++-- .../src/backend/postgres/queries.gleam | 77 ++++++++---- apps/backend/src/tasks/hex.gleam | 113 ++++++++++-------- .../src/data/decoders/search_result.gleam | 2 +- .../src/data/decoders/signature.gleam | 7 +- 9 files changed, 158 insertions(+), 98 deletions(-) create mode 100644 apps/backend/db/migrations/20240512211227_alter_table_package_release_add_retirement.sql create mode 100644 apps/backend/db/migrations/20240512214036_alter_table_package_release_rename_nature_to_kind.sql diff --git a/apps/backend/db/migrations/20240512211227_alter_table_package_release_add_retirement.sql b/apps/backend/db/migrations/20240512211227_alter_table_package_release_add_retirement.sql new file mode 100644 index 0000000..a0ca938 --- /dev/null +++ b/apps/backend/db/migrations/20240512211227_alter_table_package_release_add_retirement.sql @@ -0,0 +1,7 @@ +-- migrate:up +alter table package_release + add column retirement jsonb; + +-- migrate:down +alter table package_release + drop column retirement; diff --git a/apps/backend/db/migrations/20240512214036_alter_table_package_release_rename_nature_to_kind.sql b/apps/backend/db/migrations/20240512214036_alter_table_package_release_rename_nature_to_kind.sql new file mode 100644 index 0000000..6ef6ec2 --- /dev/null +++ b/apps/backend/db/migrations/20240512214036_alter_table_package_release_rename_nature_to_kind.sql @@ -0,0 +1,7 @@ +-- migrate:up +alter table package_type_fun_signature + rename column nature to kind; + +-- migrate:down +alter table package_type_fun_signature + rename column kind to nature; diff --git a/apps/backend/db/schema.sql b/apps/backend/db/schema.sql index a48b7e1..ddb7fac 100644 --- a/apps/backend/db/schema.sql +++ b/apps/backend/db/schema.sql @@ -190,7 +190,8 @@ CREATE TABLE public.package_release ( created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, package_interface text, - gleam_toml text + gleam_toml text, + retirement jsonb ); @@ -218,7 +219,7 @@ CREATE TABLE public.package_type_fun_signature ( documentation text NOT NULL, signature_ text NOT NULL, json_signature jsonb NOT NULL, - nature public.type_nature NOT NULL, + kind public.type_nature NOT NULL, parameters integer[] NOT NULL, deprecation text, implementations text, @@ -456,4 +457,6 @@ INSERT INTO public.schema_migrations (version) VALUES ('20240412155056'), ('20240412155057'), ('20240413164020'), - ('20240506110519'); + ('20240506110519'), + ('20240512211227'), + ('20240512214036'); diff --git a/apps/backend/src/api/signatures.gleam b/apps/backend/src/api/signatures.gleam index 64a1fbb..73789fb 100644 --- a/apps/backend/src/api/signatures.gleam +++ b/apps/backend/src/api/signatures.gleam @@ -35,7 +35,7 @@ fn upsert_type_definitions(ctx: Context, module: context.Module) { let _ = queries.upsert_package_type_fun_signature( db: ctx.db, - nature: queries.TypeDefinition, + kind: queries.TypeDefinition, name: type_name, documentation: option.None, metadata: json.null(), @@ -49,7 +49,7 @@ fn upsert_type_definitions(ctx: Context, module: context.Module) { use gen <- result.try(type_definition_to_json(ctx, type_name, type_def)) queries.upsert_package_type_fun_signature( db: ctx.db, - nature: queries.TypeDefinition, + kind: queries.TypeDefinition, name: type_name, documentation: type_def.documentation, metadata: metadata.generate(type_def.deprecation, None), @@ -73,7 +73,7 @@ fn upsert_type_aliases(ctx: Context, module: context.Module) { let _ = queries.upsert_package_type_fun_signature( db: ctx.db, - nature: queries.TypeAlias, + kind: queries.TypeAlias, name: type_name, documentation: option.None, metadata: json.null(), @@ -88,7 +88,7 @@ fn upsert_type_aliases(ctx: Context, module: context.Module) { queries.upsert_package_type_fun_signature( db: ctx.db, name: type_name, - nature: queries.TypeAlias, + kind: queries.TypeAlias, documentation: type_alias.documentation, metadata: metadata.generate(type_alias.deprecation, None), signature: type_alias_to_string(type_name, type_alias), @@ -111,7 +111,7 @@ fn upsert_constants(ctx: Context, module: context.Module) { queries.upsert_package_type_fun_signature( db: ctx.db, name: constant_name, - nature: queries.Constant, + kind: queries.Constant, documentation: constant.documentation, metadata: Some(constant.implementations) |> metadata.generate(constant.deprecation, _), @@ -135,7 +135,7 @@ fn upsert_functions(ctx: Context, module: context.Module) { queries.upsert_package_type_fun_signature( db: ctx.db, name: function_name, - nature: queries.Function, + kind: queries.Function, documentation: function.documentation, metadata: Some(function.implementations) |> metadata.generate(function.deprecation, _), diff --git a/apps/backend/src/backend/gleam/generate/types.gleam b/apps/backend/src/backend/gleam/generate/types.gleam index 87b4534..35c4ff9 100644 --- a/apps/backend/src/backend/gleam/generate/types.gleam +++ b/apps/backend/src/backend/gleam/generate/types.gleam @@ -40,7 +40,7 @@ pub fn type_definition_to_json( use gen <- result.map(reduce_components(type_def.constructors, mapper)) use constructors <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ - #("nature", json.string("type-definition")), + #("kind", json.string("type-definition")), #("name", json.string(type_name)), #("documentation", json.nullable(type_def.documentation, json.string)), #("deprecation", json.nullable(type_def.documentation, json.string)), @@ -54,7 +54,7 @@ fn type_constructor_to_json(ctx: Context, constructor: TypeConstructor) { use gen <- result.map(reduce_components(constructor.parameters, mapper)) use parameters <- pair.map_first(gen) json.object([ - #("type", json.string("type-constructor")), + #("kind", json.string("type-constructor")), #("documentation", json.nullable(constructor.documentation, json.string)), #("name", json.string(constructor.name)), #("parameters", json.preprocessed_array(parameters)), @@ -65,9 +65,9 @@ fn parameters_to_json(ctx: Context, parameter: Parameter) { use gen <- result.map(type_to_json(ctx, parameter.type_)) use type_ <- pair.map_first(gen) json.object([ - #("type", json.string("parameter")), + #("kind", json.string("parameter")), #("label", json.nullable(parameter.label, json.string)), - #("params_type", type_), + #("type", type_), ]) } @@ -78,7 +78,7 @@ fn type_to_json(ctx: Context, type_: Type) { use gen <- result.map(reduce_components(elements, mapper)) use elements <- pair.map_first(gen) json.object([ - #("type", json.string("tuple")), + #("kind", json.string("tuple")), #("elements", json.preprocessed_array(elements)), ]) } @@ -88,7 +88,7 @@ fn type_to_json(ctx: Context, type_: Type) { use gen <- result.map(type_to_json(ctx, return)) let new_params = set.union(of: params, and: gen.1) json.object([ - #("type", json.string("fn")), + #("kind", json.string("fn")), #("params", json.preprocessed_array(elements)), #("return", gen.0), ]) @@ -96,7 +96,7 @@ fn type_to_json(ctx: Context, type_: Type) { } package_interface.Variable(id) -> { let json = - json.object([#("type", json.string("variable")), #("id", json.int(id))]) + json.object([#("kind", json.string("variable")), #("id", json.int(id))]) Ok(#(json, set.new())) } package_interface.Named(name, package, module, parameters) -> { @@ -109,7 +109,7 @@ fn type_to_json(ctx: Context, type_: Type) { option.Some(ref) -> set.insert(gen.1, ref.1) } json.object([ - #("type", json.string("named")), + #("kind", json.string("named")), #("ref", json.nullable(option.map(ref, fn(r) { r.0 }), json.string)), #("name", json.string(name)), #("package", json.string(package)), @@ -266,7 +266,7 @@ pub fn type_alias_to_json( use gen <- result.map(type_to_json(ctx, type_alias.alias)) use alias <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ - #("nature", json.string("type-alias")), + #("kind", json.string("type-alias")), #("name", json.string(type_name)), #("documentation", json.nullable(type_alias.documentation, json.string)), #("deprecation", json.nullable(type_alias.documentation, json.string)), @@ -288,7 +288,7 @@ pub fn constant_to_json(ctx: Context, constant_name: String, constant: Constant) use gen <- result.map(type_to_json(ctx, constant.type_)) use type_ <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ - #("nature", json.string("constant")), + #("kind", json.string("constant")), #("name", json.string(constant_name)), #("documentation", json.nullable(constant.documentation, json.string)), #("deprecation", json.nullable(constant.documentation, json.string)), @@ -305,7 +305,7 @@ pub fn function_to_json(ctx: Context, function_name: String, function: Function) |> pair.map_second(fn(s) { set.to_list(set.union(s, ret.1)) }) |> pair.map_first(fn(parameters) { json.object([ - #("nature", json.string("function")), + #("kind", json.string("function")), #("name", json.string(function_name)), #("documentation", json.nullable(function.documentation, json.string)), #("deprecation", json.nullable(function.documentation, json.string)), diff --git a/apps/backend/src/backend/postgres/queries.gleam b/apps/backend/src/backend/postgres/queries.gleam index b271fe6..10dea99 100644 --- a/apps/backend/src/backend/postgres/queries.gleam +++ b/apps/backend/src/backend/postgres/queries.gleam @@ -17,7 +17,7 @@ import gleam/result import gleam/string import helpers -pub type SignatureNature { +pub type SignatureKind { TypeAlias TypeDefinition Constant @@ -195,18 +195,27 @@ pub fn upsert_release( let gleam_toml = pgo.nullable(pgo.text, gleam_toml) let args = [package_id, version, url, package_interface, gleam_toml] "INSERT INTO package_release ( - package_id, - version, - url, - package_interface, - gleam_toml - ) VALUES ($1, $2, $3, $4, $5) + package_id, + version, + url, + package_interface, + gleam_toml + ) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (package_id, version) DO UPDATE SET url = $3, package_interface = $4, - gleam_toml = $5" - |> pgo.execute(db, args, dynamic.dynamic) + gleam_toml = $5 + RETURNING id, package_interface, gleam_toml" + |> pgo.execute( + db, + args, + dynamic.tuple3( + dynamic.int, + dynamic.optional(dynamic.string), + dynamic.optional(dynamic.string), + ), + ) |> result.map_error(error.DatabaseError) } @@ -218,13 +227,14 @@ pub fn lookup_release( let package_id = pgo.int(package_id) let version = pgo.text(release.version) let args = [package_id, version] - "SELECT package_interface, gleam_toml + "SELECT id, package_interface, gleam_toml FROM package_release WHERE package_id = $1 AND version = $2" |> pgo.execute( db, args, - dynamic.tuple2( + dynamic.tuple3( + dynamic.int, dynamic.optional(dynamic.string), dynamic.optional(dynamic.string), ), @@ -250,6 +260,33 @@ pub fn add_package_gleam_constraint( |> result.map_error(error.DatabaseError) } +pub fn add_package_gleam_retirement( + db: pgo.Connection, + retirement: hexpm.ReleaseRetirement, + release_id: Int, +) { + let retirement = + json.object([ + #("message", json.nullable(retirement.message, json.string)), + #("reason", { + json.string(case retirement.reason { + hexpm.OtherReason -> "other-reason" + hexpm.Invalid -> "invalid" + hexpm.Security -> "security" + hexpm.Deprecated -> "deprecated" + hexpm.Renamed -> "renamed" + }) + }), + ]) + |> json.to_string() + |> pgo.text() + let release_id = pgo.int(release_id) + "UPDATE package_release SET retirement = $1 WHERE id = $2" + |> pgo.execute(db, [retirement, release_id], dynamic.dynamic) + |> result.replace(Nil) + |> result.map_error(error.DatabaseError) +} + pub fn get_package_release_ids( db: pgo.Connection, package: package_interface.Package, @@ -311,7 +348,7 @@ fn implementations_pgo(implementations: package_interface.Implementations) { pub fn upsert_package_type_fun_signature( db db: pgo.Connection, - nature nature: SignatureNature, + kind kind: SignatureKind, name name: String, documentation documentation: Option(String), metadata metadata: json.Json, @@ -322,7 +359,7 @@ pub fn upsert_package_type_fun_signature( deprecation deprecation: Option(package_interface.Deprecation), implementations implementations: Option(package_interface.Implementations), ) { - let nature = case nature { + let kind = case kind { Function -> "function" TypeAlias -> "type_alias" TypeDefinition -> "type_definition" @@ -333,7 +370,7 @@ pub fn upsert_package_type_fun_signature( documentation, signature_, json_signature, - nature, + kind, parameters, metadata, package_module_id, @@ -345,7 +382,7 @@ pub fn upsert_package_type_fun_signature( documentation = $2, signature_ = $3, json_signature = $4, - nature = $5, + kind = $5, parameters = $6, metadata = $7, deprecation = $9, @@ -362,7 +399,7 @@ pub fn upsert_package_type_fun_signature( json_signature |> json.to_string() |> pgo.text(), - pgo.text(nature), + pgo.text(kind), dynamic.unsafe_coerce(dynamic.from(parameters)), metadata |> json.to_string() @@ -384,12 +421,12 @@ pub fn upsert_package_type_fun_signature( pub fn name_search(db: pgo.Connection, query: String) { let query = pgo.text(query) - "SELECT DISTINCT ON (type_name, nature, module_name) * + "SELECT DISTINCT ON (type_name, kind, module_name) * FROM ( SELECT s.name type_name, s.documentation, - s.nature, + s.kind, s.metadata, s.json_signature, m.name module_name, @@ -418,7 +455,7 @@ fn decode_type_search(dyn) { json.object([ #("name", json.string(a)), #("documentation", json.string(b)), - #("nature", json.string(c)), + #("kind", json.string(c)), #("metadata", dynamic.unsafe_coerce(d)), #("json_signature", dynamic.unsafe_coerce(e)), #("module_name", json.string(f)), @@ -442,7 +479,7 @@ pub fn search(db: pgo.Connection, q: String) { "SELECT s.name, s.documentation, - s.nature, + s.kind, s.metadata, s.json_signature, m.name, diff --git a/apps/backend/src/tasks/hex.gleam b/apps/backend/src/tasks/hex.gleam index d948fad..8a26e00 100644 --- a/apps/backend/src/tasks/hex.gleam +++ b/apps/backend/src/tasks/hex.gleam @@ -132,7 +132,7 @@ pub fn sync_package(ctx: Context, package: hexpm.Package) { } fn log_retirement_data(release: String, retirement: hexpm.ReleaseRetirement) { - wisp.log_info("Release " <> release <> " is retired. Skipping.") + wisp.log_info("Release " <> release <> " is retired.") case retirement.message { option.Some(m) -> wisp.log_debug(" Retired because " <> m) option.None -> Nil @@ -152,8 +152,10 @@ fn extract_release_interfaces_from_db( release: hexpm.Release, ) { use _ <- result.try_recover(queries.lookup_release(state.db, id, release)) - queries.upsert_release(state.db, id, release, None, None) - |> result.replace(#(None, None)) + use r <- result.try(queries.upsert_release(state.db, id, release, None, None)) + r.rows + |> list.first() + |> result.replace_error(error.UnknownError("")) } fn extract_release_interfaces_from_hex( @@ -176,20 +178,37 @@ fn extract_release_interfaces( id: Int, package: hexpm.Package, release: hexpm.Release, - interfaces: #(Option(String), Option(String)), + interfaces: #(Int, Option(String), Option(String)), ) { use _ <- result.try_recover(case interfaces { - #(Some(interface), Some(toml)) -> - hex_repo.parse_files(interface, toml) - |> result.map(fn(content) { - wisp.log_debug("Using interfaces from database") - content - }) + #(_, Some(interface), Some(toml)) -> { + use content <- result.map(hex_repo.parse_files(interface, toml)) + wisp.log_debug("Using interfaces from database") + content + } _ -> Error(error.UnknownError("No release data")) }) extract_release_interfaces_from_hex(state, id, package, release) } +fn save_retirement_data( + state: State, + release_id: Int, + package: hexpm.Package, + release: hexpm.Release, +) { + case release.retirement { + option.None -> Nil + option.Some(retirement) -> { + let release = package.name <> " v" <> release.version + log_retirement_data(release, retirement) + let _ = + queries.add_package_gleam_retirement(state.db, retirement, release_id) + Nil + } + } +} + fn insert_package_and_releases( package: hexpm.Package, releases: List(hexpm.Release), @@ -204,54 +223,42 @@ fn insert_package_and_releases( |> string.join(", v") wisp.log_info("Saving " <> package.name <> " v" <> versions) use id <- result.try(queries.upsert_package(state.db, package)) + wisp.log_debug("Saving owners for " <> package.name) use owners <- result.try(api.get_package_owners(package.name, secret: secret)) use _ <- result.try(queries.sync_package_owners(state.db, id, owners)) + wisp.log_debug("Saving releases for " <> package.name) - list.try_each(releases, fn(r) { - let release = package.name <> " v" <> r.version - use _ <- result.try_recover({ - queries.lookup_release(state.db, id, r) - |> result.replace(Nil) - |> case force_old_release_update { - True -> result.try(_, fn(_) { Error(error.EmptyError) }) - False -> function.identity - } - }) - wisp.log_debug("Handling release " <> r.version) - use interfaces <- result.map({ - extract_release_interfaces_from_db(state, id, r) - }) - case r.retirement { - option.Some(retirement) -> log_retirement_data(release, retirement) - option.None -> { - case children { - None -> { - let _ = do_extract_package(state, id, r, package, interfaces, False) - Nil - } - Some(children) -> { - supervisor.add(children, { - use _ <- supervisor.worker() - use iterations <- retrier.retry() - let it = int.to_string(iterations) - wisp.log_notice("Trying iteration " <> it <> " for " <> release) - do_extract_package( - state, - id, - r, - package, - interfaces, - iterations == 0, - ) - }) - Nil - } - } - Nil - } + use r <- list.try_each(releases) + let release = package.name <> " v" <> r.version + use _ <- result.try_recover({ + queries.lookup_release(state.db, id, r) + |> result.replace(Nil) + |> case force_old_release_update { + True -> result.try(_, fn(_) { Error(error.EmptyError) }) + False -> function.identity } }) + wisp.log_debug("Handling release " <> r.version) + use interfaces <- result.map(extract_release_interfaces_from_db(state, id, r)) + save_retirement_data(state, interfaces.0, package, r) + let _ = case children { + None -> { + let _ = do_extract_package(state, id, r, package, interfaces, False) + Nil + } + Some(children) -> { + supervisor.add(children, { + use _ <- supervisor.worker() + use iterations <- retrier.retry() + let it = int.to_string(iterations) + wisp.log_notice("Trying iteration " <> it <> " for " <> release) + do_extract_package(state, id, r, package, interfaces, iterations == 0) + }) + Nil + } + } + Nil } fn do_extract_package( @@ -259,7 +266,7 @@ fn do_extract_package( id: Int, release: hexpm.Release, package: hexpm.Package, - interfaces: #(Option(String), Option(String)), + interfaces: #(Int, Option(String), Option(String)), ignore_errors: Bool, ) { use #(package, gleam_toml) <- result.try({ diff --git a/apps/frontend/src/data/decoders/search_result.gleam b/apps/frontend/src/data/decoders/search_result.gleam index d62d2bd..b5bd529 100644 --- a/apps/frontend/src/data/decoders/search_result.gleam +++ b/apps/frontend/src/data/decoders/search_result.gleam @@ -52,7 +52,7 @@ pub fn decode_search_result(dyn) { dynamic.field("documentation", dynamic.string), dynamic.field("module_name", dynamic.string), dynamic.field("name", dynamic.string), - dynamic.field("nature", nature.decode_nature), + dynamic.field("kind", nature.decode_nature), dynamic.field("package_name", dynamic.string), dynamic.field("json_signature", signature.decode_signature), dynamic.field("metadata", decode_metadata), diff --git a/apps/frontend/src/data/decoders/signature.gleam b/apps/frontend/src/data/decoders/signature.gleam index e3ff739..7c8adda 100644 --- a/apps/frontend/src/data/decoders/signature.gleam +++ b/apps/frontend/src/data/decoders/signature.gleam @@ -1,6 +1,5 @@ import gleam/dynamic import gleam/int -import gleam/io import gleam/list import gleam/option.{type Option} import gleam/result @@ -48,7 +47,7 @@ pub type Signature { } pub fn decode_signature(dyn) { - use res <- result.try(dynamic.field("nature", dynamic.string)(dyn)) + use res <- result.try(dynamic.field("kind", dynamic.string)(dyn)) case res { "constant" -> decode_constant(dyn) "function" -> decode_function(dyn) @@ -59,7 +58,7 @@ pub fn decode_signature(dyn) { } fn decode_type(dyn) { - use res <- result.try(dynamic.field("type", dynamic.string)(dyn)) + use res <- result.try(dynamic.field("kind", dynamic.string)(dyn)) case res { "variable" -> decode_variable(dyn) "fn" -> decode_fn(dyn) @@ -134,7 +133,7 @@ fn decode_parameter(dyn) { Parameter(width, a, b) }, dynamic.field("label", dynamic.optional(dynamic.string)), - dynamic.field("params_type", decode_type), + dynamic.field("type", decode_type), )(dyn) }