diff --git a/.rubocop-bundler.yml b/.rubocop-bundler.yml index 9ae7ace2..31122b06 100644 --- a/.rubocop-bundler.yml +++ b/.rubocop-bundler.yml @@ -18,7 +18,6 @@ Lint/UnusedMethodArgument: Lint/UriEscapeUnescape: Enabled: true - # Style Layout/EndAlignment: @@ -92,7 +91,10 @@ Style/SpecialGlobalVars: Enabled: false Naming/VariableNumber: - EnforcedStyle: 'snake_case' + EnforcedStyle: "snake_case" + AllowedIdentifiers: + - sha256 + - capture3 Naming/MemoizedInstanceVariableName: Enabled: false diff --git a/docs/gemstash-configuration.5.md b/docs/gemstash-configuration.5.md index a7590023..36d4ec04 100644 --- a/docs/gemstash-configuration.5.md +++ b/docs/gemstash-configuration.5.md @@ -221,10 +221,10 @@ Boolean values `true` or `false` `:fetch_timeout` This is the number of seconds to allow for fetching a gem from upstream. -It covers establishing the connection and receiving the response. Fetching -gems over a slow connection may cause timeout errors. If you experience -timeout errors, you may want to increase this value. The default is `20` -seconds. +It covers establishing the connection and receiving the response. +Fetching gems over a slow connection may cause timeout errors. If you +experience timeout errors, you may want to increase this value. The +default is `20` seconds. ## Default value @@ -239,10 +239,10 @@ Integer value with a minimum of `1` `:open_timeout` The timeout setting for opening the connection to an upstream gem -server. On high-latency networks, even establishing the connection -to an upstream gem server can take a while. If you experience -connection failures instead of timeout errors, you may want to -increase this value. The default is `2` seconds. +server. On high-latency networks, even establishing the connection to an +upstream gem server can take a while. If you experience connection +failures instead of timeout errors, you may want to increase this value. +The default is `2` seconds. ## Default value diff --git a/gemstash.gemspec b/gemstash.gemspec index dc630fd1..3c7075a0 100644 --- a/gemstash.gemspec +++ b/gemstash.gemspec @@ -32,6 +32,7 @@ you push your own private gems as well." spec.required_ruby_version = ">= 3.1" spec.add_runtime_dependency "activesupport", ">= 4.2", "< 8" + spec.add_runtime_dependency "compact_index", "~> 0.15.0" spec.add_runtime_dependency "dalli", ">= 3.2.3", "< 4" spec.add_runtime_dependency "faraday", ">= 1", "< 3" spec.add_runtime_dependency "faraday_middleware", "~> 1.0" diff --git a/lib/gemstash.rb b/lib/gemstash.rb index d56b31ab..75939d76 100644 --- a/lib/gemstash.rb +++ b/lib/gemstash.rb @@ -7,6 +7,7 @@ module Gemstash autoload :DB, "gemstash/db" autoload :Cache, "gemstash/cache" autoload :CLI, "gemstash/cli" + autoload :CompactIndexBuilder, "gemstash/compact_index_builder" autoload :Configuration, "gemstash/configuration" autoload :Dependencies, "gemstash/dependencies" autoload :Env, "gemstash/env" diff --git a/lib/gemstash/cache.rb b/lib/gemstash/cache.rb index 87e64b15..a99bf0eb 100644 --- a/lib/gemstash/cache.rb +++ b/lib/gemstash/cache.rb @@ -43,7 +43,10 @@ def set_dependency(scope, gem, value) def invalidate_gem(scope, gem) @client.delete("deps/v1/#{scope}/#{gem}") - Gemstash::SpecsBuilder.invalidate_stored if scope == "private" + if scope == "private" + Gemstash::SpecsBuilder.invalidate_stored + Gemstash::CompactIndexBuilder.invalidate_stored(gem) + end end end diff --git a/lib/gemstash/cli/info.rb b/lib/gemstash/cli/info.rb index 100dcfe2..5ee921d9 100644 --- a/lib/gemstash/cli/info.rb +++ b/lib/gemstash/cli/info.rb @@ -12,6 +12,9 @@ class Info < Gemstash::CLI::Base def run prepare list_config + + # Gemstash::DB + # Gemstash::Env.current.db.dump_schema_migration(same_db: true) end private diff --git a/lib/gemstash/compact_index_builder.rb b/lib/gemstash/compact_index_builder.rb new file mode 100644 index 00000000..cef36ad2 --- /dev/null +++ b/lib/gemstash/compact_index_builder.rb @@ -0,0 +1,257 @@ +# frozen_string_literal: true + +require "active_support/core_ext/string/filters" +require "compact_index" +require "gemstash" +require "stringio" +require "zlib" + +module Gemstash + # Comment + class CompactIndexBuilder + include Gemstash::Env::Helper + attr_reader :result + + def self.serve(app, ...) + app.content_type "text/plain; charset=utf-8" + body = new(app.auth, ...).serve + app.etag Digest::MD5.hexdigest(body) + sha256 = Digest::SHA256.base64digest(body) + app.headers "Accept-Ranges" => "bytes", "Digest" => "sha-256=#{sha256}", "Repr-Digest" => "sha-256=:#{sha256}:", + "Content-Length" => body.bytesize.to_s + body + end + + def self.invalidate_stored(name) + storage = Gemstash::Storage.for("private").for("compact_index") + storage.resource("names").delete(:names) + storage.resource("versions").delete(:versions) + storage.resource("info/#{name}").delete(:info) + end + + def initialize(auth) + @auth = auth + end + + def serve + check_auth if gemstash_env.config[:protected_fetch] + fetch_from_storage + return result if result + + build_result + store_result + result + end + + private + + def storage + @storage ||= Gemstash::Storage.for("private").for("compact_index") + end + + def fetch_from_storage + resource = fetch_resource + return unless resource.exist?(key) + + @result = resource.load(key).content(key) + rescue StandardError + # On the off-chance of a race condition between specs.exist? and specs.load + @result = nil + end + + def store_result + fetch_resource.save(key => @result) + end + + def check_auth + @auth.check("fetch") + end + + # Comment + class Versions < CompactIndexBuilder + def fetch_resource + storage.resource("versions") + end + + def build_result(force_rebuild: false) + resource = fetch_resource + base = !force_rebuild && resource.exist?("versions.list") && resource.content("versions.list") + Tempfile.create("versions.list") do |file| + versions_file = CompactIndex::VersionsFile.new(file.path) + if base + file.write(base) + file.close + @result = versions_file.contents( + compact_index_versions(versions_file.updated_at.to_time) + ) + else + ts = Time.now.iso8601 + versions_file.create( + compact_index_public_versions(ts), ts + ) + @result = file.read + resource.save("versions.list" => @result) + end + end + end + + private + + def compact_index_versions(date) + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a + SELECT r.name as name, v.created_at as date, v.info_checksum as info_checksum, v.number as number, v.platform as platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + v.created_at > ? + + UNION ALL + + SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number as number, v.platform as platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + v.indexed is false AND + v.yanked_at > ? + + ORDER BY date, number, platform, name + SQL + + # not ordered correctly in sqlite for some reason + all_versions.sort_by! {|v| [v[:date], v[:number], v[:platform], v[:name]] } + map_gem_versions(all_versions.map {|v| [v[:name], [v]] }) + end + + def compact_index_public_versions(date) + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a + SELECT r.name, v.indexed, COALESCE(v.yanked_at, v.created_at) as stamp, + COALESCE(v.yanked_info_checksum, v.info_checksum) as info_checksum, v.number, v.platform + FROM rubygems AS r, versions AS v + WHERE v.rubygem_id = r.id AND + (v.created_at <= ? OR v.yanked_at <= ?) + ORDER BY name, COALESCE(v.yanked_at, v.created_at), number, platform + SQL + + versions_by_gem = all_versions.group_by {|row| row[:name] } + versions_by_gem.each_value do |versions| + info_checksum = versions.last[:info_checksum] + versions.select! {|v| v[:indexed] == true } + # Set all versions' info_checksum to work around https://github.com/bundler/compact_index/pull/20 + versions.each {|v| v[:info_checksum] = info_checksum } + end + + map_gem_versions(versions_by_gem) + end + + def map_gem_versions(versions_by_gem) + versions_by_gem.map do |name, versions| + CompactIndex::Gem.new( + name, + versions.map do |row| + CompactIndex::GemVersion.new( + row[:number], + row[:platform], + nil, # sha256 + row[:info_checksum], + nil, # dependencies + nil, # version.required_ruby_version, + nil, # version.required_rubygems_version + ) + end + ) + end + end + + def key + :versions + end + end + + # Comment + class Info < CompactIndexBuilder + def initialize(auth, name) + super(auth) + @name = name + end + + def fetch_resource + storage.resource("info/#{@name}") + end + + def build_result + @result = CompactIndex.info(requirements_and_dependencies) + end + + private + + def requirements_and_dependencies + group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at" + + dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY dependencies.rubygem_name, dependencies.id) as dep_req_agg" + + dep_name_agg = "string_agg(dependencies.rubygem_name, ',' ORDER BY dependencies.rubygem_name) AS dep_name_agg" + + DB::Rubygem.db[<<~SQL.squish, @name]. + SELECT #{group_by_columns}, #{dep_req_agg}, #{dep_name_agg} + FROM rubygems + LEFT JOIN versions ON versions.rubygem_id = rubygems.id + LEFT JOIN dependencies ON dependencies.version_id = versions.id + WHERE rubygems.name = ? AND versions.indexed = true + GROUP BY #{group_by_columns} + ORDER BY versions.created_at, number, platform, dep_name_agg + SQL + map do |row| + reqs = row[:dep_req_agg]&.split("@") + dep_names = row[:dep_name_agg]&.split(",") + + raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names&.size != reqs&.size + + deps = [] + if reqs + dep_names.zip(reqs).each do |name, req| + deps << CompactIndex::Dependency.new(name, req) + end + end + + CompactIndex::GemVersion.new( + row[:number], + row[:platform], + row[:sha256], + nil, # info_checksum + deps, + row[:required_ruby_version], + row[:required_rubygems_version] + ) + end + end + + def key + :info + end + end + + # Comment + class Names < CompactIndexBuilder + def fetch_resource + storage.resource("names") + end + + def build_result + names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } + SELECT name + FROM rubygems + LEFT JOIN versions ON versions.rubygem_id = rubygems.id + WHERE versions.indexed = true + GROUP BY name + HAVING COUNT(versions.id) > 0 + ORDER BY name + SQL + @result = CompactIndex.names(names).encode("UTF-8") + end + + private + + def key + :names + end + end + end +end diff --git a/lib/gemstash/db.rb b/lib/gemstash/db.rb index e28743f2..47789d73 100644 --- a/lib/gemstash/db.rb +++ b/lib/gemstash/db.rb @@ -10,6 +10,7 @@ module DB Sequel::Model.db = Gemstash::Env.current.db Sequel::Model.raise_on_save_failure = true Sequel::Model.plugin :timestamps, update_on_create: true + Sequel::Model.db.extension :schema_dumper autoload :Authorization, "gemstash/db/authorization" autoload :CachedRubygem, "gemstash/db/cached_rubygem" autoload :Dependency, "gemstash/db/dependency" diff --git a/lib/gemstash/db/version.rb b/lib/gemstash/db/version.rb index 3f42c3a5..032c03fa 100644 --- a/lib/gemstash/db/version.rb +++ b/lib/gemstash/db/version.rb @@ -9,7 +9,11 @@ class Version < Sequel::Model many_to_one :rubygem def deindex - update(indexed: false) + info = Gemstash::CompactIndexBuilder::Info.new(nil, rubygem.name).tap(&:build_result).result + prefix = number.dup + prefix << "-#{platform}" if platform != "ruby" + info.gsub!(/^#{Regexp.escape(prefix)} .*?\n/, "") + update(indexed: false, yanked_at: Time.now.utc, yanked_info_checksum: Digest::MD5.hexdigest(info)) end def reindex @@ -28,8 +32,10 @@ def self.slug(params) end def self.for_spec_collection(prerelease: false, latest: false) - versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem) - latest ? select_latest(versions) : versions + versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem). + order { [rubygem[:name], platform.desc] } + versions = select_latest(versions) if latest + order_for_spec_collection(versions) end def self.select_latest(versions) @@ -40,6 +46,17 @@ def self.select_latest(versions) map {|gem_versions| gem_versions.max_by {|version| Gem::Version.new(version.number) } } end + def self.order_for_spec_collection(versions) + versions.to_enum.group_by(&:rubygem_id).flat_map do |_, gem_versions| + versions = Hash.new {|h, k| h[k] = Gem::Version.new(k) } + numbers = gem_versions.map {|version| versions[version.number] } + numbers.sort! + gem_versions.sort_by do |version| + [-numbers.index(version.number), version.platform] + end.reverse + end + end + def self.find_by_spec(gem_id, spec) self[rubygem_id: gem_id, number: spec.version.to_s, @@ -54,14 +71,33 @@ def self.find_by_full_name(full_name) self[full_name: "#{full_name}-ruby"] end - def self.insert_by_spec(gem_id, spec) + def self.insert_by_spec(gem_id, spec, sha256:) gem_name = Gemstash::DB::Rubygem[gem_id].name + info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result + info << CompactIndex::GemVersion.new( + spec.version.to_s, + spec.platform.to_s, + sha256, + nil, # info_checksum + spec.runtime_dependencies.map do |dep| + requirements = dep.requirement.requirements + requirements = requirements.map {|r| "#{r.first} #{r.last}" } + requirements = requirements.join(", ") + CompactIndex::Dependency.new(dep.name, requirements) + end, + spec.required_ruby_version&.to_s, + spec.required_rubygems_version&.to_s + ).to_line << "\n" new(rubygem_id: gem_id, number: spec.version.to_s, platform: spec.platform.to_s, full_name: "#{gem_name}-#{spec.version}-#{spec.platform}", storage_id: spec.full_name, indexed: true, + sha256: sha256, + info_checksum: Digest::MD5.hexdigest(info), + required_ruby_version: spec.required_ruby_version&.to_s, + required_rubygems_version: spec.required_rubygems_version&.to_s, prerelease: spec.version.prerelease?).tap(&:save).id end end diff --git a/lib/gemstash/gem_pusher.rb b/lib/gemstash/gem_pusher.rb index 8cc4f8a2..01a1a7aa 100644 --- a/lib/gemstash/gem_pusher.rb +++ b/lib/gemstash/gem_pusher.rb @@ -79,7 +79,7 @@ def save_to_database raise ExistingVersionError, "Cannot push to an existing version!" if existing && existing.indexed raise YankedVersionError, "Cannot push to a yanked version!" if existing && !existing.indexed - version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec) + version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec, sha256: Digest::SHA256.hexdigest(@content)) Gemstash::DB::Dependency.insert_by_spec(version_id, spec) end end diff --git a/lib/gemstash/gem_source.rb b/lib/gemstash/gem_source.rb index 56cce779..48c02c07 100644 --- a/lib/gemstash/gem_source.rb +++ b/lib/gemstash/gem_source.rb @@ -29,7 +29,8 @@ class Base include Gemstash::Logging def_delegators :@app, :cache_control, :content_type, :env, :halt, - :headers, :http_client_for, :params, :redirect, :request + :headers, :http_client_for, :params, :redirect, :request, + :etag def initialize(app) @app = app diff --git a/lib/gemstash/gem_source/private_source.rb b/lib/gemstash/gem_source/private_source.rb index 025312d0..606e8cab 100644 --- a/lib/gemstash/gem_source/private_source.rb +++ b/lib/gemstash/gem_source/private_source.rb @@ -43,15 +43,17 @@ def serve_remove_spec_json end def serve_names - halt 403, "Not yet supported" + protected(CompactIndexBuilder::Names) end def serve_versions - halt 404, "Not yet supported" + protected(CompactIndexBuilder::Versions) end def serve_info(name) - halt 403, "Not yet supported" + halt(404, { "Content-Type" => "text/plain; charset=utf-8" }, "This gem could not be found") unless DB::Rubygem.where(name: name).limit(1).count > 0 + + protected(CompactIndexBuilder::Info, name) end def serve_marshal(id) @@ -96,8 +98,8 @@ def serve_prerelease_specs private - def protected(servable) - authorization.protect(self) { servable.serve(self) } + def protected(servable, ...) + authorization.protect(self) { servable.serve(self, ...) } end def authorization diff --git a/lib/gemstash/gem_yanker.rb b/lib/gemstash/gem_yanker.rb index c27ecf5f..dd9824d4 100644 --- a/lib/gemstash/gem_yanker.rb +++ b/lib/gemstash/gem_yanker.rb @@ -66,6 +66,7 @@ def update_database def invalidate_cache gemstash_env.cache.invalidate_gem("private", @gem_name) + Gemstash::CompactIndexBuilder.invalidate_stored(@gem_name) end end end diff --git a/lib/gemstash/migrations/06_compact_index.rb b/lib/gemstash/migrations/06_compact_index.rb new file mode 100644 index 00000000..d55e0d34 --- /dev/null +++ b/lib/gemstash/migrations/06_compact_index.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + alter_table :versions do # TODO: backfill info_checksum, sha256, required_ruby_version, required_rubygems_version + add_column :info_checksum, String, :size => 40 + add_column :yanked_info_checksum, String, :size => 40 + add_column :yanked_at, DateTime, :null => true + add_column :sha256, String, :size => 64 + add_column :required_ruby_version, String, :size => 255 + add_column :required_rubygems_version, String, :size => 255 + end + end +end diff --git a/lib/gemstash/web.rb b/lib/gemstash/web.rb index ae2428c8..bff5e95f 100644 --- a/lib/gemstash/web.rb +++ b/lib/gemstash/web.rb @@ -27,12 +27,19 @@ def http_client_for(server_url) not_found do status 404 + return body response.body if response.body && !response.body.empty? + body JSON.dump("error" => "Not found", "code" => 404) end error GemPusher::ExistingVersionError do - status 422 - body JSON.dump("error" => "Version already exists", "code" => 422) + status 409 + body JSON.dump("error" => "Version already exists", "code" => 409) + end + + error Gemstash::GemYanker::UnknownGemError, Gemstash::GemYanker::UnknownVersionError do |e| + status 404 + body JSON.dump("error" => e.message, "code" => 404) end get "/" do diff --git a/schema.rb b/schema.rb new file mode 100644 index 00000000..76b704fa --- /dev/null +++ b/schema.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + create_table(:authorizations) do + primary_key :id + column :auth_key, "varchar(191)", :null => false + column :permissions, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + column :name, "varchar(191)" + + index [:auth_key], :unique => true + index [:name], :unique => true + end + + create_table(:cached_rubygems) do + primary_key :id + column :upstream_id, "INTEGER", :null => false + column :name, "varchar(191)", :null => false + column :resource_type, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:name] + index %i[upstream_id resource_type name], :unique => true + end + + create_table(:dependencies) do + primary_key :id + column :version_id, "INTEGER", :null => false + column :rubygem_name, "varchar(191)", :null => false + column :requirements, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:rubygem_name] + index [:version_id] + end + + create_table(:health_tests) do + primary_key :id + column :string, "varchar(255)" + end + + create_table(:rubygems) do + primary_key :id + column :name, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:name], :unique => true + end + + create_table(:schema_info) do + column :version, "INTEGER", :default => 0, :null => false + end + + create_table(:upstreams) do + primary_key :id + column :uri, "varchar(191)", :null => false + column :host_id, "varchar(191)", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + + index [:host_id], :unique => true + index [:uri], :unique => true + end + + create_table(:versions) do + primary_key :id + column :rubygem_id, "INTEGER", :null => false + column :storage_id, "varchar(191)", :null => false + column :number, "varchar(191)", :null => false + column :platform, "varchar(191)", :null => false + column :full_name, "varchar(191)", :null => false + column :indexed, "boolean", :default => true, :null => false + column :prerelease, "boolean", :null => false + column :created_at, "timestamp", :null => false + column :updated_at, "timestamp", :null => false + column :info_checksum, "varchar(40)" + column :sha256, "varchar(64)" + column :required_ruby_version, "varchar(191)" + column :required_rubygems_version, "varchar(191)" + + index [:full_name], :unique => true + index [:indexed] + index %i[indexed prerelease] + index [:number] + index %i[rubygem_id number platform], :unique => true + index [:storage_id], :unique => true + end + end +end diff --git a/spec/gemstash/compact_index_builder_spec.rb b/spec/gemstash/compact_index_builder_spec.rb new file mode 100644 index 00000000..729c9687 --- /dev/null +++ b/spec/gemstash/compact_index_builder_spec.rb @@ -0,0 +1,233 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gemstash::CompactIndexBuilder do + let(:auth) { Gemstash::ApiKeyAuthorization.new(auth_key) } + let(:auth_with_invalid_auth_key) { Gemstash::ApiKeyAuthorization.new(invalid_auth_key) } + let(:auth_without_permission) { Gemstash::ApiKeyAuthorization.new(auth_key_without_permission) } + let(:auth_key) { "auth-key" } + let(:invalid_auth_key) { "invalid-auth-key" } + let(:auth_key_without_permission) { "auth-key-without-permission" } + + before do + Gemstash::Authorization.authorize(auth_key, "all") + Gemstash::Authorization.authorize(auth_key_without_permission, ["push"]) + allow(Time).to receive(:now).and_return(Time.new(1990, in: "UTC")) + end + + context "with no private gems" do + it "returns empty versions" do + result = described_class::Versions.new(auth).serve + expect(result).to eq(<<~VERSIONS) + created_at: 1990-01-01T00:00:00Z + --- + VERSIONS + end + + it "returns empty names" do + result = described_class::Names.new(auth).serve + expect(result).to eq(<<~NAMES) + --- + + NAMES + end + + it "returns 404 for info" do + result = described_class::Info.new(auth, "something").serve + expect(result).to eq(<<~INFO) + --- + INFO + end + end + + context "with some private gems" do + before do + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + insert_version(gem_id, "0.0.2") + insert_version(gem_id, "0.0.2", platform: "java") + gem_id = insert_rubygem("other-example") + insert_version(gem_id, "0.1.0") + end + + it "returns versions" do + Gemstash::CompactIndexBuilder::Versions.new(auth).serve + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq <<~VERSIONS + created_at: 1990-01-01T00:00:00Z + --- + example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c + example 0.0.2-java 02fd7dc9130d37b37fb21e7b3c870ada + example 0.0.2 be6954d4377b5262bee5bf4018e6227f + other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 + VERSIONS + end + + it "returns info" do + result = Gemstash::CompactIndexBuilder::Info.new(auth, "example").serve + expect(result).to eq <<~INFO + --- + 0.0.1 |checksum:786b0634cdc056d7fbb027802bbd6e13a6056143adc69047db6aded595754554 + 0.0.2-java |checksum:fd67cdfe89ddbd20e499efccffdc828384acf01e4a3068dbf414150ad7515f5f + 0.0.2 |checksum:bfad311d42610c3d1be9d18064f6e688152560e75c716ff63abb5cbb29673f63 + INFO + end + + it "returns names" do + result = Gemstash::CompactIndexBuilder::Names.new(auth).serve + expect(result).to eq <<~NAMES + --- + example + other-example + NAMES + end + end + + context "with some yanked gems" do + let(:expected_specs) do + [["example", Gem::Version.new("0.0.1"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "java"], + ["other-example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:expected_latest_specs) do + [["example", Gem::Version.new("0.0.2"), "ruby"], + ["example", Gem::Version.new("0.0.2"), "java"], + ["other-example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:expected_prerelease_specs) do + [["example", Gem::Version.new("0.0.2.rc1"), "ruby"], + ["example", Gem::Version.new("0.0.2.rc2"), "ruby"], + ["example", Gem::Version.new("0.0.2.rc2"), "java"], + ["other-example", Gem::Version.new("0.1.1.rc1"), "ruby"]] + end + + before do + Gemstash::CompactIndexBuilder::Versions.new(auth).serve + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + insert_version(gem_id, "0.0.2.rc1", prerelease: true) + insert_version(gem_id, "0.0.2.rc2", prerelease: true) + insert_version(gem_id, "0.0.2.rc2", platform: "java", prerelease: true) + insert_version(gem_id, "0.0.2") + insert_version(gem_id, "0.0.2", platform: "java") + insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true) + insert_version(gem_id, "0.0.3", indexed: false) + insert_version(gem_id, "0.0.3.rc1", indexed: false, prerelease: true, platform: "java") + insert_version(gem_id, "0.0.3", indexed: false, platform: "java") + gem_id = insert_rubygem("other-example") + insert_version(gem_id, "0.0.1", indexed: false) + insert_version(gem_id, "0.0.1.rc1", indexed: false, prerelease: true) + insert_version(gem_id, "0.1.0") + insert_version(gem_id, "0.1.1.rc1", prerelease: true) + end + + it "returns versions" do + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq <<~VERSIONS + created_at: 1990-01-01T00:00:00Z + --- + example 0.0.1 1e6fae87f01f5e16ef83205a1a12646c + other-example 0.0.1 6105347ebb9825ac754615ca55ff3b0c + other-example 0.0.1.rc1 6105347ebb9825ac754615ca55ff3b0c + example 0.0.2-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.2 1c60ca76f3375ac0473e16c9920a41c6 + example 0.0.2.rc1 d6f36de1e2fbebb92b6051fc6977ff0a + example 0.0.2.rc2-java 11850dde5a9df04c3fb2aba44704085d + example 0.0.2.rc2 48a1807ddf7e6a29c84d0f261cf4df64 + example 0.0.3-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3.rc1-java 30b1ce74f9d06e512e354c697280c5e0 + example 0.0.3.rc1 30b1ce74f9d06e512e354c697280c5e0 + other-example 0.1.0 ff0722a59d13124677a2edd0da268bd1 + other-example 0.1.1.rc1 1b239fe769f037ab38a4c89ea6b37320 + VERSIONS + end + end + + context "with a new spec pushed" do + before do + Gemstash::Authorization.authorize(auth_key, "all") + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + end + + it "busts the cache" do + # before + Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve + # after + end + end + + context "with a spec yanked" do + let(:initial_specs) do + [["example", Gem::Version.new("0.0.1"), "ruby"], + ["example", Gem::Version.new("0.1.0"), "ruby"]] + end + + let(:latest_specs) { [["example", Gem::Version.new("0.1.0"), "ruby"]] } + + let(:specs_after_yank) { [["example", Gem::Version.new("0.0.1"), "ruby"]] } + + before do + Gemstash::Authorization.authorize(auth_key, "all") + gem_id = insert_rubygem("example") + insert_version(gem_id, "0.0.1") + Gemstash::GemPusher.new(auth, read_gem("example", "0.1.0")).serve + end + + it "busts the cache" do + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to match_array(initial_specs) + Gemstash::GemYanker.new(auth, "example", "0.1.0").serve + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to match_array(specs_after_yank) + end + end + + context "with protected fetch disabled" do + it "serves versions without authorization" do + result = Gemstash::CompactIndexBuilder::Versions.new(auth).serve + expect(result).to eq(<<~VERSIONS) + created_at: 1990-01-01T00:00:00Z + --- + VERSIONS + end + end + + xcontext "with protected fetch enabled" do + before do + @test_env = test_env + config = Gemstash::Configuration.new(config: { protected_fetch: true }) + Gemstash::Env.current = Gemstash::Env.new(config) + end + + after do + Gemstash::Env.current = @test_env + end + + context "with valid authorization" do + it "serves specs" do + result = Gemstash::SpecsBuilder.new(auth).serve + expect(Marshal.load(gunzip(result))).to eq([]) + end + end + + context "with invalid authorization" do + it "prevents serving specs" do + expect { Gemstash::SpecsBuilder.new(auth_with_invalid_auth_key).serve }. + to raise_error(Gemstash::NotAuthorizedError) + end + end + + context "with invalid permission" do + it "prevents serving specs" do + expect { Gemstash::SpecsBuilder.new(auth_without_permission).serve }. + to raise_error(Gemstash::NotAuthorizedError) + end + end + end +end diff --git a/spec/gemstash/web_spec.rb b/spec/gemstash/web_spec.rb index 57ffe839..f745a868 100644 --- a/spec/gemstash/web_spec.rb +++ b/spec/gemstash/web_spec.rb @@ -431,8 +431,8 @@ def for(server_url, timeout = 20) post "/api/v1/gems", read_gem("example", "0.1.0"), env expect(last_response).to_not be_ok - expect(last_response.status).to eq(422) - expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 422) + expect(last_response.status).to eq(409) + expect(JSON.parse(last_response.body)).to eq("error" => "Version already exists", "code" => 409) end end end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 49a1de7e..d376f94e 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -69,6 +69,14 @@ @gemstash_empty_rubygems.start end + before(:each) do + $test_gemstash_server_current_test = self # rubocop:disable Style/GlobalVars + end + + after(:each) do + $test_gemstash_server_current_test = nil # rubocop:disable Style/GlobalVars + end + let(:platform_message) do if RUBY_PLATFORM == "java" "Java" @@ -116,6 +124,16 @@ clean_env env_name end + it "is a conformant gem server", db_transaction: false do + @gemstash.env.cache.flush + expect( + execute("bin/gem_server_conformance", ["--fail-fast", "--format", "progress", "--tag=~content_length_header"], + dir: "/Users/segiddins/Development/github.com/rubygems/gem_server_conformance", + env: { "UPSTREAM" => host, "GEM_HOST_API_KEY" => auth_key }) + ). + to exit_success + end + context "pushing a gem" do before do expect(deps.fetch(%w[speaker])).to match_dependencies([]) diff --git a/spec/support/db_helpers.rb b/spec/support/db_helpers.rb index d5a478ca..ec0075d9 100644 --- a/spec/support/db_helpers.rb +++ b/spec/support/db_helpers.rb @@ -31,9 +31,12 @@ def insert_version(gem_id, number, platform: "ruby", indexed: true, prerelease: :storage_id => storage_id, :indexed => indexed, :prerelease => prerelease, + :sha256 => Digest::SHA256.hexdigest(storage_id), :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ) + ).tap do |version_id| + update_info_checksum(version_id) + end end def insert_dependency(version_id, gem_name, requirements) @@ -43,6 +46,16 @@ def insert_dependency(version_id, gem_name, requirements) :requirements => requirements, :created_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP, :updated_at => Sequel::SQL::Constants::CURRENT_TIMESTAMP - ) + ).tap do + update_info_checksum(version_id) + end + end + + def update_info_checksum(version_id) + gem_id = Gemstash::Env.current.db[:versions][id: version_id][:rubygem_id] + gem_name = Gemstash::Env.current.db[:rubygems][id: gem_id][:name] + info = Gemstash::CompactIndexBuilder::Info.new(nil, gem_name).tap(&:build_result).result + Gemstash::DB::Version.where(id: version_id).update(info_checksum: Digest::MD5.hexdigest(info)) + raise "Failed to update info checksum for version #{version_id}" unless Gemstash::Env.current.db[:versions][id: version_id][:info_checksum] end end diff --git a/spec/support/exec_helpers.rb b/spec/support/exec_helpers.rb index 3820bad8..1838a688 100644 --- a/spec/support/exec_helpers.rb +++ b/spec/support/exec_helpers.rb @@ -76,7 +76,10 @@ def clear_ruby_env "BUNDLER_SETUP" => nil, "GEM_PATH" => original_gem_path, "RUBYLIB" => nil, - "RUBYOPT" => nil + "RUBYOPT" => nil, + "XDG_CONFIG_HOME" => nil, + "XDG_CACHE_HOME" => nil, + "XDG_DATA_HOME" => nil } end diff --git a/spec/support/test_gemstash_server.ru b/spec/support/test_gemstash_server.ru index 358a5878..c7317dcb 100644 --- a/spec/support/test_gemstash_server.ru +++ b/spec/support/test_gemstash_server.ru @@ -5,4 +5,18 @@ use Rack::Deflater use Gemstash::Env::RackMiddleware, $test_gemstash_server_env use Gemstash::GemSource::RackMiddleware use Gemstash::Health::RackMiddleware +map "/set_time" do + run lambda {|env| + now = Time.iso8601(Rack::Request.new(env).body.read) + $test_gemstash_server_current_test.allow(Time). + to $test_gemstash_server_current_test.receive(:now).and_return(now) + [200, {}, ["OK"]] + } +end +map "/rebuild_versions_list" do + run lambda {|_env| + Gemstash::CompactIndexBuilder::Versions.new(nil).build_result(force_rebuild: true) + [200, {}, ["OK"]] + } +end run Gemstash::Web.new(gemstash_env: $test_gemstash_server_env)