From 1935548d3a2244766c2b2f5c5d8cc054c2437ee4 Mon Sep 17 00:00:00 2001 From: Andrei Kaleshka Date: Mon, 6 May 2024 12:01:00 +0200 Subject: [PATCH] Multiple databases support (#56) * multidb support * fix phantom_migrations rake task * fix tests * fix cop --- .rubocop.yml | 4 + lib/actual_db_schema.rb | 8 + lib/actual_db_schema/commands/base.rb | 4 - .../patches/migration_context.rb | 1 - lib/tasks/db.rake | 11 +- .../migrate_secondary/.keep} | 0 test/dummy_app/db/secondary_schema.rb | 14 ++ test/rake_task_secondary_test.rb | 86 ++++++++++ test/{rake_task_test.rb => rake_task_tes.rb} | 7 +- test/rake_tasks_all_databases_test.rb | 115 +++++++++++++ test/support/test_utils.rb | 154 ++++++++++++++---- test/test_helper.rb | 27 ++- 12 files changed, 380 insertions(+), 51 deletions(-) rename test/dummy_app/{config/database.yml => db/migrate_secondary/.keep} (100%) create mode 100644 test/dummy_app/db/secondary_schema.rb create mode 100644 test/rake_task_secondary_test.rb rename test/{rake_task_test.rb => rake_task_tes.rb} (88%) create mode 100644 test/rake_tasks_all_databases_test.rb diff --git a/.rubocop.yml b/.rubocop.yml index c62aebb..80c4805 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -24,3 +24,7 @@ Metrics/BlockLength: Metrics/MethodLength: Exclude: - test/**/* + +Metrics/ClassLength: + Exclude: + - test/**/* diff --git a/lib/actual_db_schema.rb b/lib/actual_db_schema.rb index 0ad70f5..7ab6f23 100644 --- a/lib/actual_db_schema.rb +++ b/lib/actual_db_schema.rb @@ -56,6 +56,14 @@ def self.migrations_paths def self.migration_filename(fullpath) fullpath.split("/").last end + + def self.for_each_db_connection + configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env) + configs.each do |db_config| + ActiveRecord::Base.establish_connection(db_config) + yield + end + end end ActiveRecord::MigrationProxy.prepend(ActualDbSchema::Patches::MigrationProxy) diff --git a/lib/actual_db_schema/commands/base.rb b/lib/actual_db_schema/commands/base.rb index 016d06e..bd43026 100644 --- a/lib/actual_db_schema/commands/base.rb +++ b/lib/actual_db_schema/commands/base.rb @@ -9,10 +9,6 @@ def call raise "ActualDbSchema is disabled. Set ActualDbSchema.config[:enabled] = true to enable it." end - if ActiveRecord::Migration.current_version >= 6 - ActiveRecord::Tasks::DatabaseTasks.raise_for_multi_db(command: "db:rollback_branches") - end - call_impl end diff --git a/lib/actual_db_schema/patches/migration_context.rb b/lib/actual_db_schema/patches/migration_context.rb index bdd2254..fa8c9ac 100644 --- a/lib/actual_db_schema/patches/migration_context.rb +++ b/lib/actual_db_schema/patches/migration_context.rb @@ -5,7 +5,6 @@ module Patches # Add new command to roll back the phantom migrations module MigrationContext def rollback_branches - ActualDbSchema.failed = [] migrations.reverse_each do |migration| migrator = down_migrator_for(migration) migrator.extend(ActualDbSchema::Patches::Migrator) diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 8cdeb74..b17a7a9 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -3,13 +3,18 @@ namespace :db do desc "Rollback migrations that were run inside not a merged branch." task rollback_branches: :load_config do - ActualDbSchema::Commands::Rollback.new.call + ActualDbSchema.failed = [] + ActualDbSchema.for_each_db_connection do + ActualDbSchema::Commands::Rollback.new.call + end end desc "List all phantom migrations - non-relevant migrations that were run inside not a merged branch." task phantom_migrations: :load_config do - ActualDbSchema::Commands::List.new.call + ActualDbSchema.for_each_db_connection do + ActualDbSchema::Commands::List.new.call + end end - task _dump: :rollback_branches + task "schema:dump" => :rollback_branches end diff --git a/test/dummy_app/config/database.yml b/test/dummy_app/db/migrate_secondary/.keep similarity index 100% rename from test/dummy_app/config/database.yml rename to test/dummy_app/db/migrate_secondary/.keep diff --git a/test/dummy_app/db/secondary_schema.rb b/test/dummy_app/db/secondary_schema.rb new file mode 100644 index 0000000..52a1883 --- /dev/null +++ b/test/dummy_app/db/secondary_schema.rb @@ -0,0 +1,14 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.0].define(version: 2013_09_06_111513) do +end diff --git a/test/rake_task_secondary_test.rb b/test/rake_task_secondary_test.rb new file mode 100644 index 0000000..c0c420c --- /dev/null +++ b/test/rake_task_secondary_test.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require "test_helper" + +describe "second db support" do + let(:utils) do + TestUtils.new(migrations_path: "db/migrate_secondary", migrated_path: "tmp/migrated_migrate_secondary") + end + + before do + ActiveRecord::Base.configurations = { "test" => TestingState.db_config["secondary"] } + ActiveRecord::Tasks::DatabaseTasks.database_configuration = { "test" => TestingState.db_config["secondary"] } + ActiveRecord::Base.establish_connection(**TestingState.db_config["secondary"]) + utils.cleanup + end + + describe "db:rollback_branches" do + it "creates the tmp/migrated_migrate_secondary folder" do + refute File.exist?(utils.app_file("tmp/migrated_migrate_secondary")) + utils.run_migrations + assert File.exist?(utils.app_file("tmp/migrated_migrate_secondary")) + end + + it "migrates the migrations" do + assert_empty utils.applied_migrations + utils.run_migrations + assert_equal %w[20130906111511 20130906111512], utils.applied_migrations + end + + it "keeps migrated migrations in tmp/migrated folder" do + utils.run_migrations + assert_equal %w[20130906111511_first.rb 20130906111512_second.rb], utils.migrated_files + end + + it "rolls back the migrations in the reversed order" do + utils.prepare_phantom_migrations + assert_empty TestingState.down + utils.run_migrations + assert_equal %i[second first], TestingState.down + end + + describe "with irreversible migration" do + before do + utils.define_migration_file("20130906111513_irreversible.rb", <<~RUBY) + class Irreversible < ActiveRecord::Migration[6.0] + def up + TestingState.up << :irreversible + end + + def down + raise ActiveRecord::IrreversibleMigration + end + end + RUBY + end + + it "keeps track of the irreversible migrations" do + utils.prepare_phantom_migrations + assert_equal %i[first second irreversible], TestingState.up + assert_empty ActualDbSchema.failed + utils.run_migrations + assert_equal(%w[20130906111513_irreversible.rb], ActualDbSchema.failed.map { |m| File.basename(m.filename) }) + end + end + end + + describe "db:phantom_migrations" do + it "shows the list of phantom migrations" do + ActualDbSchema::Git.stub(:current_branch, "fix-bug") do + utils.prepare_phantom_migrations + Rake::Task["db:phantom_migrations"].invoke + Rake::Task["db:phantom_migrations"].reenable + assert_match(/ Status Migration ID Branch Migration File/, TestingState.output) + assert_match(/---------------------------------------------------/, TestingState.output) + assert_match( + %r{ up 20130906111511 fix-bug tmp/migrated_migrate_secondary/20130906111511_first.rb}, + TestingState.output + ) + assert_match( + %r{ up 20130906111512 fix-bug tmp/migrated_migrate_secondary/20130906111512_second.rb}, + TestingState.output + ) + end + end + end +end diff --git a/test/rake_task_test.rb b/test/rake_task_tes.rb similarity index 88% rename from test/rake_task_test.rb rename to test/rake_task_tes.rb index 2311577..95952c6 100644 --- a/test/rake_task_test.rb +++ b/test/rake_task_tes.rb @@ -5,7 +5,12 @@ describe "single db" do let(:utils) { TestUtils.new } - before { utils.cleanup } + before do + ActiveRecord::Base.configurations = { "test" => TestingState.db_config["primary"] } + ActiveRecord::Tasks::DatabaseTasks.database_configuration = { "test" => TestingState.db_config["primary"] } + ActiveRecord::Base.establish_connection(**TestingState.db_config["primary"]) + utils.cleanup + end describe "db:rollback_branches" do it "creates the tmp/migrated folder" do diff --git a/test/rake_tasks_all_databases_test.rb b/test/rake_tasks_all_databases_test.rb new file mode 100644 index 0000000..6307ee7 --- /dev/null +++ b/test/rake_tasks_all_databases_test.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require "test_helper" + +describe "multipe db support" do + let(:utils) do + TestUtils.new( + migrations_path: ["db/migrate", "db/migrate_secondary"], + migrated_path: ["tmp/migrated", "tmp/migrated_migrate_secondary"] + ) + end + + before do + ActiveRecord::Base.configurations = { "test" => TestingState.db_config } + ActiveRecord::Tasks::DatabaseTasks.database_configuration = { "test" => TestingState.db_config } + utils.cleanup(TestingState.db_config) + end + + describe "db:rollback_branches" do + it "creates the tmp/migrated folder" do + refute File.exist?(utils.app_file("tmp/migrated")) + refute File.exist?(utils.app_file("tmp/migrated_migrate_secondary")) + utils.run_migrations + assert File.exist?(utils.app_file("tmp/migrated")) + assert File.exist?(utils.app_file("tmp/migrated_migrate_secondary")) + end + + it "migrates the migrations" do + assert_empty utils.applied_migrations(TestingState.db_config) + utils.run_migrations + assert_equal( + %w[20130906111511 20130906111512 20130906111514 20130906111515], + utils.applied_migrations(TestingState.db_config) + ) + end + + it "keeps migrated migrations in tmp/migrated folder" do + utils.run_migrations + assert_equal( + %w[ + 20130906111511_first_primary.rb + 20130906111512_second_primary.rb + 20130906111514_first_secondary.rb + 20130906111515_second_secondary.rb + ], + utils.migrated_files(TestingState.db_config) + ) + end + + it "rolls back the migrations in the reversed order" do + utils.prepare_phantom_migrations(TestingState.db_config) + assert_empty TestingState.down + utils.run_migrations + assert_equal %i[second_primary first_primary second_secondary first_secondary], TestingState.down + end + + describe "with irreversible migration" do + before do + %w[primary secondary].each do |prefix| + utils.define_migration_file("20130906111513_irreversible_#{prefix}.rb", <<~RUBY, prefix: prefix) + class Irreversible#{prefix.camelize} < ActiveRecord::Migration[6.0] + def up + TestingState.up << :irreversible_#{prefix} + end + + def down + raise ActiveRecord::IrreversibleMigration + end + end + RUBY + end + end + + it "keeps track of the irreversible migrations" do + utils.prepare_phantom_migrations(TestingState.db_config) + assert_equal( + %i[first_primary second_primary irreversible_primary irreversible_secondary first_secondary second_secondary], + TestingState.up + ) + assert_empty ActualDbSchema.failed + utils.run_migrations + failed = ActualDbSchema.failed.map { |m| File.basename(m.filename) } + assert_equal(%w[20130906111513_irreversible_primary.rb 20130906111513_irreversible_secondary.rb], failed) + end + end + end + + describe "db:phantom_migrations" do + it "shows the list of phantom migrations" do + ActualDbSchema::Git.stub(:current_branch, "fix-bug") do + utils.prepare_phantom_migrations(TestingState.db_config) + Rake::Task["db:phantom_migrations"].invoke + Rake::Task["db:phantom_migrations"].reenable + assert_match(/ Status Migration ID Branch Migration File/, TestingState.output) + assert_match(/---------------------------------------------------/, TestingState.output) + assert_match( + %r{ up 20130906111511 fix-bug tmp/migrated/20130906111511_first_primary.rb}, + TestingState.output + ) + assert_match( + %r{ up 20130906111512 fix-bug tmp/migrated/20130906111512_second_primary.rb}, + TestingState.output + ) + assert_match( + %r{ up 20130906111514 fix-bug tmp/migrated_migrate_secondary/20130906111514_first_secondary.rb}, + TestingState.output + ) + assert_match( + %r{ up 20130906111515 fix-bug tmp/migrated_migrate_secondary/20130906111515_second_secondary.rb}, + TestingState.output + ) + end + end + end +end diff --git a/test/support/test_utils.rb b/test/support/test_utils.rb index f94fa8d..fd3fe1a 100644 --- a/test/support/test_utils.rb +++ b/test/support/test_utils.rb @@ -1,11 +1,27 @@ # frozen_string_literal: true class TestUtils - attr_accessor :migrations_path, :migrated_path + attr_accessor :migrations_paths, :migrated_paths, :migration_timestamps, :connection_prefix + + MIGRATED_PATHS = { + primary: "tmp/migrated", + secondary: "tmp/migrated_migrate_secondary" + }.freeze + + MIGRATION_PATHS = { + primary: "db/migrate", + secondary: "db/migrate_secondary" + }.freeze def initialize(migrations_path: "db/migrate", migrated_path: "tmp/migrated") - @migrations_path = migrations_path - @migrated_path = migrated_path + @migrations_paths = Array.wrap(migrations_path) + @migrated_paths = Array.wrap(migrated_path) + @migration_timestamps = %w[ + 20130906111511 + 20130906111512 + 20130906111514 + 20130906111515 + ] end def app_file(path) @@ -22,69 +38,141 @@ def run_migrations Rake::Task["db:rollback_branches"].reenable end - def run_sql(sql) - ActiveRecord::Base.connection.execute(sql) + def applied_migrations(db_config = nil) + if db_config + db_config.each_with_object([]) do |(_, config), acc| + ActiveRecord::Base.establish_connection(**config) + acc.concat(applied_migrations_call) + end + else + applied_migrations_call + end end - def applied_migrations - run_sql("select * from schema_migrations").map do |row| - row["version"] + def clear_schema(db_config = nil) + if db_config + db_config.each_value do |config| + ActiveRecord::Base.establish_connection(**config) + clear_schema_call + end + else + clear_schema_call end end - def clear_schema - run_sql("delete from schema_migrations") + def delete_migrations_files(prefix_name = nil) + path = MIGRATION_PATHS.fetch(prefix_name&.to_sym, migrations_paths.first) + delete_migrations_files_for(path) end - def delete_migrations_files - Dir.glob(app_file("#{migrations_path}/**/*.rb")).each do |file| - remove_app_dir(file) - end + def define_migration_file(filename, content, prefix: nil) + path = + case prefix + when "primary" + "db/migrate" + when "secondary" + "db/migrate_secondary" + when nil + migrations_paths.first + else + raise "Unknown prefix: #{prefix}" + end + File.write(app_file("#{path}/#{filename}"), content, mode: "w") end - def define_migration_file(filename, content) - File.write(app_file("#{migrations_path}/#{filename}"), content, mode: "w") - end + def define_migrations(prefix_name = nil) + prefix = "_#{prefix_name}" if prefix_name + raise "No migration timestamps left" if @migration_timestamps.size < 2 - def define_migrations { - first: "20130906111511_first.rb", - second: "20130906111512_second.rb" + first: "#{@migration_timestamps.shift}_first#{prefix}.rb", + second: "#{@migration_timestamps.shift}_second#{prefix}.rb" }.each do |key, file_name| - define_migration_file(file_name, <<~RUBY) - class #{key.to_s.camelize} < ActiveRecord::Migration[6.0] + define_migration_file(file_name, <<~RUBY, prefix: prefix_name) + class #{key.to_s.camelize}#{prefix_name.to_s.camelize} < ActiveRecord::Migration[6.0] def up - TestingState.up << :#{key} + TestingState.up << :#{key}#{prefix} end def down - TestingState.down << :#{key} + TestingState.down << :#{key}#{prefix} end end RUBY end end - def prepare_phantom_migrations + def prepare_phantom_migrations(db_config = nil) run_migrations - delete_migrations_files # simulate switching branches + if db_config + db_config.each_key do |name| + delete_migrations_files(name) # simulate switching branches + end + else + delete_migrations_files + end + end + + def cleanup(db_config = nil) + if db_config + db_config.each do |name, c| + ActiveRecord::Base.establish_connection(**c) + cleanup_call(name) + end + else + cleanup_call + end + TestingState.reset + end + + def migrated_files(db_config = nil) + if db_config + db_config.each_with_object([]) do |(prefix_name, config), acc| + ActiveRecord::Base.establish_connection(**config) + acc.concat(migrated_files_call(prefix_name)) + end + else + migrated_files_call + end end - def cleanup - delete_migrations_files + private + + def cleanup_call(prefix_name = nil) + delete_migrations_files(prefix_name) if ActiveRecord::SchemaMigration.respond_to?(:create_table) ActiveRecord::SchemaMigration.create_table else ActiveRecord::SchemaMigration.new(ActiveRecord::Base.connection).create_table end run_sql("delete from schema_migrations") - remove_app_dir(migrated_path) - define_migrations + remove_app_dir(MIGRATED_PATHS.fetch(prefix_name&.to_sym, migrated_paths.first)) + define_migrations(prefix_name) Rails.application.load_tasks - TestingState.reset end - def migrated_files - Dir.glob(app_file("#{migrated_path}/*.rb")).map { |f| File.basename(f) }.sort + def delete_migrations_files_for(path) + Dir.glob(app_file("#{path}/**/*.rb")).each do |file| + remove_app_dir(file) + end + end + + def migrated_files_call(prefix_name = nil) + path = MIGRATED_PATHS.fetch(prefix_name&.to_sym, migrated_paths.first) + Dir.glob(app_file("#{path}/*.rb")).map { |f| File.basename(f) }.sort + end + + def clear_schema_call + run_sql("delete from schema_migrations") + end + + def applied_migrations_call + run_sql("select * from schema_migrations").map do |row| + row["version"] + end + end + + def run_sql(sql) + ActiveRecord::Base.connection.execute(sql) end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 1db5699..3d5f832 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -18,15 +18,6 @@ def initialize Rails.application = FakeApplication.new -db_config = { - adapter: "sqlite3", - database: "tmp/test.sqlite3" -} -ActiveRecord::Tasks::DatabaseTasks.database_configuration = { test: db_config } -ActiveRecord::Base.establish_connection(**db_config) - -ActualDbSchema.config[:enabled] = true - class TestingState class << self attr_accessor :up, :down, :output @@ -35,12 +26,30 @@ class << self def self.reset self.up = [] self.down = [] + ActualDbSchema.failed = [] self.output = +"" end + def self.db_config + { + "primary" => { + "adapter" => "sqlite3", + "database" => "tmp/primary.sqlite3", + "migrations_paths" => Rails.root.join("db", "migrate").to_s + }, + "secondary" => { + "adapter" => "sqlite3", + "database" => "tmp/secondary.sqlite3", + "migrations_paths" => Rails.root.join("db", "migrate_secondary").to_s + } + } + end + reset end +ActualDbSchema.config[:enabled] = true + module Kernel alias original_puts puts