From efd034348a4c3ffdf3d9def99cd07298d09c5df8 Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Mon, 1 Aug 2016 16:21:49 -0400 Subject: [PATCH 1/3] update to handle new Solr unique key field layer_slug_s --- lib/geomonitor/indexer.rb | 3 +-- lib/tasks/solr.rake | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/geomonitor/indexer.rb b/lib/geomonitor/indexer.rb index 6fd416d..94c80a0 100644 --- a/lib/geomonitor/indexer.rb +++ b/lib/geomonitor/indexer.rb @@ -29,8 +29,7 @@ def query_solr(search_params = params || {}) end def update_by_id(params) - uuid = uuid_from_id(params[:id]) - data = [{ uuid: uuid, layer_availability_score_f: { set: params[:score] } }] + data = [{ layer_slug_s: params[:id], layer_availability_score_f: { set: params[:score] } }] update(data) end diff --git a/lib/tasks/solr.rake b/lib/tasks/solr.rake index 8ca9709..bbffdf0 100644 --- a/lib/tasks/solr.rake +++ b/lib/tasks/solr.rake @@ -6,23 +6,31 @@ namespace :solr do l = Layer.find_by_name(name_id) if l.nil? doc = Geomonitor.find_document(name_id) - if doc.present? && doc['uuid'].present? - wms = JSON.parse(doc['dct_references_s']).try(:[], 'http://www.opengis.net/def/serviceType/ogc/wms') + if doc.present? && doc['layer_slug_s'].present? + begin + wms = JSON.parse(doc['dct_references_s']).try(:[], 'http://www.opengis.net/def/serviceType/ogc/wms') + rescue JSON::ParserError + wms = nil + end if wms wms = wms.gsub('/wms', '') doc_institution = doc['dct_provenance_s'] institution = Institution.find_or_create_by(name: doc_institution) - host = Host.find_or_create_by(url: wms, institution_id: institution.id) do |host| - host.name = "#{institution.name}" + begin + host = Host.find_or_create_by(url: wms, institution_id: institution.id) do |host| + host.name = "#{institution.name}" + end + rescue ActiveRecord::RecordNotUnique + host = Host.where(url: wms, institution_id: institution.id).first end begin - georss_bbox = doc['georss_box_s'].split(' ') + envelope = doc['solr_geom'].gsub('ENVELOPE(', '').split(', ') Layer.create( name: name_id, host_id: host.id, geoserver_layername: doc['layer_id_s'], access: doc['dc_rights_s'], - bbox: "#{georss_bbox[1]} #{georss_bbox[0]} #{georss_bbox[3]} #{georss_bbox[2]}", + bbox: "#{envelope[0]} #{envelope[3]} #{envelope[1]} #{envelope[2]}", active: true ) rescue NoMethodError => e From c5eed9b3f64f382370921d3aa7e4ad677102a32f Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Mon, 1 Aug 2016 16:25:09 -0400 Subject: [PATCH 2/3] update Rails and Nokogiri --- Gemfile | 2 +- Gemfile.lock | 88 +++++++++++++++++++++++++++------------------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/Gemfile b/Gemfile index fb01f8a..32dd2d5 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '4.2.5.1' +gem 'rails', '~> 4.2' gem 'bootstrap-sass', '~> 3.3.4.1' gem 'autoprefixer-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 95a5010..37ecadb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,36 +1,36 @@ GEM remote: https://rubygems.org/ specs: - actionmailer (4.2.5.1) - actionpack (= 4.2.5.1) - actionview (= 4.2.5.1) - activejob (= 4.2.5.1) + actionmailer (4.2.7) + actionpack (= 4.2.7) + actionview (= 4.2.7) + activejob (= 4.2.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 1.0, >= 1.0.5) - actionpack (4.2.5.1) - actionview (= 4.2.5.1) - activesupport (= 4.2.5.1) + actionpack (4.2.7) + actionview (= 4.2.7) + activesupport (= 4.2.7) rack (~> 1.6) rack-test (~> 0.6.2) rails-dom-testing (~> 1.0, >= 1.0.5) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (4.2.5.1) - activesupport (= 4.2.5.1) + actionview (4.2.7) + activesupport (= 4.2.7) builder (~> 3.1) erubis (~> 2.7.0) rails-dom-testing (~> 1.0, >= 1.0.5) rails-html-sanitizer (~> 1.0, >= 1.0.2) - activejob (4.2.5.1) - activesupport (= 4.2.5.1) + activejob (4.2.7) + activesupport (= 4.2.7) globalid (>= 0.3.0) - activemodel (4.2.5.1) - activesupport (= 4.2.5.1) + activemodel (4.2.7) + activesupport (= 4.2.7) builder (~> 3.1) - activerecord (4.2.5.1) - activemodel (= 4.2.5.1) - activesupport (= 4.2.5.1) + activerecord (4.2.7) + activemodel (= 4.2.7) + activesupport (= 4.2.7) arel (~> 6.0) - activesupport (4.2.5.1) + activesupport (4.2.7) i18n (~> 0.7) json (~> 1.7, >= 1.7.7) minitest (~> 5.1) @@ -83,7 +83,7 @@ GEM execjs coffee-script-source (1.9.1) colorize (0.7.7) - concurrent-ruby (1.0.0) + concurrent-ruby (1.0.2) database_cleaner (1.4.1) diff-lcs (1.2.5) domain_name (0.5.25) @@ -99,7 +99,7 @@ GEM multipart-post (>= 1.2, < 3) friendly_id (5.1.0) activerecord (>= 4.0.0) - globalid (0.3.6) + globalid (0.3.7) activesupport (>= 4.1.0) http-cookie (1.0.2) domain_name (~> 0.5) @@ -126,11 +126,11 @@ GEM capistrano-bundle_audit (>= 0.0.3) capistrano-one_time_key capistrano-releaseboard - mail (2.6.3) - mime-types (>= 1.16, < 3) - mime-types (2.99) - mini_portile2 (2.0.0) - minitest (5.8.4) + mail (2.6.4) + mime-types (>= 1.16, < 4) + mime-types (2.99.2) + mini_portile2 (2.1.0) + minitest (5.9.0) multi_json (1.11.1) multipart-post (2.0.0) net-ping (1.7.7) @@ -139,12 +139,14 @@ GEM net-ssh (2.9.2) netrc (0.11.0) newrelic_rpm (3.10.0.279) - nokogiri (1.6.7.2) - mini_portile2 (~> 2.0.0.rc2) + nokogiri (1.6.8) + mini_portile2 (~> 2.1.0) + pkg-config (~> 1.1.7) pg (0.18.1) pg_csv (0.1.8) activerecord pg (~> 0.17) + pkg-config (1.1.7) poltergeist (1.6.0) capybara (~> 2.1) cliver (~> 0.3.1) @@ -153,16 +155,16 @@ GEM rack (1.6.4) rack-test (0.6.3) rack (>= 1.0) - rails (4.2.5.1) - actionmailer (= 4.2.5.1) - actionpack (= 4.2.5.1) - actionview (= 4.2.5.1) - activejob (= 4.2.5.1) - activemodel (= 4.2.5.1) - activerecord (= 4.2.5.1) - activesupport (= 4.2.5.1) + rails (4.2.7) + actionmailer (= 4.2.7) + actionpack (= 4.2.7) + actionview (= 4.2.7) + activejob (= 4.2.7) + activemodel (= 4.2.7) + activerecord (= 4.2.7) + activesupport (= 4.2.7) bundler (>= 1.3.0, < 2.0) - railties (= 4.2.5.1) + railties (= 4.2.7) sprockets-rails rails-deprecated_sanitizer (1.0.3) activesupport (>= 4.2.0.alpha) @@ -174,12 +176,12 @@ GEM loofah (~> 2.0) rails_config (0.4.2) activesupport (>= 3.0) - railties (4.2.5.1) - actionpack (= 4.2.5.1) - activesupport (= 4.2.5.1) + railties (4.2.7) + actionpack (= 4.2.7) + activesupport (= 4.2.7) rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) - rake (10.5.0) + rake (11.2.2) rdoc (4.2.0) json (~> 1.4) ref (1.0.5) @@ -216,10 +218,10 @@ GEM sdoc (0.4.1) json (~> 1.7, >= 1.7.7) rdoc (~> 4.0) - sprockets (3.5.2) + sprockets (3.7.0) concurrent-ruby (~> 1.0) rack (> 1, < 3) - sprockets-rails (3.0.0) + sprockets-rails (3.1.1) actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) @@ -284,7 +286,7 @@ DEPENDENCIES pg pg_csv poltergeist - rails (= 4.2.5.1) + rails (~> 4.2) rails_config rake rest-client @@ -300,4 +302,4 @@ DEPENDENCIES whenever BUNDLED WITH - 1.10.6 + 1.12.5 From f5684d49cfd038efbb6b1ecea616ecd7eaa22026 Mon Sep 17 00:00:00 2001 From: Jack Reed Date: Tue, 2 Aug 2016 11:17:44 -0400 Subject: [PATCH 3/3] add Endpoint and move relationships from Host, Layer, and Ping --- app/models/endpoint.rb | 10 +++++++++ app/models/host.rb | 8 ++----- app/models/institution.rb | 3 ++- app/models/layer.rb | 8 ++++--- app/models/ping.rb | 12 +++++------ app/views/hosts/index.html.erb | 2 +- db/migrate/20160802114709_create_endpoints.rb | 10 +++++++++ .../20160802121117_add_endpoint_to_layer.rb | 6 ++++++ .../20160802134412_add_ping_to_endpoint.rb | 8 +++++++ db/schema.rb | 21 +++++++++++++------ lib/geomonitor/client.rb | 2 +- lib/tasks/check_status.rake | 19 +++++++++-------- lib/tasks/solr.rake | 18 +++++++++------- 13 files changed, 87 insertions(+), 40 deletions(-) create mode 100644 app/models/endpoint.rb create mode 100644 db/migrate/20160802114709_create_endpoints.rb create mode 100644 db/migrate/20160802121117_add_endpoint_to_layer.rb create mode 100644 db/migrate/20160802134412_add_ping_to_endpoint.rb diff --git a/app/models/endpoint.rb b/app/models/endpoint.rb new file mode 100644 index 0000000..d056888 --- /dev/null +++ b/app/models/endpoint.rb @@ -0,0 +1,10 @@ +class Endpoint < ActiveRecord::Base + has_many :pings + belongs_to :host + validates :url, presence: true + has_many :layers + + def last_ping_status + pings.recent_status + end +end diff --git a/app/models/host.rb b/app/models/host.rb index b62417d..334eb0c 100644 --- a/app/models/host.rb +++ b/app/models/host.rb @@ -1,7 +1,7 @@ class Host < ActiveRecord::Base belongs_to :institution - has_many :layers - has_many :pings + has_many :endpoints + has_many :layers, through: :endpoints validates :institution_id, presence: true before_create :check_and_increment @@ -18,10 +18,6 @@ def overall_status(options = {}) end end - def last_ping_status - pings.recent_status - end - def layers_count(options = {}) if options[:force_update] Rails.cache.write("host/#{id}/layers_count", calculate_layers_count) diff --git a/app/models/institution.rb b/app/models/institution.rb index 811b660..4515f05 100644 --- a/app/models/institution.rb +++ b/app/models/institution.rb @@ -3,6 +3,7 @@ class Institution < ActiveRecord::Base def layer_count hosts = Host.where(institution_id: id).ids - Layer.where(host_id: hosts).count + endpoints = Endpoint.where(host: hosts) + Layer.where(endpoint: endpoints).count end end diff --git a/app/models/layer.rb b/app/models/layer.rb index 9bc8428..55909f5 100644 --- a/app/models/layer.rb +++ b/app/models/layer.rb @@ -1,8 +1,9 @@ class Layer < ActiveRecord::Base extend FriendlyId friendly_id :name, use: [:slugged, :finders] - belongs_to :host, counter_cache: true - validates :host_id, presence: true + belongs_to :endpoint + has_one :host, through: :endpoint + validates :endpoint_id, presence: true has_many :statuses has_one :latest_status, -> (object) { where("latest = ?", true) }, :class_name => 'Status' @@ -38,7 +39,8 @@ def update_solr_score end def self.current_recent_status(params) - where(host_id: params[:id], active: true) + endpoints = Endpoint.where(host: params[:id]) + where(endpoint: endpoints, active: true) .with_current_status(params[:status]) .includes(:latest_status) .order(updated_at: :desc) diff --git a/app/models/ping.rb b/app/models/ping.rb index bab1766..2681c5f 100644 --- a/app/models/ping.rb +++ b/app/models/ping.rb @@ -1,18 +1,18 @@ class Ping < ActiveRecord::Base - belongs_to :host, counter_cache: true, touch: true + belongs_to :endpoint, counter_cache: true, touch: true - def self.check_status(host) - Geomonitor.logger.debug "Pinging #{host.url} ..." + def self.check_status(endpoint) + Geomonitor.logger.debug "Pinging #{endpoint.url} ..." begin - p = Net::Ping::HTTP.new(host.url).ping? + p = Net::Ping::HTTP.new(endpoint.url).ping? rescue URI::InvalidURIError => error Geomonitor.logger.error "#{error} url is \"#{host.url}\" for host_id #{host.id}" p = false end Geomonitor.logger.info "Ping result: #{p}" - current = Ping.create(host_id: host.id, status: p, latest: true) + current = Ping.create(endpoint: endpoint, status: p, latest: true) - old_pings = Ping.where(host_id: host.id).where('id != ?', current.id) + old_pings = Ping.where(endpoint: endpoint).where('id != ?', current.id) if old_pings.length > 0 old_pings.each do |old| diff --git a/app/views/hosts/index.html.erb b/app/views/hosts/index.html.erb index 7741571..f7293ae 100644 --- a/app/views/hosts/index.html.erb +++ b/app/views/hosts/index.html.erb @@ -15,7 +15,7 @@ <% @hosts.each do |host| %> - <%= ping_status(host.last_ping_status) if host.last_ping_status %> <%= link_to host.name, host %> + <%= link_to host.name, host %> <%= link_to host.url, host %> <%= link_to host.institution_name, host.institution %> <%= number_with_delimiter(host.layers_count) %> diff --git a/db/migrate/20160802114709_create_endpoints.rb b/db/migrate/20160802114709_create_endpoints.rb new file mode 100644 index 0000000..d74c6cb --- /dev/null +++ b/db/migrate/20160802114709_create_endpoints.rb @@ -0,0 +1,10 @@ +class CreateEndpoints < ActiveRecord::Migration + def change + create_table :endpoints do |t| + t.string :name + t.string :url + t.references :host + t.timestamps null: false + end + end +end diff --git a/db/migrate/20160802121117_add_endpoint_to_layer.rb b/db/migrate/20160802121117_add_endpoint_to_layer.rb new file mode 100644 index 0000000..18621cc --- /dev/null +++ b/db/migrate/20160802121117_add_endpoint_to_layer.rb @@ -0,0 +1,6 @@ +class AddEndpointToLayer < ActiveRecord::Migration + def change + add_reference :layers, :endpoint, index: true, foreign_key: true + remove_reference(:layers, :host, index: true) + end +end diff --git a/db/migrate/20160802134412_add_ping_to_endpoint.rb b/db/migrate/20160802134412_add_ping_to_endpoint.rb new file mode 100644 index 0000000..6969f51 --- /dev/null +++ b/db/migrate/20160802134412_add_ping_to_endpoint.rb @@ -0,0 +1,8 @@ +class AddPingToEndpoint < ActiveRecord::Migration + def change + add_reference :pings, :endpoint, index: true + remove_reference(:pings, :host, index: true) + add_column(:endpoints, :pings_count, :integer) + remove_column(:hosts, :pings_count) + end +end diff --git a/db/schema.rb b/db/schema.rb index e65da59..13387ba 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,11 +11,20 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150123224019) do +ActiveRecord::Schema.define(version: 20160802134412) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "endpoints", force: :cascade do |t| + t.string "name" + t.string "url" + t.integer "host_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.integer "pings_count" + end + create_table "friendly_id_slugs", force: :cascade do |t| t.string "slug", null: false t.integer "sluggable_id", null: false @@ -35,7 +44,6 @@ t.string "url" t.integer "institution_id" t.integer "layers_count" - t.integer "pings_count" t.datetime "created_at" t.datetime "updated_at" end @@ -57,25 +65,25 @@ t.string "access" t.text "description" t.string "bbox" - t.integer "host_id" t.integer "statuses_count" t.datetime "created_at" t.datetime "updated_at" t.string "slug" t.boolean "active" + t.integer "endpoint_id" end - add_index "layers", ["host_id"], name: "index_layers_on_host_id", using: :btree + add_index "layers", ["endpoint_id"], name: "index_layers_on_endpoint_id", using: :btree create_table "pings", force: :cascade do |t| t.boolean "status" t.boolean "latest" - t.integer "host_id" t.datetime "created_at" t.datetime "updated_at" + t.integer "endpoint_id" end - add_index "pings", ["host_id"], name: "index_pings_on_host_id", using: :btree + add_index "pings", ["endpoint_id"], name: "index_pings_on_endpoint_id", using: :btree create_table "statuses", force: :cascade do |t| t.string "res_code" @@ -92,4 +100,5 @@ add_index "statuses", ["layer_id"], name: "index_statuses_on_layer_id", using: :btree + add_foreign_key "layers", "endpoints" end diff --git a/lib/geomonitor/client.rb b/lib/geomonitor/client.rb index fd89f74..bf39e7a 100644 --- a/lib/geomonitor/client.rb +++ b/lib/geomonitor/client.rb @@ -35,7 +35,7 @@ def request_params # url used for tile request # @return [String] def url - @layer.host.url + '/wms' + @layer.endpoint.url + '/wms' end ## diff --git a/lib/tasks/check_status.rake b/lib/tasks/check_status.rake index a925c31..208c965 100644 --- a/lib/tasks/check_status.rake +++ b/lib/tasks/check_status.rake @@ -9,8 +9,8 @@ layers.shuffle.each do |layer| # Skip if host is not pingable - next if layer.host.pings.last.nil? - next unless layer.host.pings.last.status + next if layer.endpoint.pings.last.nil? + next unless layer.endpoint.pings.last.status # Skip if access is restricted next if layer[:access] == 'Restricted' @@ -23,7 +23,7 @@ layers.shuffle.each do |layer| # Skip if host is not pingable - next unless layer.host.pings.last.status + next unless layer.endpoint.pings.last.status # Skip if access is restricted next if layer[:access] == 'Restricted' @@ -36,7 +36,8 @@ task :check_stanford => :environment do institution = Institution.find_by name: "Stanford" stanford_hosts = Host.where(institution_id: institution.id) - layers = Layer.where(host_id: stanford_hosts, active: true) + stanford_endpoints = Endpoint.where(host: stanford_hosts) + layers = Layer.where(endpoint: stanford_endpoints, active: true) layers.shuffle.each do |layer| Status.run_check(layer) end @@ -44,11 +45,11 @@ end namespace :ping do - desc 'Ping all hosts' - task :hosts => :environment do - hosts = Host.all() - hosts.shuffle.each do |host| - Ping.check_status(host) + desc 'Ping all endpoint' + task :endpoints => :environment do + endpoints = Endpoint.all + endpoints.shuffle.each do |endpoint| + Ping.check_status(endpoint) end end end diff --git a/lib/tasks/solr.rake b/lib/tasks/solr.rake index bbffdf0..bba53e5 100644 --- a/lib/tasks/solr.rake +++ b/lib/tasks/solr.rake @@ -16,18 +16,22 @@ namespace :solr do wms = wms.gsub('/wms', '') doc_institution = doc['dct_provenance_s'] institution = Institution.find_or_create_by(name: doc_institution) + host_url = URI.parse(wms).host begin - host = Host.find_or_create_by(url: wms, institution_id: institution.id) do |host| + host = Host.find_or_create_by(url: host_url, institution_id: institution.id) do |host| host.name = "#{institution.name}" end rescue ActiveRecord::RecordNotUnique - host = Host.where(url: wms, institution_id: institution.id).first + host = Host.where(url: host_url, institution_id: institution.id).first + end + begin + endpoint = Endpoint.find_or_create_by(host: host, url: wms) end begin envelope = doc['solr_geom'].gsub('ENVELOPE(', '').split(', ') Layer.create( name: name_id, - host_id: host.id, + endpoint: endpoint, geoserver_layername: doc['layer_id_s'], access: doc['dc_rights_s'], bbox: "#{envelope[0]} #{envelope[3]} #{envelope[1]} #{envelope[2]}", @@ -41,10 +45,10 @@ namespace :solr do end end end - desc 'Save new layers from Solr and ping hosts' + desc 'Save new layers from Solr and ping endpoints' task update_and_ping: :environment do Rake::Task['solr:update_layers'].invoke - Rake::Task['ping:hosts'].invoke + Rake::Task['ping:endpoints'].invoke end desc 'Deactivate non-Solr layers' task deactivate: :environment do @@ -53,7 +57,7 @@ namespace :solr do Layer.where.not(name: solr_current).each do |layer| layer.active = false layer.save - Rails.cache.delete("host/#{layer.host_id}/layers_count") + Rails.cache.delete("host/#{layer.endpoint.host_id}/layers_count") end end end @@ -65,7 +69,7 @@ namespace :solr do if layer layer.active = true layer.save - Rails.cache.delete("host/#{layer.host_id}/layers_count") + Rails.cache.delete("host/#{layer.endpoint.host_id}/layers_count") end end end