diff --git a/.travis.yml b/.travis.yml index 269ce3536..1530723ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ addons: postgresql: "9.6" services: + - xvfb - postgresql # We don't start elasticsearch here; instead we download it later and run it # directly in order to enable the test cluster to start up. See @@ -20,7 +21,6 @@ cache: before_install: - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz - tar -xvf elasticsearch-5.6.4.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d914c17..30c589aab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). It uses [CalVer](https://calver.org/) as of May 2019. +## [19.07](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.07) - 2019-07-01 +### Added +* Compresses http responses +* `get_approximate_count` method on `Notice` and `InfringingUrl` (uses postgres reltuples) +* Whitelists logged-in users using the web interface (if they would not be throttled using the API) + +### Changed +* Updates numerous dependencies +* Refactors court order reporting cron job +* Only includes notices of type supporting in this cron job + +### Fixed +* Bug with captcha validation in notice requests (pushed out before this release as a hotfix) +* Broken link in admin site +* Strips special characters from cron job filenames to suppress annoying warning emails to admins + ## [19.05](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.05) - 2019-05-20 ### Added diff --git a/README.md b/README.md index 1d5f0b40f..fc28edebf 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Most of these are optional and have sensible defaults (which may vary by environ - `SMTP_VERIFY_MODE` - `TABLES_USING_NAIVE_COUNTS` - prevents Kaminari from doing slow/full counts of the specified tables - `TEST_CLUSTER_COMMAND` -- `USER_CRON_EMAIL` - for use in sending reports of court order files +- `USER_CRON_EMAIL` - for use in sending reports of court order files; can be a string or a list (in a JSON.parse-able format) - `USER_CRON_MAGIC_DIR` - `WEB_CONCURRENCY` - number of Unicorn workers - `WEB_TIMEOUT` - Unicorn timeout diff --git a/lib/tasks/lumen.rake b/lib/tasks/lumen.rake index ad81a276d..79465f634 100644 --- a/lib/tasks/lumen.rake +++ b/lib/tasks/lumen.rake @@ -827,7 +827,7 @@ class CourtOrderReporter fetch_files write_files make_archive - email_user + email_users end def setup_directories @@ -878,7 +878,14 @@ class CourtOrderReporter def notify_about_unredacted_files(f) return if f.notice.file_uploads.where(kind: 'supporting').present? - File.write(@info_filepath, app.notice_url(f.notice), mode: 'a') + File.write( + @info_filepath, + Rails.application.routes.url_helpers.notice_url( + f.notice, + host: Rails.application.config.action_mailer.default_url_options[:host] + ), + mode: 'a' + ) end def make_archive @@ -888,23 +895,39 @@ class CourtOrderReporter system("rm -r #{@working_dir}") end - def email_user - email = ENV['USER_CRON_EMAIL'] - unless (email && defined? SMTP_SETTINGS) + def email_users + unless (emails && defined? SMTP_SETTINGS) Rails.logger.warn '[rake] Missing email or SMTP_SETTINGS; not emailing court order report' exit end Rails.logger.info '[rake] Sending court order report email' - mailtext = <<~HEREDOC - Subject: Latest email archive from Lumen - The latest archive of Lumen court order files can be found at - #{ Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz. - HEREDOC + emails.each do |email| + email_single_user(email) + end + end + def emails + @emails ||= begin + JSON.parse ENV['USER_CRON_EMAIL'] # list of emails + rescue JSON::ParserError + [ENV['USER_CRON_EMAIL']] # single-email string, as 1-item list + end + end + + def email_single_user(email) Net::SMTP.start(SMTP_SETTINGS[:address]) do |smtp| smtp.send_message mailtext, 'no-reply@lumendatabase.org', email end end + + def mailtext + @mailtext ||= <<~HEREDOC + Subject: Latest email archive from Lumen + + The latest archive of Lumen court order files can be found at + #{Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz. + HEREDOC + end end diff --git a/spec/config/initializers/rack-attack_spec.rb b/spec/config/initializers/rack-attack_spec.rb index 96c7be04a..a6e8c43d6 100644 --- a/spec/config/initializers/rack-attack_spec.rb +++ b/spec/config/initializers/rack-attack_spec.rb @@ -9,7 +9,7 @@ def app end describe 'throttling excessive API requests by IP address', cache: true do - let(:limit) { 5 } + let(:limit) { 1 } context 'number of requests is lower than the limit' do it 'allows requests' do @@ -31,7 +31,7 @@ def app end describe 'throttling excessive HTTP responses by IP address' do - let(:limit) { 10 } + let(:limit) { 1 } context 'number of requests is lower than the limit', cache: true do it 'allows requests' do diff --git a/spec/integration/getting_topics_through_api_spec.rb b/spec/integration/getting_topics_through_api_spec.rb index 797c17129..83d362133 100644 --- a/spec/integration/getting_topics_through_api_spec.rb +++ b/spec/integration/getting_topics_through_api_spec.rb @@ -2,6 +2,7 @@ feature "Topic lists as json" do scenario "an API consumer gets a list of topics" do + Topic.destroy_all create(:topic, name: 'A topic') create(:topic, name: 'Another topic') diff --git a/spec/tasks/generate_court_order_report_spec.rb b/spec/tasks/generate_court_order_report_spec.rb new file mode 100644 index 000000000..071d6c9e3 --- /dev/null +++ b/spec/tasks/generate_court_order_report_spec.rb @@ -0,0 +1,43 @@ +require 'rails_helper' + +describe 'rake lumen:generate_court_order_report', type: :task do + before :all do + create(:court_order, :with_document) + @cached_user_cron_email = ENV['USER_CRON_EMAIL'] + end + + before(:each) do + stub_const('SMTP_SETTINGS', { address: 'test@example.com' }) + end + + after :all do + ENV['USER_CRON_EMAIL'] = @cached_user_cron_email + CourtOrder.destroy_all + end + + it 'sends a single email' do + ENV['USER_CRON_EMAIL'] = 'foo@example.com' + stub_smtp + expect(@fake_smtp).to receive(:send_message) + .with(anything, anything, 'foo@example.com') + task.execute + end + + it 'sends emails to a list' do + ENV['USER_CRON_EMAIL'] = '["foo@example.com", "bar@example.com"]' + stub_smtp + expect(@fake_smtp).to receive(:send_message) + .with(anything, anything, 'foo@example.com') + expect(@fake_smtp).to receive(:send_message) + .with(anything, anything, 'bar@example.com') + task.execute + end + + # SMTP may or may not be configured on the machines this test will run on, + # so let's stub it out. + def stub_smtp + @fake_smtp = double('Net::STMP.new') + allow(@fake_smtp).to receive(:send_message) + Net::SMTP.stub(:start).and_yield @fake_smtp + end +end diff --git a/spec/views/notices/new.html.erb_spec.rb b/spec/views/notices/new.html.erb_spec.rb index f602e2e2e..4a6df3949 100644 --- a/spec/views/notices/new.html.erb_spec.rb +++ b/spec/views/notices/new.html.erb_spec.rb @@ -204,12 +204,6 @@ text: third_topic.name ) end - - private - - def have_nth_option(n, value) - have_css("option:nth-child(#{n})", text: value) - end end context 'step headings' do