Merge branch 'master' into master-legacy

.travis.yml

@@ -9,6 +9,7 @@ addons:
   postgresql: "9.6"
 
 services:
+  - xvfb
   - postgresql
   # We don't start elasticsearch here; instead we download it later and run it
   # directly in order to enable the test cluster to start up. See
@@ -20,7 +21,6 @@ cache:
 
 before_install:
   - export DISPLAY=:99.0
-  - sh -e /etc/init.d/xvfb start
   - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz
   - tar -xvf elasticsearch-5.6.4.tar.gz
 

CHANGELOG.md

@@ -1,5 +1,21 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). It uses [CalVer](https://calver.org/) as of May 2019.
 
+## [19.07](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.07) - 2019-07-01
+### Added
+* Compresses http responses
+* `get_approximate_count` method on `Notice` and `InfringingUrl` (uses postgres reltuples)
+* Whitelists logged-in users using the web interface (if they would not be throttled using the API)
+
+### Changed
+* Updates numerous dependencies
+* Refactors court order reporting cron job
+* Only includes notices of type supporting in this cron job
+
+### Fixed
+* Bug with captcha validation in notice requests (pushed out before this release as a hotfix)
+* Broken link in admin site
+* Strips special characters from cron job filenames to suppress annoying warning emails to admins
+
 ## [19.05](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.05) - 2019-05-20
 
 ### Added

README.md

@@ -131,7 +131,7 @@ Most of these are optional and have sensible defaults (which may vary by environ
 - `SMTP_VERIFY_MODE`
 - `TABLES_USING_NAIVE_COUNTS` - prevents Kaminari from doing slow/full counts of the specified tables
 - `TEST_CLUSTER_COMMAND`
-- `USER_CRON_EMAIL` - for use in sending reports of court order files
+- `USER_CRON_EMAIL` - for use in sending reports of court order files; can be a string or a list (in a JSON.parse-able format)
 - `USER_CRON_MAGIC_DIR`
 - `WEB_CONCURRENCY` - number of Unicorn workers
 - `WEB_TIMEOUT` - Unicorn timeout

lib/tasks/lumen.rake

@@ -827,7 +827,7 @@ class CourtOrderReporter
     fetch_files
     write_files
     make_archive
-    email_user
+    email_users
   end
 
   def setup_directories
@@ -878,7 +878,14 @@ class CourtOrderReporter
 
   def notify_about_unredacted_files(f)
     return if f.notice.file_uploads.where(kind: 'supporting').present?
-    File.write(@info_filepath, app.notice_url(f.notice), mode: 'a')
+    File.write(
+      @info_filepath,
+      Rails.application.routes.url_helpers.notice_url(
+        f.notice,
+        host: Rails.application.config.action_mailer.default_url_options[:host]
+      ),
+      mode: 'a'
+    )
   end
 
   def make_archive
@@ -888,23 +895,39 @@ class CourtOrderReporter
     system("rm -r #{@working_dir}")
   end
 
-  def email_user
-    email = ENV['USER_CRON_EMAIL']
-    unless (email && defined? SMTP_SETTINGS)
+  def email_users
+    unless (emails && defined? SMTP_SETTINGS)
       Rails.logger.warn '[rake] Missing email or SMTP_SETTINGS; not emailing court order report'
       exit
     end
 
     Rails.logger.info '[rake] Sending court order report email'
-    mailtext = <<~HEREDOC
-      Subject: Latest email archive from Lumen
 
-      The latest archive of Lumen court order files can be found at
-      #{ Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz.
-    HEREDOC
+    emails.each do |email|
+      email_single_user(email)
+    end
+  end
 
+  def emails
+    @emails ||= begin
+      JSON.parse ENV['USER_CRON_EMAIL']   # list of emails
+    rescue JSON::ParserError
+      [ENV['USER_CRON_EMAIL']]            # single-email string, as 1-item list
+    end
+  end
+
+  def email_single_user(email)
     Net::SMTP.start(SMTP_SETTINGS[:address]) do |smtp|
       smtp.send_message mailtext, 'no-reply@lumendatabase.org', email
     end
   end
+
+  def mailtext
+    @mailtext ||= <<~HEREDOC
+        Subject: Latest email archive from Lumen
+
+        The latest archive of Lumen court order files can be found at
+        #{Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz.
+    HEREDOC
+  end
 end

spec/config/initializers/rack-attack_spec.rb

@@ -9,7 +9,7 @@ def app
   end
 
   describe 'throttling excessive API requests by IP address', cache: true do
-    let(:limit) { 5 }
+    let(:limit) { 1 }
 
     context 'number of requests is lower than the limit' do
       it 'allows requests' do
@@ -31,7 +31,7 @@ def app
   end
 
   describe 'throttling excessive HTTP responses by IP address' do
-    let(:limit) { 10 }
+    let(:limit) { 1 }
 
     context 'number of requests is lower than the limit', cache: true do
       it 'allows requests' do

spec/integration/getting_topics_through_api_spec.rb

@@ -2,6 +2,7 @@
 
 feature "Topic lists as json" do
   scenario "an API consumer gets a list of topics" do
+    Topic.destroy_all
     create(:topic, name: 'A topic')
     create(:topic, name: 'Another topic')
 

spec/tasks/generate_court_order_report_spec.rb

@@ -0,0 +1,43 @@
+require 'rails_helper'
+
+describe 'rake lumen:generate_court_order_report', type: :task do
+  before :all do
+    create(:court_order, :with_document)
+    @cached_user_cron_email = ENV['USER_CRON_EMAIL']
+  end
+
+  before(:each) do
+    stub_const('SMTP_SETTINGS', { address: 'test@example.com' })
+  end
+
+  after :all do
+    ENV['USER_CRON_EMAIL'] = @cached_user_cron_email
+    CourtOrder.destroy_all
+  end
+
+  it 'sends a single email' do
+    ENV['USER_CRON_EMAIL'] = 'foo@example.com'
+    stub_smtp
+    expect(@fake_smtp).to receive(:send_message)
+      .with(anything, anything, 'foo@example.com')
+    task.execute
+  end
+
+  it 'sends emails to a list' do
+    ENV['USER_CRON_EMAIL'] = '["foo@example.com", "bar@example.com"]'
+    stub_smtp
+    expect(@fake_smtp).to receive(:send_message)
+      .with(anything, anything, 'foo@example.com')
+    expect(@fake_smtp).to receive(:send_message)
+      .with(anything, anything, 'bar@example.com')
+    task.execute
+  end
+
+  # SMTP may or may not be configured on the machines this test will run on,
+  # so let's stub it out.
+  def stub_smtp
+    @fake_smtp = double('Net::STMP.new')
+    allow(@fake_smtp).to receive(:send_message)
+    Net::SMTP.stub(:start).and_yield @fake_smtp
+  end
+end

spec/views/notices/new.html.erb_spec.rb

@@ -204,12 +204,6 @@
         text: third_topic.name
       )
     end
-
-    private
-
-    def have_nth_option(n, value)
-      have_css("option:nth-child(#{n})", text: value)
-    end
   end
 
   context 'step headings' do