Merge branch 'dev'

.travis.yml

@@ -9,6 +9,7 @@ addons:
   postgresql: "9.6"
 
 services:
+  - xvfb
   - postgresql
   # We don't start elasticsearch here; instead we download it later and run it
   # directly in order to enable the test cluster to start up. See
@@ -20,7 +21,6 @@ cache:
 
 before_install:
   - export DISPLAY=:99.0
-  - sh -e /etc/init.d/xvfb start
   - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz
   - tar -xvf elasticsearch-5.6.4.tar.gz
 
@@ -37,7 +37,7 @@ before_install:
       phantomjs -v
 
 env:
-  - TEST_CLUSTER_COMMAND=elasticsearch-5.6.4/bin/elasticsearch
+  - TEST_CLUSTER_COMMAND=elasticsearch-5.6.4/bin/elasticsearch SEARCH_SLEEP=1.5
 
 before_script:
   - psql -c 'create database chill_test;' -U postgres

CHANGELOG.md

@@ -1,5 +1,21 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). It uses [CalVer](https://calver.org/) as of May 2019.
 
+## [19.07](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.07) - 2019-07-01
+### Added
+* Compresses http responses
+* `get_approximate_count` method on `Notice` and `InfringingUrl` (uses postgres reltuples)
+* Whitelists logged-in users using the web interface (if they would not be throttled using the API)
+
+### Changed
+* Updates numerous dependencies
+* Refactors court order reporting cron job
+* Only includes notices of type supporting in this cron job
+
+### Fixed
+* Bug with captcha validation in notice requests (pushed out before this release as a hotfix)
+* Broken link in admin site
+* Strips special characters from cron job filenames to suppress annoying warning emails to admins
+
 ## [19.05](https://github.com/berkmancenter/lumendatabase/releases/tag/2019.05) - 2019-05-20
 
 ### Added

Gemfile

@@ -59,7 +59,7 @@ group :development do
 end
 
 group :development, :test do
-  gem 'factory_girl_rails'
+  gem "factory_bot_rails"
   gem 'phantomjs'
   gem 'pry', '~> 0.10.4'
   gem 'pry-byebug'

Gemfile.lock

@@ -141,11 +141,11 @@ GEM
       multi_json
     erubis (2.7.0)
     execjs (2.7.0)
-    factory_girl (4.9.0)
-      activesupport (>= 3.0.0)
-    factory_girl_rails (4.9.0)
-      factory_girl (~> 4.9.0)
-      railties (>= 3.0.0)
+    factory_bot (5.0.2)
+      activesupport (>= 4.2.0)
+    factory_bot_rails (5.0.2)
+      factory_bot (~> 5.0.2)
+      railties (>= 4.2.0)
     fakeweb (1.3.0)
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
@@ -419,7 +419,7 @@ DEPENDENCIES
   elasticsearch-extensions
   elasticsearch-model (~> 5.0)
   elasticsearch-rails (~> 5.0)
-  factory_girl_rails
+  factory_bot_rails
   fakeweb
   flutie
   high_voltage

README.md

@@ -42,7 +42,7 @@ By default the app will try to connect to Elasticsearch on `http://localhost:920
 
     $BROWSER 'http://localhost:3000'
 
-You can customize behavior during seeding (db:setup) with a couple environment variables:
+You can customize behavior during seeding (db:setup) with a couple of environment variables:
 
 * `NOTICE_COUNT=10` will generate 10 (or any number you pass it) notices
   instead of the default 500
@@ -89,24 +89,61 @@ their own commit.
 
 #### Environment variables
 
-- `RAILS_LOG_LEVEL` - logging level
-- `WEB_CONCURRENCY` - number of Unicorn workers
-- `WEB_TIMEOUT` - Unicorn timeout
-- `MAILER_DELIVERY_METHOD` - mailer delivery method
+Here are all the environment variables which Lumen recognizes. Find them in the code for documentation.
+
+Environment variables should be set in `.env` and are managed by the `dotenv` gem. `.env` is not version-controlled so you can safely write secrets to it (but will also need to set these on all servers).
+
+Unless setting an environment variable on the command line in the context of a command-line process, environment variables should ONLY be set in `.env`.
+
+Most of these are optional and have sensible defaults (which may vary by environment).
+
+- `BATCH_SIZE` - batch size of model items indexed during each run of Elasticsearch re-indexing
+- `BUNDLE_GEMFILE`
+- `BROWSER_VALIDATIONS` - enable user html5 browser form validations
 - `DEFAULT_SENDER` - default mailer sender
+- `ELASTICSEARCH_URL`
+- `EMAIL_DOMAIN`
+- `ES_INDEX_SUFFIX` - can be used to specify a suffix for the name of elasticsearch indexes
+- `FILE_NAME` - name of csv file to import as blog entries
+- `from` - a date formatted `'%Y-%m-%d'` for use in recreating elasticsearch indexes after said date
+- `GOOGLE_CUSTOM_BLOG_SEARCH_ID`
+- `IMPORT_NAME`
+- `LOG_ELASTICSEARCH` - only used in tests
+- `LUMEN_USE_OINK`
+- `MAILER_DELIVERY_METHOD`
+- `NOTICE_COUNT` - how many fake notices to create when seeding the db
+- `RACK_ENV` - don't use this; it's overridden by `RAILS_ENV`
+- `RAILS_ENV`
+- `RAILS_LOG_LEVEL`
+- `RAILS_SERVE_STATIC_FILES` - if present (with any value) will enable rails to serve static files
+- `RECAPTCHA_SITE_KEY` - reCAPTCHA public key
+- `RECAPTCHA_SECRET_KEY` - reCAPTCHA private key
 - `RETURN_PATH` - default mailer return path
+- `SEARCH_SLEEP` - used in specs only, time out of Elasticsearch searches
+- `SECRET_KEY_BASE` - the Rails secret token; _required in prod_
 - `SITE_HOST` - site host, used in mailer templates
+- `SKIP_FAKE_DATA` - don't generate fake data when seeding the database
 - `SMTP_ADDRESS` - SMTP server address
 - `SMTP_DOMAIN` - SMTP server domain
 - `SMTP_USERNAME` - SMTP server username
 - `SMTP_PASSWORD` - SMTP server password
 - `SMTP_PORT` - SMTP server port
-- `RECAPTCHA_SITE_KEY` - reCAPTCHA public key
-- `RECAPTCHA_SECRET_KEY` - reCAPTCHA private key
-- `BROWSER_VALIDATIONS` - enable user browser form validations
-- `BATCH_SIZE` - batch size of model items indexed during each run of Elasticsearch re-indexing
-- `SEARCH_SLEEP` - used in specs only, time out of Elasticsearch searches
-- `RACK_ENV` - Rack environment
+- `SMTP_VERIFY_MODE`
+- `TABLES_USING_NAIVE_COUNTS` - prevents Kaminari from doing slow/full counts of the specified tables
+- `TEST_CLUSTER_COMMAND`
+- `USER_CRON_EMAIL` - for use in sending reports of court order files; can be a string or a list (in a JSON.parse-able format)
+- `USER_CRON_MAGIC_DIR`
+- `WEB_CONCURRENCY` - number of Unicorn workers
+- `WEB_TIMEOUT` - Unicorn timeout
+- The following are used only for imports from oldchill:
+  - `BASE_DIRECTORY`
+  - `MYSQL_DATABASE`
+  - `MYSQL_HOST`
+  - `MYSQL_USERNAME`
+  - `MYSQL_PASSWORD`
+  - `MYSQL_PORT`
+  - `RESTART_SEQUENCE_WITH` - for compatibility between oldchill imports and new Lumen notices. Should not ever be needed at this point, nor have any effect in production.
+  - `WHERE`
 
 #### Email setup
 

lib/tasks/lumen.rake

@@ -746,7 +746,7 @@ class CourtOrderReporter
     fetch_files
     write_files
     make_archive
-    email_user
+    email_users
   end
 
   def setup_directories
@@ -797,7 +797,14 @@ class CourtOrderReporter
 
   def notify_about_unredacted_files(f)
     return if f.notice.file_uploads.where(kind: 'supporting').present?
-    File.write(@info_filepath, app.notice_url(f.notice), mode: 'a')
+    File.write(
+      @info_filepath,
+      Rails.application.routes.url_helpers.notice_url(
+        f.notice,
+        host: Rails.application.config.action_mailer.default_url_options[:host]
+      ),
+      mode: 'a'
+    )
   end
 
   def make_archive
@@ -807,23 +814,39 @@ class CourtOrderReporter
     system("rm -r #{@working_dir}")
   end
 
-  def email_user
-    email = ENV['USER_CRON_EMAIL']
-    unless (email && defined? SMTP_SETTINGS)
+  def email_users
+    unless (emails && defined? SMTP_SETTINGS)
       Rails.logger.warn '[rake] Missing email or SMTP_SETTINGS; not emailing court order report'
       exit
     end
 
     Rails.logger.info '[rake] Sending court order report email'
-    mailtext = <<~HEREDOC
-      Subject: Latest email archive from Lumen
 
-      The latest archive of Lumen court order files can be found at
-      #{ Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz.
-    HEREDOC
+    emails.each do |email|
+      email_single_user(email)
+    end
+  end
 
+  def emails
+    @emails ||= begin
+      JSON.parse ENV['USER_CRON_EMAIL']   # list of emails
+    rescue JSON::ParserError
+      [ENV['USER_CRON_EMAIL']]            # single-email string, as 1-item list
+    end
+  end
+
+  def email_single_user(email)
     Net::SMTP.start(SMTP_SETTINGS[:address]) do |smtp|
       smtp.send_message mailtext, 'no-reply@lumendatabase.org', email
     end
   end
+
+  def mailtext
+    @mailtext ||= <<~HEREDOC
+        Subject: Latest email archive from Lumen
+
+        The latest archive of Lumen court order files can be found at
+        #{Chill::Application.config.site_host}/#{@magic_dir}/#{@archive_filename}.tar.gz.
+    HEREDOC
+  end
 end

spec/config/initializers/rack-attack_spec.rb

@@ -9,7 +9,7 @@ def app
   end
 
   describe 'throttling excessive API requests by IP address', cache: true do
-    let(:limit) { 5 }
+    let(:limit) { 1 }
 
     context 'number of requests is lower than the limit' do
       it 'allows requests' do
@@ -31,7 +31,7 @@ def app
   end
 
   describe 'throttling excessive HTTP responses by IP address' do
-    let(:limit) { 10 }
+    let(:limit) { 1 }
 
     context 'number of requests is lower than the limit', cache: true do
       it 'allows requests' do