diff --git a/CRAN-RELEASE b/CRAN-RELEASE
new file mode 100644
index 00000000..b8f6162b
--- /dev/null
+++ b/CRAN-RELEASE
@@ -0,0 +1,2 @@
+This package was submitted to CRAN on 2019-08-26.
+Once it is accepted, delete this file and tag the release (commit 5bb75fc46d).
diff --git a/DESCRIPTION b/DESCRIPTION
index 3ff2e184..cd29b660 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: yardstick
 Type: Package
 Title: Tidy Characterizations of Model Performance
-Version: 0.0.3.9000
+Version: 0.0.4
 Authors@R: c(
     person("Max", "Kuhn", , "max@rstudio.com", c("aut")),
     person("Davis", "Vaughan", , "davis@rstudio.com", c("aut", "cre")),
diff --git a/NEWS.md b/NEWS.md
index 6cea40a7..46259097 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,4 +1,4 @@
-# yardstick (development version)
+# yardstick 0.0.4
 
 ## New metrics and functionality
 
diff --git a/R/metrics.R b/R/metrics.R
index fd3bac24..1cc838cf 100644
--- a/R/metrics.R
+++ b/R/metrics.R
@@ -348,6 +348,7 @@ make_prob_class_metric_function <- function(fns) {
 
   class(metric_function) <- c(
     "class_prob_metric_set",
+    "metric_set",
     class(metric_function)
   )
 
@@ -387,6 +388,7 @@ make_numeric_metric_function <- function(fns) {
 
   class(metric_function) <- c(
     "numeric_metric_set",
+    "metric_set",
     class(metric_function)
   )
 
diff --git a/cran-comments.md b/cran-comments.md
index ac2f93ce..21129a78 100644
--- a/cran-comments.md
+++ b/cran-comments.md
@@ -1,14 +1,9 @@
 ## Comments
 
-This is a small release mainly to resolve failing tests on CRAN. In particular, we have fixed:
-- "unable to re-encode" warnings
-- Failing tests due to R 3.6 `sample()` changes
-- An issue with an old release of R and dplyr 0.8.0.1 causing factors to be coerced to integers
-
-It also introduces a few new metrics, and squashes a few non-CRAN related bugs.
+This is a small release with a few new metrics, and a few squashed bugs.
 
 ## Test environments
-* local OS X install, R 3.5.2
+* local OS X install, R 3.6.0
 * ubuntu 14.04 (on travis-ci) (devel and release)
 * win-builder (devel and release)
 
diff --git a/tests/testthat/test_metrics.R b/tests/testthat/test_metrics.R
index 6e6afcb4..4ede6cde 100644
--- a/tests/testthat/test_metrics.R
+++ b/tests/testthat/test_metrics.R
@@ -135,6 +135,14 @@ test_that('metric set functions are classed', {
     metric_set(mae),
     "numeric_metric_set"
   )
+  expect_is(
+    metric_set(accuracy, roc_auc),
+    "metric_set"
+  )
+  expect_is(
+    metric_set(mae),
+    "metric_set"
+  )
 })
 
 test_that('metric set functions retain class/prob metric functions', {