diff --git a/dopamine/agents/rainbow/rainbow_agent.py b/dopamine/agents/rainbow/rainbow_agent.py
index 00b6d138..7e3e4ebb 100644
--- a/dopamine/agents/rainbow/rainbow_agent.py
+++ b/dopamine/agents/rainbow/rainbow_agent.py
@@ -194,7 +194,8 @@ def _build_replay_buffer(self, use_staging):
         stack_size=self.stack_size,
         use_staging=use_staging,
         update_horizon=self.update_horizon,
-        gamma=self.gamma)
+        gamma=self.gamma,
+        observation_dtype=self.observation_dtype.as_numpy_dtype)
 
   def _build_target_distribution(self):
     """Builds the C51 target distribution as per Bellemare et al. (2017).