Updates to take_observation (#1009)

Thanks @danjampro for suggestions!

* * Check for POINTING keyword in the metadata, not in the FITS header.
* "Better" assigning of metadata to fits keywords.
* More explicit variable names for FITS metadata.

* * `take_observation` has a `blocking` param.
* Fixing some of the dslr simulator tests.

* * Changelog updates.

* Using `get_quantity_value` for `exptime` lookup.

* * Adding camera defaults section and passing that to cameras.
* Fix the default keyword params for booleans.

* * Typo fix in config.

* * Comment out the focuser and cooliing

* * Update changelog to reflection changes to config options.

CHANGELOG.rst

@@ -11,11 +11,11 @@ Added
 * A "developer" version of the ``panoptes-pocs`` docker image is cloudbuilt automatically on merge with ``develop``. (@wtgee #1010)
 * Better error checking in cameras, including ability to store error. (@AnthonyHorton #1007)
 * Added ``error.InvalidConfig`` exception. (@wtgee #1007)
-* Config options to control observation processing options: (@wtgee #1007)
+* Config options to control camera processing options and allow for `defaults` in the config that applies to all cameras: (@wtgee #1007)
 
-  * ``observations.compress_fits`` if FITS files should be fpacked. Default True.
-  * ``observations.record_observations`` if observation metadata should be recorded. Default True.
-  * ``observations.make_pretty_images`` to make jpgs from images. Default True.
+  * ``cameras.defaults.compress_fits`` if FITS files should be fpacked. Default True.
+  * ``cameras.defaults.record_observations`` if observation metadata should be recorded. Default True.
+  * ``cameras.defaults.make_pretty_images`` to make jpgs from images. Default True.
 
 Breaking
 ~~~~~~~~
@@ -35,6 +35,17 @@ Bug fixes
 Changed
 ~~~~~~~
 
+* Camera observation updates:
+
+  * headers param fixed so truly optional. The POINTING keyword is checked in the metadata, not original headers. Closes #1002. (@wtgee #1009)
+  * Passing approved headers will actually write them to file. (@wtgee #1009)
+  * ``blocking=False`` param added. If True, will wait on observation_event. (@wtgee #1009)
+  * Renamed metadata variables to be consistent. (@wtgee #1009)
+  * ``_process_fits`` is responsible for writing the headers rather than calling out to panoptes-utils. Allows for easier overrides. (@wtgee #1009)
+  * dslr simulator readout time improved. (@wtgee #1009)
+  * ``process_exposure`` doesn't require the exposure_event to be passed because that is the cameras is_exposing property. (@wtgee #1009)
+
+
 * Changelog cleanup. (@wtgee #1008)
 * ``panoptes-utils`` updates:
 

conf_files/pocs.yaml

@@ -22,7 +22,7 @@ location:
   flat_horizon: -6 deg # Flats when sun between this and focus horizon.
   focus_horizon: -12 deg # Dark enough to focus on stars.
   observe_horizon: -18 deg # Sun below this limit to observe.
-  obsctructions: []
+  obstructions: [ ]
   timezone: US/Hawaii
   gmt_offset: -600 # Offset in minutes from GMT during.
 
@@ -69,11 +69,41 @@ pointing:
   max_iterations: 5
 
 cameras:
-  auto_detect: True
-  primary: 14d3bd
+  defaults:
+    primary: None
+    auto_detect: False
+    file_extension: fits
+    compress_fits: False
+    make_pretty_images: False
+    keep_jpgs: False
+    readout_time: 0.5  # seconds
+    timeout: 10  # seconds
+    filter_type: RGGB
+    cooling:
+      enabled: False
+      temperature:
+        target: 0  # celsius
+        tolerance: 0.1  # celsius
+      stable_time: 60  # seconds
+      check_interval: 5  # seconds
+      timeout: 300  # seconds
+    filterwheel:
+      model: panoptes.pocs.filterwheel.simulator.FilterWheel
+      filter_names: [ ]
+      move_time: 0.1  # seconds
+      timeout: 0.5  # seconds
+    focuser:
+      enabled: False
+      autofocus_seconds: 0.1  # seconds
+      autofocus_size: 500  # seconds
+      autofocus_keep_files: False
   devices:
-    - model: canon_gphoto2
-    - model: canon_gphoto2
+    - model: panoptes.pocs.camera.gphoto.canon.Camera
+      name: dslr.00
+      file_extension: cr2
+    - model: panoptes.pocs.camera.gphoto.canon.Camera
+      name: dslr.01
+      file_extension: cr2
 
 ######################### Environmental Sensors ################################
 # Configure the environmental sensors that are attached.

docker/developer/Dockerfile

@@ -46,7 +46,7 @@ RUN echo "Installing developer tools" && \
         && \
     # Set some jupyterlab defaults.
     mkdir -p /home/panoptes/.jupyter && \
-    /usr/bin/env zsh -c "${PANDIR}/conda/bin/jupyter-lab --no-browser --generate-config" && \
+    /usr/bin/env zsh -c "${PANDIR}/conda/bin/jupyter-lab -y --no-browser --generate-config" && \
     # Jupyterlab extensions.
     echo "c.JupyterApp.answer_yes = True" >> \
         "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \

src/panoptes/pocs/camera/__init__.py

@@ -87,12 +87,14 @@ def create_cameras_from_config(config=None,
         # cameras section either missing or empty
         logger.info('No camera information in config.')
         return None
+
     logger.debug(f"{camera_config=}")
+    camera_defaults = camera_config.get('defaults', dict())
 
     cameras = cameras or OrderedDict()
     ports = list()
 
-    auto_detect = camera_config.get('auto_detect', False)
+    auto_detect = camera_defaults.get('auto_detect', False)
 
     # Lookup the connected ports
     if auto_detect:
@@ -110,7 +112,11 @@ def create_cameras_from_config(config=None,
     primary_camera = None
 
     device_info = camera_config['devices']
-    for cam_num, device_config in enumerate(device_info):
+    for cam_num, cfg in enumerate(device_info):
+        # Get a copy of the camera defaults and update with device config.
+        device_config = camera_defaults.copy()
+        device_config.update(cfg)
+
         cam_name = device_config.setdefault('name', f'Cam{cam_num:02d}')
 
         # Check for proper connection method.

src/panoptes/pocs/camera/camera.py

@@ -369,7 +369,7 @@ def exposure_error(self):
     def connect(self):
         raise NotImplementedError  # pragma: no cover
 
-    def take_observation(self, observation, headers=None, filename=None, **kwargs):
+    def take_observation(self, observation, headers=None, filename=None, blocking=False, **kwargs):
         """Take an observation
 
         Gathers various header information, sets the file path, and calls
@@ -383,7 +383,9 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs):
                 describing the observation
             headers (dict, optional): Header data to be saved along with the file.
             filename (str, optional): pass a filename for the output FITS file to
-                overrride the default file naming system
+                override the default file naming system.
+            blocking (bool): If method should wait for observation event to be complete
+                before returning, default False.
             **kwargs (dict): Optional keyword arguments (`exptime`, dark)
 
         Returns:
@@ -400,11 +402,11 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs):
         exptime = kwargs.pop('exptime', observation.exptime.value)
 
         # start the exposure
-        exposure_event = self.take_exposure(seconds=exptime, filename=file_path, **kwargs)
+        self.take_exposure(seconds=exptime, filename=file_path, blocking=blocking, **kwargs)
 
         # Add most recent exposure to list
         if self.is_primary:
-            if 'POINTING' in headers:
+            if 'POINTING' in metadata:
                 observation.pointing_images[image_id] = file_path
             else:
                 observation.exposure_list[image_id] = file_path
@@ -414,11 +416,15 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs):
         t = threading.Thread(
             name=f'Thread-{image_id}',
             target=self.process_exposure,
-            args=(metadata, observation_event, exposure_event),
+            args=(metadata, observation_event),
             daemon=True)
-        t.name = f'{self.name}Thread'
         t.start()
 
+        if blocking:
+            while not observation_event.is_set():
+                self.logger.trace(f'Waiting for observation event')
+                time.sleep(0.5)
+
         return observation_event
 
     def take_exposure(self,
@@ -521,9 +527,8 @@ def take_exposure(self,
         return self._is_exposing_event
 
     def process_exposure(self,
-                         info,
+                         metadata,
                          observation_event,
-                         exposure_event=None,
                          compress_fits=None,
                          record_observations=None,
                          make_pretty_images=None):
@@ -542,11 +547,9 @@ def process_exposure(self,
         `current` collection. Saves metadata to `observations` collection for all images.
 
         Args:
-            info (dict): Header metadata saved for the image
+            metadata (dict): Header metadata saved for the image
             observation_event (threading.Event): An event that is set signifying that the
                 camera is done with this exposure
-            exposure_event (threading.Event, optional): An event that should be set
-                when the exposure is complete, triggering the processing.
             compress_fits (bool or None): If FITS files should be fpacked into .fits.fz.
                 If None (default), checks the `observations.compress_fits` config-server key.
             record_observations (bool or None): If observation metadata should be saved.
@@ -559,52 +562,56 @@ def process_exposure(self,
         Raises:
             FileNotFoundError: If the FITS file isn't at the specified location.
         """
-        # If passed an Event that signals the end of the exposure wait for it to be set
-        compress_fits = compress_fits or self.get_config('observations.compress_fits')
-        make_pretty_images = make_pretty_images or self.get_config('observations.make_pretty_images')
-
-        # Wait for exposure to complete.
+        # Wait for exposure to complete. Timeout handled by exposure thread.
         while self.is_exposing:
             time.sleep(1)
 
-        image_id = info['image_id']
-        seq_id = info['sequence_id']
-        file_path = info['file_path']
-        exptime = info['exptime']
-        field_name = info['field_name']
+        self.logger.debug(f'Starting exposure processing for {observation_event}')
+
+        if compress_fits is None:
+            compress_fits = self.get_config('observations.compress_fits', default=False)
+
+        if make_pretty_images is None:
+            make_pretty_images = self.get_config('observations.make_pretty_images', default=False)
+
+        image_id = metadata['image_id']
+        seq_id = metadata['sequence_id']
+        file_path = metadata['file_path']
+        exptime = metadata['exptime']
+        field_name = metadata['field_name']
 
         # Make sure image exists.
         if not os.path.exists(file_path):
             observation_event.set()
-            raise FileNotFoundError(f"Expected image at '{file_path}' does not exist or " +
+            raise FileNotFoundError(f"Expected image at {file_path=} does not exist or " +
                                     "cannot be accessed, cannot process.")
 
         self.logger.debug(f'Starting FITS processing for {file_path}')
-        file_path = self._process_fits(file_path, info)
+        file_path = self._process_fits(file_path, metadata)
         self.logger.debug(f'Finished FITS processing for {file_path}')
 
+        # TODO make this async and take it out of camera.
         if make_pretty_images:
             try:
                 image_title = f'{field_name} [{exptime}s] {seq_id}'
 
                 self.logger.debug(f"Making pretty image for {file_path=}")
                 link_path = None
-                if info['is_primary']:
+                if metadata['is_primary']:
                     # This should be in the config somewhere.
                     link_path = os.path.expandvars('$PANDIR/images/latest.jpg')
 
                 img_utils.make_pretty_image(file_path,
                                             title=image_title,
                                             link_path=link_path)
             except Exception as e:  # pragma: no cover
-                self.logger.warning('Problem with extracting pretty image: {e!r}')
+                self.logger.warning(f'Problem with extracting pretty image: {e!r}')
 
-        with suppress(Exception):
-            info['exptime'] = info['exptime'].value
+        metadata['exptime'] = get_quantity_value(metadata['exptime'], unit='seconds')
 
         if record_observations:
             self.logger.debug(f"Adding current observation to db: {image_id}")
-            self.db.insert_current('observations', info)
+            self.db.insert_current('observations', metadata)
 
         if compress_fits:
             self.logger.debug(f'Compressing {file_path=}')
@@ -921,20 +928,53 @@ def _setup_observation(self, observation, headers, filename, **kwargs):
             metadata['filter_request'] = observation.filter_name
 
         if headers is not None:
+            self.logger.trace(f'Updating {file_path} metadata with provided headers')
             metadata.update(headers)
 
-        self.logger.debug(
-            f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} '
-            f'metadata={metadata}')
+        self.logger.debug(f'Observation setup: {exptime=} {file_path=} {image_id=} {metadata=}')
+
         return exptime, file_path, image_id, metadata
 
-    def _process_fits(self, file_path, info):
+    def _process_fits(self, file_path, metadata):
         """
-        Add FITS headers from info the same as images.cr2_to_fits()
+        Add FITS headers from metadata the same as images.cr2_to_fits()
         """
-        self.logger.debug(f"Updating FITS headers: {file_path}")
-        fits_utils.update_observation_headers(file_path, info)
-        self.logger.debug(f"Finished FITS headers: {file_path}")
+        # TODO (wtgee) I don't like this one bit.
+        fields = {
+            'image_id': {'keyword': 'IMAGEID'},
+            'sequence_id': {'keyword': 'SEQID'},
+            'field_name': {'keyword': 'FIELD'},
+            'ra_mnt': {'keyword': 'RA-MNT', 'comment': 'Degrees'},
+            'ha_mnt': {'keyword': 'HA-MNT', 'comment': 'Degrees'},
+            'dec_mnt': {'keyword': 'DEC-MNT', 'comment': 'Degrees'},
+            'equinox': {'keyword': 'EQUINOX', 'default': 2000.},
+            'airmass': {'keyword': 'AIRMASS', 'comment': 'Sec(z)'},
+            'filter': {'keyword': 'FILTER'},
+            'latitude': {'keyword': 'LAT-OBS', 'comment': 'Degrees'},
+            'longitude': {'keyword': 'LONG-OBS', 'comment': 'Degrees'},
+            'elevation': {'keyword': 'ELEV-OBS', 'comment': 'Meters'},
+            'moon_separation': {'keyword': 'MOONSEP', 'comment': 'Degrees'},
+            'moon_fraction': {'keyword': 'MOONFRAC'},
+            'creator': {'keyword': 'CREATOR', 'comment': 'POCS Software version'},
+            'camera_uid': {'keyword': 'INSTRUME', 'comment': 'Camera ID'},
+            'observer': {'keyword': 'OBSERVER', 'comment': 'PANOPTES Unit ID'},
+            'origin': {'keyword': 'ORIGIN'},
+            'tracking_rate_ra': {'keyword': 'RA-RATE', 'comment': 'RA Tracking Rate'},
+        }
+
+        self.logger.debug(f"Updating FITS headers: {file_path} with {metadata=}")
+        with fits.open(file_path, 'update') as f:
+            hdu = f[0]
+            for metadata_key, field_info in fields.items():
+                fits_key = field_info['keyword']
+                fits_comment = field_info.get('comment', '')
+                # Get the value from either the metadata, the default, or use blank string.
+                fits_value = metadata.get(metadata_key, field_info.get('default', ''))
+
+                self.logger.trace(f'Setting {fits_key=} = {fits_value=} {fits_comment=}')
+                hdu.header.set(fits_key, fits_value, fits_comment)
+
+            self.logger.debug(f"Finished FITS headers: {file_path}")
 
         return file_path
 

src/panoptes/pocs/camera/gphoto/canon.py

@@ -121,7 +121,7 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw
         # Process the image after a set amount of time
         wait_time = exptime + self.readout_time
 
-        t = Timer(wait_time, self.process_exposure, (metadata, observation_event, exposure_event))
+        t = Timer(wait_time, self.process_exposure, (metadata, observation_event))
         t.name = f'{self.name}Thread'
         t.start()
 

src/panoptes/pocs/camera/simulator/dslr.py

@@ -12,7 +12,7 @@
 
 from panoptes.pocs.camera import AbstractCamera
 from panoptes.utils.images import fits as fits_utils
-from panoptes.utils import get_quantity_value
+from panoptes.utils import get_quantity_value, CountdownTimer
 
 
 class Camera(AbstractCamera):
@@ -64,6 +64,7 @@ def _start_exposure(self, seconds=None, filename=None, dark=False, header=None,
         return readout_args
 
     def _readout(self, filename=None, header=None):
+        timer = CountdownTimer(duration=self.readout_time)
         self.logger.trace(f'Calling _readout for {self}')
         # Get example FITS file from test data directory
         file_path = os.path.join(
@@ -78,12 +79,14 @@ def _readout(self, filename=None, header=None):
             fake_data = np.random.randint(low=975, high=1026,
                                           size=fake_data.shape,
                                           dtype=fake_data.dtype)
-        time.sleep(self.readout_time)
         self.logger.debug(f'Writing {filename=} for {self}')
         fits_utils.write_fits(fake_data, header, filename)
 
-    def _process_fits(self, file_path, info):
-        file_path = super()._process_fits(file_path, info)
+        # Sleep for the remainder of the readout time.
+        timer.sleep()
+
+    def _process_fits(self, file_path, metadata):
+        file_path = super()._process_fits(file_path, metadata)
         self.logger.debug('Overriding mount coordinates for camera simulator')
         # TODO get the path as package data or something better.
         solved_path = os.path.join(