diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5c50b7260..99b6107f1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,11 +11,11 @@ Added * A "developer" version of the ``panoptes-pocs`` docker image is cloudbuilt automatically on merge with ``develop``. (@wtgee #1010) * Better error checking in cameras, including ability to store error. (@AnthonyHorton #1007) * Added ``error.InvalidConfig`` exception. (@wtgee #1007) -* Config options to control observation processing options: (@wtgee #1007) +* Config options to control camera processing options and allow for `defaults` in the config that applies to all cameras: (@wtgee #1007) - * ``observations.compress_fits`` if FITS files should be fpacked. Default True. - * ``observations.record_observations`` if observation metadata should be recorded. Default True. - * ``observations.make_pretty_images`` to make jpgs from images. Default True. + * ``cameras.defaults.compress_fits`` if FITS files should be fpacked. Default True. + * ``cameras.defaults.record_observations`` if observation metadata should be recorded. Default True. + * ``cameras.defaults.make_pretty_images`` to make jpgs from images. Default True. Breaking ~~~~~~~~ @@ -35,6 +35,17 @@ Bug fixes Changed ~~~~~~~ +* Camera observation updates: + + * headers param fixed so truly optional. The POINTING keyword is checked in the metadata, not original headers. Closes #1002. (@wtgee #1009) + * Passing approved headers will actually write them to file. (@wtgee #1009) + * ``blocking=False`` param added. If True, will wait on observation_event. (@wtgee #1009) + * Renamed metadata variables to be consistent. (@wtgee #1009) + * ``_process_fits`` is responsible for writing the headers rather than calling out to panoptes-utils. Allows for easier overrides. (@wtgee #1009) + * dslr simulator readout time improved. (@wtgee #1009) + * ``process_exposure`` doesn't require the exposure_event to be passed because that is the cameras is_exposing property. (@wtgee #1009) + + * Changelog cleanup. (@wtgee #1008) * ``panoptes-utils`` updates: diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 58b9f7669..f72962358 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -22,7 +22,7 @@ location: flat_horizon: -6 deg # Flats when sun between this and focus horizon. focus_horizon: -12 deg # Dark enough to focus on stars. observe_horizon: -18 deg # Sun below this limit to observe. - obsctructions: [] + obstructions: [ ] timezone: US/Hawaii gmt_offset: -600 # Offset in minutes from GMT during. @@ -69,11 +69,41 @@ pointing: max_iterations: 5 cameras: - auto_detect: True - primary: 14d3bd + defaults: + primary: None + auto_detect: False + file_extension: fits + compress_fits: False + make_pretty_images: False + keep_jpgs: False + readout_time: 0.5 # seconds + timeout: 10 # seconds + filter_type: RGGB + cooling: + enabled: False + temperature: + target: 0 # celsius + tolerance: 0.1 # celsius + stable_time: 60 # seconds + check_interval: 5 # seconds + timeout: 300 # seconds + filterwheel: + model: panoptes.pocs.filterwheel.simulator.FilterWheel + filter_names: [ ] + move_time: 0.1 # seconds + timeout: 0.5 # seconds + focuser: + enabled: False + autofocus_seconds: 0.1 # seconds + autofocus_size: 500 # seconds + autofocus_keep_files: False devices: - - model: canon_gphoto2 - - model: canon_gphoto2 + - model: panoptes.pocs.camera.gphoto.canon.Camera + name: dslr.00 + file_extension: cr2 + - model: panoptes.pocs.camera.gphoto.canon.Camera + name: dslr.01 + file_extension: cr2 ######################### Environmental Sensors ################################ # Configure the environmental sensors that are attached. diff --git a/docker/developer/Dockerfile b/docker/developer/Dockerfile index 6583e0f79..f1f102eff 100644 --- a/docker/developer/Dockerfile +++ b/docker/developer/Dockerfile @@ -46,7 +46,7 @@ RUN echo "Installing developer tools" && \ && \ # Set some jupyterlab defaults. mkdir -p /home/panoptes/.jupyter && \ - /usr/bin/env zsh -c "${PANDIR}/conda/bin/jupyter-lab --no-browser --generate-config" && \ + /usr/bin/env zsh -c "${PANDIR}/conda/bin/jupyter-lab -y --no-browser --generate-config" && \ # Jupyterlab extensions. echo "c.JupyterApp.answer_yes = True" >> \ "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ diff --git a/src/panoptes/pocs/camera/__init__.py b/src/panoptes/pocs/camera/__init__.py index 49632613b..23f64433d 100644 --- a/src/panoptes/pocs/camera/__init__.py +++ b/src/panoptes/pocs/camera/__init__.py @@ -87,12 +87,14 @@ def create_cameras_from_config(config=None, # cameras section either missing or empty logger.info('No camera information in config.') return None + logger.debug(f"{camera_config=}") + camera_defaults = camera_config.get('defaults', dict()) cameras = cameras or OrderedDict() ports = list() - auto_detect = camera_config.get('auto_detect', False) + auto_detect = camera_defaults.get('auto_detect', False) # Lookup the connected ports if auto_detect: @@ -110,7 +112,11 @@ def create_cameras_from_config(config=None, primary_camera = None device_info = camera_config['devices'] - for cam_num, device_config in enumerate(device_info): + for cam_num, cfg in enumerate(device_info): + # Get a copy of the camera defaults and update with device config. + device_config = camera_defaults.copy() + device_config.update(cfg) + cam_name = device_config.setdefault('name', f'Cam{cam_num:02d}') # Check for proper connection method. diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index fc0d36b79..a1108f3b2 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -369,7 +369,7 @@ def exposure_error(self): def connect(self): raise NotImplementedError # pragma: no cover - def take_observation(self, observation, headers=None, filename=None, **kwargs): + def take_observation(self, observation, headers=None, filename=None, blocking=False, **kwargs): """Take an observation Gathers various header information, sets the file path, and calls @@ -383,7 +383,9 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs): describing the observation headers (dict, optional): Header data to be saved along with the file. filename (str, optional): pass a filename for the output FITS file to - overrride the default file naming system + override the default file naming system. + blocking (bool): If method should wait for observation event to be complete + before returning, default False. **kwargs (dict): Optional keyword arguments (`exptime`, dark) Returns: @@ -400,11 +402,11 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs): exptime = kwargs.pop('exptime', observation.exptime.value) # start the exposure - exposure_event = self.take_exposure(seconds=exptime, filename=file_path, **kwargs) + self.take_exposure(seconds=exptime, filename=file_path, blocking=blocking, **kwargs) # Add most recent exposure to list if self.is_primary: - if 'POINTING' in headers: + if 'POINTING' in metadata: observation.pointing_images[image_id] = file_path else: observation.exposure_list[image_id] = file_path @@ -414,11 +416,15 @@ def take_observation(self, observation, headers=None, filename=None, **kwargs): t = threading.Thread( name=f'Thread-{image_id}', target=self.process_exposure, - args=(metadata, observation_event, exposure_event), + args=(metadata, observation_event), daemon=True) - t.name = f'{self.name}Thread' t.start() + if blocking: + while not observation_event.is_set(): + self.logger.trace(f'Waiting for observation event') + time.sleep(0.5) + return observation_event def take_exposure(self, @@ -521,9 +527,8 @@ def take_exposure(self, return self._is_exposing_event def process_exposure(self, - info, + metadata, observation_event, - exposure_event=None, compress_fits=None, record_observations=None, make_pretty_images=None): @@ -542,11 +547,9 @@ def process_exposure(self, `current` collection. Saves metadata to `observations` collection for all images. Args: - info (dict): Header metadata saved for the image + metadata (dict): Header metadata saved for the image observation_event (threading.Event): An event that is set signifying that the camera is done with this exposure - exposure_event (threading.Event, optional): An event that should be set - when the exposure is complete, triggering the processing. compress_fits (bool or None): If FITS files should be fpacked into .fits.fz. If None (default), checks the `observations.compress_fits` config-server key. record_observations (bool or None): If observation metadata should be saved. @@ -559,37 +562,42 @@ def process_exposure(self, Raises: FileNotFoundError: If the FITS file isn't at the specified location. """ - # If passed an Event that signals the end of the exposure wait for it to be set - compress_fits = compress_fits or self.get_config('observations.compress_fits') - make_pretty_images = make_pretty_images or self.get_config('observations.make_pretty_images') - - # Wait for exposure to complete. + # Wait for exposure to complete. Timeout handled by exposure thread. while self.is_exposing: time.sleep(1) - image_id = info['image_id'] - seq_id = info['sequence_id'] - file_path = info['file_path'] - exptime = info['exptime'] - field_name = info['field_name'] + self.logger.debug(f'Starting exposure processing for {observation_event}') + + if compress_fits is None: + compress_fits = self.get_config('observations.compress_fits', default=False) + + if make_pretty_images is None: + make_pretty_images = self.get_config('observations.make_pretty_images', default=False) + + image_id = metadata['image_id'] + seq_id = metadata['sequence_id'] + file_path = metadata['file_path'] + exptime = metadata['exptime'] + field_name = metadata['field_name'] # Make sure image exists. if not os.path.exists(file_path): observation_event.set() - raise FileNotFoundError(f"Expected image at '{file_path}' does not exist or " + + raise FileNotFoundError(f"Expected image at {file_path=} does not exist or " + "cannot be accessed, cannot process.") self.logger.debug(f'Starting FITS processing for {file_path}') - file_path = self._process_fits(file_path, info) + file_path = self._process_fits(file_path, metadata) self.logger.debug(f'Finished FITS processing for {file_path}') + # TODO make this async and take it out of camera. if make_pretty_images: try: image_title = f'{field_name} [{exptime}s] {seq_id}' self.logger.debug(f"Making pretty image for {file_path=}") link_path = None - if info['is_primary']: + if metadata['is_primary']: # This should be in the config somewhere. link_path = os.path.expandvars('$PANDIR/images/latest.jpg') @@ -597,14 +605,13 @@ def process_exposure(self, title=image_title, link_path=link_path) except Exception as e: # pragma: no cover - self.logger.warning('Problem with extracting pretty image: {e!r}') + self.logger.warning(f'Problem with extracting pretty image: {e!r}') - with suppress(Exception): - info['exptime'] = info['exptime'].value + metadata['exptime'] = get_quantity_value(metadata['exptime'], unit='seconds') if record_observations: self.logger.debug(f"Adding current observation to db: {image_id}") - self.db.insert_current('observations', info) + self.db.insert_current('observations', metadata) if compress_fits: self.logger.debug(f'Compressing {file_path=}') @@ -921,20 +928,53 @@ def _setup_observation(self, observation, headers, filename, **kwargs): metadata['filter_request'] = observation.filter_name if headers is not None: + self.logger.trace(f'Updating {file_path} metadata with provided headers') metadata.update(headers) - self.logger.debug( - f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} ' - f'metadata={metadata}') + self.logger.debug(f'Observation setup: {exptime=} {file_path=} {image_id=} {metadata=}') + return exptime, file_path, image_id, metadata - def _process_fits(self, file_path, info): + def _process_fits(self, file_path, metadata): """ - Add FITS headers from info the same as images.cr2_to_fits() + Add FITS headers from metadata the same as images.cr2_to_fits() """ - self.logger.debug(f"Updating FITS headers: {file_path}") - fits_utils.update_observation_headers(file_path, info) - self.logger.debug(f"Finished FITS headers: {file_path}") + # TODO (wtgee) I don't like this one bit. + fields = { + 'image_id': {'keyword': 'IMAGEID'}, + 'sequence_id': {'keyword': 'SEQID'}, + 'field_name': {'keyword': 'FIELD'}, + 'ra_mnt': {'keyword': 'RA-MNT', 'comment': 'Degrees'}, + 'ha_mnt': {'keyword': 'HA-MNT', 'comment': 'Degrees'}, + 'dec_mnt': {'keyword': 'DEC-MNT', 'comment': 'Degrees'}, + 'equinox': {'keyword': 'EQUINOX', 'default': 2000.}, + 'airmass': {'keyword': 'AIRMASS', 'comment': 'Sec(z)'}, + 'filter': {'keyword': 'FILTER'}, + 'latitude': {'keyword': 'LAT-OBS', 'comment': 'Degrees'}, + 'longitude': {'keyword': 'LONG-OBS', 'comment': 'Degrees'}, + 'elevation': {'keyword': 'ELEV-OBS', 'comment': 'Meters'}, + 'moon_separation': {'keyword': 'MOONSEP', 'comment': 'Degrees'}, + 'moon_fraction': {'keyword': 'MOONFRAC'}, + 'creator': {'keyword': 'CREATOR', 'comment': 'POCS Software version'}, + 'camera_uid': {'keyword': 'INSTRUME', 'comment': 'Camera ID'}, + 'observer': {'keyword': 'OBSERVER', 'comment': 'PANOPTES Unit ID'}, + 'origin': {'keyword': 'ORIGIN'}, + 'tracking_rate_ra': {'keyword': 'RA-RATE', 'comment': 'RA Tracking Rate'}, + } + + self.logger.debug(f"Updating FITS headers: {file_path} with {metadata=}") + with fits.open(file_path, 'update') as f: + hdu = f[0] + for metadata_key, field_info in fields.items(): + fits_key = field_info['keyword'] + fits_comment = field_info.get('comment', '') + # Get the value from either the metadata, the default, or use blank string. + fits_value = metadata.get(metadata_key, field_info.get('default', '')) + + self.logger.trace(f'Setting {fits_key=} = {fits_value=} {fits_comment=}') + hdu.header.set(fits_key, fits_value, fits_comment) + + self.logger.debug(f"Finished FITS headers: {file_path}") return file_path diff --git a/src/panoptes/pocs/camera/gphoto/canon.py b/src/panoptes/pocs/camera/gphoto/canon.py index 7a0c01951..24b46a56a 100644 --- a/src/panoptes/pocs/camera/gphoto/canon.py +++ b/src/panoptes/pocs/camera/gphoto/canon.py @@ -121,7 +121,7 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw # Process the image after a set amount of time wait_time = exptime + self.readout_time - t = Timer(wait_time, self.process_exposure, (metadata, observation_event, exposure_event)) + t = Timer(wait_time, self.process_exposure, (metadata, observation_event)) t.name = f'{self.name}Thread' t.start() diff --git a/src/panoptes/pocs/camera/simulator/dslr.py b/src/panoptes/pocs/camera/simulator/dslr.py index 9a7070b3b..ce435f035 100644 --- a/src/panoptes/pocs/camera/simulator/dslr.py +++ b/src/panoptes/pocs/camera/simulator/dslr.py @@ -12,7 +12,7 @@ from panoptes.pocs.camera import AbstractCamera from panoptes.utils.images import fits as fits_utils -from panoptes.utils import get_quantity_value +from panoptes.utils import get_quantity_value, CountdownTimer class Camera(AbstractCamera): @@ -64,6 +64,7 @@ def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, return readout_args def _readout(self, filename=None, header=None): + timer = CountdownTimer(duration=self.readout_time) self.logger.trace(f'Calling _readout for {self}') # Get example FITS file from test data directory file_path = os.path.join( @@ -78,12 +79,14 @@ def _readout(self, filename=None, header=None): fake_data = np.random.randint(low=975, high=1026, size=fake_data.shape, dtype=fake_data.dtype) - time.sleep(self.readout_time) self.logger.debug(f'Writing {filename=} for {self}') fits_utils.write_fits(fake_data, header, filename) - def _process_fits(self, file_path, info): - file_path = super()._process_fits(file_path, info) + # Sleep for the remainder of the readout time. + timer.sleep() + + def _process_fits(self, file_path, metadata): + file_path = super()._process_fits(file_path, metadata) self.logger.debug('Overriding mount coordinates for camera simulator') # TODO get the path as package data or something better. solved_path = os.path.join( diff --git a/src/panoptes/pocs/tests/test_camera.py b/src/panoptes/pocs/tests/test_camera.py index 479631cae..a2aceb70c 100644 --- a/src/panoptes/pocs/tests/test_camera.py +++ b/src/panoptes/pocs/tests/test_camera.py @@ -123,7 +123,7 @@ def test_create_cameras_from_config_no_autodetect(config_host, config_port): def test_create_cameras_from_config_autodetect(config_host, config_port): - set_config('cameras.auto_detect', True) + set_config('cameras.defaults.auto_detect', True) with pytest.raises(error.CameraNotFound): create_cameras_from_config() reset_conf(config_host, config_port) @@ -485,13 +485,29 @@ def test_observation(camera, images_dir): field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') observation = Observation(field, exptime=1.5 * u.second) observation.seq_time = '19991231T235959' - camera.take_observation(observation, headers={}) - time.sleep(7) + observation_event = camera.take_observation(observation) + while not observation_event.is_set(): + camera.logger.trace(f'Waiting for observation event from inside test.') + time.sleep(1) observation_pattern = os.path.join(images_dir, 'TestObservation', camera.uid, observation.seq_time, '*.fits*') assert len(glob.glob(observation_pattern)) == 1 - for fn in glob.glob(observation_pattern): - os.remove(fn) + + +def test_observation_headers_and_blocking(camera, images_dir): + """ + Tests functionality of take_observation() + """ + field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') + observation = Observation(field, exptime=1.5 * u.second) + observation.seq_time = '19991231T235559' + camera.take_observation(observation, headers={'field_name': 'TESTVALUE'}, blocking=True) + observation_pattern = os.path.join(images_dir, 'TestObservation', + camera.uid, observation.seq_time, '*.fits*') + image_files = glob.glob(observation_pattern) + assert len(image_files) == 1 + headers = fits_utils.getheader(image_files[0]) + assert fits_utils.getval(image_files[0], 'FIELD') == 'TESTVALUE' def test_observation_nofilter(camera, images_dir): @@ -500,14 +516,11 @@ def test_observation_nofilter(camera, images_dir): """ field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') observation = Observation(field, exptime=1.5 * u.second, filter_name=None) - observation.seq_time = '19991231T235959' - camera.take_observation(observation, headers={}) - time.sleep(7) + observation.seq_time = '19991231T235159' + camera.take_observation(observation, blocking=True) observation_pattern = os.path.join(images_dir, 'TestObservation', camera.uid, observation.seq_time, '*.fits*') assert len(glob.glob(observation_pattern)) == 1 - for fn in glob.glob(observation_pattern): - os.remove(fn) def test_autofocus_coarse(camera, patterns, counter): diff --git a/tests/testing.yaml b/tests/testing.yaml index 922b5a6bc..0c39bccd9 100644 --- a/tests/testing.yaml +++ b/tests/testing.yaml @@ -59,9 +59,16 @@ pointing: exptime: 30 # seconds max_iterations: 3 cameras: - auto_detect: False defaults: - readout_time: 0.5 + primary: None + auto_detect: False + file_extension: fits + compress_fits: True + make_pretty_images: True + keep_jpgs: False + readout_time: 0.5 # seconds + timeout: 10 # seconds + filter_type: RGGB cooling: enabled: True temperature: @@ -72,8 +79,8 @@ cameras: timeout: 300 # seconds focuser: enabled: True - autofocus_seconds: 0.1 - autofocus_size: 500 + autofocus_seconds: 0.1 # seconds + autofocus_size: 500 # seconds autofocus_keep_files: False devices: - model: panoptes.pocs.camera.simulator.dslr.Camera @@ -95,8 +102,8 @@ cameras: model: panoptes.pocs.focuser.simulator.Focuser focus_port: /dev/fake/focuser.00 initial_position: 20000 - autofocus_range: [40, 80] - autofocus_step: [10, 20] + autofocus_range: [ 40, 80 ] + autofocus_step: [ 10, 20 ] autofocus_seconds: 0.1 autofocus_size: 500 autofocus_keep_files: False @@ -134,8 +141,8 @@ cameras: model: panoptes.pocs.focuser.simulator.Focuser focus_port: /dev/fake/focuser.00 initial_position: 20000 - autofocus_range: [40, 80] - autofocus_step: [10, 20] + autofocus_range: [ 40, 80 ] + autofocus_step: [ 10, 20 ] autofocus_seconds: 0.1 autofocus_size: 500 autofocus_keep_files: False @@ -168,7 +175,7 @@ cameras: ################################################################################ observations: make_timelapse: True - keep_jpgs: True + record_observations: True ######################## Google Network ######################################## # By default all images are stored on googlecloud servers and we also