diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4af28a993..95dba9133 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: python -m pip install --upgrade .[test] - name: Run unit tests run: | - pytest + python -m pytest test-coverage: name: Calculate and upload test coverage diff --git a/setup.cfg b/setup.cfg index 6660e83eb..067a87fc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,11 +32,11 @@ project_urls = packages = find: install_requires = colorlog - dspeed>=1.1 + dspeed@git+https://github.com/legend-exp/dspeed@main h5py>=3.2 iminuit - legend-daq2lh5>=1.1.0 - legend-pydataobj>=1.3 + legend-daq2lh5@git+https://github.com/legend-exp/legend-daq2lh5@main + legend-pydataobj>=1.5.0a1 matplotlib numba!=0.53.*,!=0.54.*,!=0.57 numpy>=1.21 diff --git a/src/pygama/cli.py b/src/pygama/cli.py index a6b59abaf..fb05ef658 100644 --- a/src/pygama/cli.py +++ b/src/pygama/cli.py @@ -80,7 +80,7 @@ def pygama_cli(): def add_lh5ls_parser(subparsers): - """Configure :func:`.lgdo.lh5_store.show` command line interface.""" + """Configure :func:`.lgdo.lh5.show` command line interface.""" parser_lh5ls = subparsers.add_parser( "lh5ls", description="""Inspect LEGEND HDF5 (LH5) file contents""" @@ -99,7 +99,7 @@ def add_lh5ls_parser(subparsers): def lh5_show_cli(args): - """Passes command line arguments to :func:`.lgdo.lh5_store.show`.""" + """Passes command line arguments to :func:`.lgdo.lh5.show`.""" show(args.lh5_file, args.lh5_group, attrs=args.attributes) diff --git a/src/pygama/evt/build_tcm.py b/src/pygama/evt/build_tcm.py index 7bb0bbef3..e821ee50f 100644 --- a/src/pygama/evt/build_tcm.py +++ b/src/pygama/evt/build_tcm.py @@ -49,7 +49,7 @@ def build_tcm( out_name name for the TCM table in the output file. wo_mode - mode to send to :meth:`~.lgdo.lh5_store.LH5Store.write_object`. + mode to send to :meth:`~.lgdo.lh5.LH5Store.write`. See Also -------- @@ -79,7 +79,7 @@ def build_tcm( else: array_id = len(all_tables) - 1 table = table + "/" + coin_col - coin_data.append(store.read_object(table, filename)[0].nda) + coin_data.append(store.read(table, filename)[0].nda) array_ids.append(array_id) tcm_cols = ptcm.generate_tcm_cols( @@ -94,6 +94,6 @@ def build_tcm( ) if out_file is not None: - store.write_object(tcm, out_name, out_file, wo_mode=wo_mode) + store.write(tcm, out_name, out_file, wo_mode=wo_mode) return tcm diff --git a/src/pygama/flow/data_loader.py b/src/pygama/flow/data_loader.py index 625b52fbc..a171ddfd3 100644 --- a/src/pygama/flow/data_loader.py +++ b/src/pygama/flow/data_loader.py @@ -14,7 +14,9 @@ import numpy as np import pandas as pd from dspeed.vis import WaveformBrowser -from lgdo import Array, LH5Iterator, LH5Store, Struct, Table, lgdo_utils +from lgdo.lh5 import LH5Iterator, LH5Store +from lgdo.lh5.utils import expand_vars +from lgdo.types import Array, Struct, Table from lgdo.types.vectorofvectors import build_cl, explode_arrays, explode_cl from tqdm.auto import tqdm @@ -193,9 +195,7 @@ def set_config(self, config: dict | str) -> None: # look for info in configuration if FileDB is not set if self.filedb is None: # expand $_ variables - value = lgdo_utils.expand_vars( - config["filedb"], substitute={"_": config_dir} - ) + value = expand_vars(config["filedb"], substitute={"_": config_dir}) self.filedb = FileDB(value) if not os.path.isdir(self.filedb.data_dir): @@ -584,7 +584,7 @@ def build_entry_list( tcm_table_name = self.filedb.get_table_name(tcm_tier, tcm_tb) try: - tcm_lgdo, _ = sto.read_object(tcm_table_name, tcm_path) + tcm_lgdo, _ = sto.read(tcm_table_name, tcm_path) except KeyError: log.warning(f"Cannot find table {tcm_table_name} in file {tcm_path}") continue @@ -649,7 +649,7 @@ def build_entry_list( if tb in col_tiers[file]["tables"][tier]: table_name = self.filedb.get_table_name(tier, tb) try: - tier_table, _ = sto.read_object( + tier_table, _ = sto.read( table_name, tier_path, field_mask=cut_cols[level], @@ -708,11 +708,9 @@ def build_entry_list( f_dict = f_entries.to_dict("list") f_struct = Struct(f_dict) if self.merge_files: - sto.write_object(f_struct, "entries", output_file, wo_mode="a") + sto.write(f_struct, "entries", output_file, wo_mode="a") else: - sto.write_object( - f_struct, f"entries/{file}", output_file, wo_mode="a" - ) + sto.write(f_struct, f"entries/{file}", output_file, wo_mode="a") if log.getEffectiveLevel() >= logging.INFO: progress_bar.close() @@ -862,7 +860,7 @@ def build_hit_entries( # load the data from the tier file, just the columns needed for the cut table_name = self.filedb.get_table_name(tier, tb) try: - tier_tb, _ = sto.read_object( + tier_tb, _ = sto.read( table_name, tier_path, field_mask=cut_cols ) except KeyError: @@ -902,11 +900,9 @@ def build_hit_entries( f_dict = f_entries.to_dict("list") f_struct = Struct(f_dict) if self.merge_files: - sto.write_object(f_struct, "entries", output_file, wo_mode="a") + sto.write(f_struct, "entries", output_file, wo_mode="a") else: - sto.write_object( - f_struct, f"entries/{file}", output_file, wo_mode="a" - ) + sto.write(f_struct, f"entries/{file}", output_file, wo_mode="a") if log.getEffectiveLevel() >= logging.INFO: progress_bar.close() @@ -1117,7 +1113,7 @@ def explode_evt_cols(el: pd.DataFrame, tier_table: Table): for file in files ] - tier_table, _ = sto.read_object( + tier_table, _ = sto.read( name=tb_name, lh5_file=tier_paths, idx=idx_mask, @@ -1143,7 +1139,7 @@ def explode_evt_cols(el: pd.DataFrame, tier_table: Table): f_table = utils.dict_to_table(col_dict=col_dict, attr_dict=attr_dict) if output_file: - sto.write_object(f_table, "merged_data", output_file, wo_mode="o") + sto.write(f_table, "merged_data", output_file, wo_mode="o") if in_memory: if self.output_format == "lgdo.Table": return f_table @@ -1220,7 +1216,7 @@ def explode_evt_cols(el: pd.DataFrame, tier_table: Table): raise FileNotFoundError(tier_path) table_name = self.filedb.get_table_name(tier, tb) - tier_table, _ = sto.read_object( + tier_table, _ = sto.read( table_name, tier_path, idx=idx_mask, @@ -1246,7 +1242,7 @@ def explode_evt_cols(el: pd.DataFrame, tier_table: Table): if in_memory: load_out.add_field(name=file, obj=f_table) if output_file: - sto.write_object(f_table, f"{file}", output_file, wo_mode="o") + sto.write(f_table, f"{file}", output_file, wo_mode="o") # end file loop if log.getEffectiveLevel() >= logging.INFO: @@ -1318,7 +1314,7 @@ def load_evts( ) if os.path.exists(tier_path): table_name = self.filedb.get_table_name(tier, tb) - tier_table, _ = sto.read_object( + tier_table, _ = sto.read( table_name, tier_path, idx=idx_mask, @@ -1332,7 +1328,7 @@ def load_evts( if in_memory: load_out[file] = f_table if output_file: - sto.write_object(f_table, f"file{file}", output_file, wo_mode="o") + sto.write(f_table, f"file{file}", output_file, wo_mode="o") # end file loop if in_memory: diff --git a/src/pygama/flow/file_db.py b/src/pygama/flow/file_db.py index c64e6b786..4047f8c97 100644 --- a/src/pygama/flow/file_db.py +++ b/src/pygama/flow/file_db.py @@ -9,11 +9,11 @@ import warnings import h5py -import lgdo import numpy as np import pandas as pd -from lgdo import Array, Scalar, VectorOfVectors -from lgdo import lh5_store as lh5 +from lgdo.lh5.store import LH5Store, ls +from lgdo.lh5.utils import expand_path, expand_vars +from lgdo.types import Array, Scalar, VectorOfVectors from parse import parse from . import utils @@ -185,14 +185,12 @@ def set_config(self, config: dict, config_path: str = None) -> None: if config_path is not None: subst_vars["_"] = os.path.dirname(str(config_path)) - data_dir = lgdo.lgdo_utils.expand_path( - self.config["data_dir"], substitute=subst_vars - ) + data_dir = expand_path(self.config["data_dir"], substitute=subst_vars) self.data_dir = data_dir tier_dirs = self.config["tier_dirs"] for k, val in tier_dirs.items(): - tier_dirs[k] = lgdo.lgdo_utils.expand_vars(val, substitute=subst_vars) + tier_dirs[k] = expand_vars(val, substitute=subst_vars) self.tier_dirs = tier_dirs def scan_files(self, dirs: list[str] = None) -> None: @@ -407,7 +405,7 @@ def update_tables_cols(row, tier: str, utc_cache: dict = None) -> pd.Series: ) # TODO this call here is really expensive! - groups = lh5.ls(f, wildcard) + groups = ls(f, wildcard) if len(groups) > 0 and parse(template, groups[0]) is None: log.warning(f"groups in {fpath} don't match template") else: @@ -431,7 +429,7 @@ def update_tables_cols(row, tier: str, utc_cache: dict = None) -> pd.Series: table_name = template try: - col = lh5.ls(f[table_name]) + col = ls(f[table_name]) except KeyError: log.warning(f"cannot find '{table_name}' in {fpath}") continue @@ -477,8 +475,8 @@ def update_tables_cols(row, tier: str, utc_cache: dict = None) -> pd.Series: columns_vov = VectorOfVectors( flattened_data=flattened, cumulative_length=length ) - sto = lh5.LH5Store() - sto.write_object(columns_vov, "unique_columns", to_file) + sto = LH5Store() + sto.write(columns_vov, "unique_columns", to_file) return self.columns @@ -501,12 +499,12 @@ def from_disk(self, path: str | list[str]) -> None: # expand wildcards paths = [] for p in path: - paths += lgdo.lgdo_utils.expand_path(p, list=True) + paths += expand_path(p, list=True) if not paths: raise FileNotFoundError(path) - sto = lh5.LH5Store() + sto = LH5Store() # objects/accumulators that will be used to configure the FileDB at the end _cfg = None _df = None @@ -528,7 +526,7 @@ def _replace_idx(row, trans, tier): # loop over the files for p in paths: - cfg, _ = sto.read_object("config", p) + cfg, _ = sto.read("config", p) cfg = json.loads(cfg.value.decode()) # make sure configurations are all the same @@ -540,7 +538,7 @@ def _replace_idx(row, trans, tier): ) # read in unique columns - vov, _ = sto.read_object("columns", p) + vov, _ = sto.read("columns", p) # Convert back from VoV of UTF-8 bytestrings to a list of lists of strings columns = [[v.decode("utf-8") for v in ov] for ov in list(vov)] @@ -599,14 +597,12 @@ def to_disk(self, filename: str, wo_mode="write_safe") -> None: filename output LH5 file name. wo_mode - passed to :meth:`~.lgdo.lh5_store.write_object`. + passed to :meth:`~.lgdo.lh5.write`. """ log.debug(f"writing database to {filename}") - sto = lh5.LH5Store() - sto.write_object( - Scalar(json.dumps(self.config)), "config", filename, wo_mode=wo_mode - ) + sto = LH5Store() + sto.write(Scalar(json.dumps(self.config)), "config", filename, wo_mode=wo_mode) if wo_mode in ["write_safe", "w", "overwrite_file", "of"]: wo_mode = "a" @@ -623,7 +619,7 @@ def to_disk(self, filename: str, wo_mode="write_safe") -> None: flattened_data=Array(nda=np.array(flat).astype("S")), cumulative_length=Array(nda=np.array(cum_l)), ) - sto.write_object(col_vov, "columns", filename, wo_mode=wo_mode) + sto.write(col_vov, "columns", filename, wo_mode=wo_mode) # FIXME: to_hdf() throws this: # diff --git a/src/pygama/hit/build_hit.py b/src/pygama/hit/build_hit.py index 5cc34202f..3f073380f 100644 --- a/src/pygama/hit/build_hit.py +++ b/src/pygama/hit/build_hit.py @@ -69,7 +69,7 @@ def build_hit( n_max maximum number of rows to process wo_mode - forwarded to :meth:`~.lgdo.lh5_store.write_object`. + forwarded to :meth:`~.lgdo.lh5.write`. """ store = LH5Store() @@ -168,7 +168,7 @@ def build_hit( if col not in cfg["outputs"]: outtbl_obj.remove_column(col, delete=True) - store.write_object( + store.write( obj=outtbl_obj, name=tbl.replace("/dsp", "/hit"), lh5_file=outfile, diff --git a/src/pygama/pargen/AoE_cal.py b/src/pygama/pargen/AoE_cal.py index bbe0ee171..227aec4e2 100644 --- a/src/pygama/pargen/AoE_cal.py +++ b/src/pygama/pargen/AoE_cal.py @@ -15,7 +15,7 @@ import matplotlib as mpl mpl.use("agg") -import lgdo.lh5_store as lh5 +import lgdo.lh5 as lh5 import matplotlib.cm as cmx import matplotlib.colors as mcolors import matplotlib.dates as mdates diff --git a/src/pygama/pargen/cuts.py b/src/pygama/pargen/cuts.py index e6c6b571e..c4ab158d7 100644 --- a/src/pygama/pargen/cuts.py +++ b/src/pygama/pargen/cuts.py @@ -9,7 +9,7 @@ import logging import os -import lgdo.lh5_store as lh5 +import lgdo.lh5 as lh5 import numpy as np import pandas as pd from scipy import stats diff --git a/src/pygama/pargen/ecal_th.py b/src/pygama/pargen/ecal_th.py index bd957c3d4..e3526c63a 100644 --- a/src/pygama/pargen/ecal_th.py +++ b/src/pygama/pargen/ecal_th.py @@ -15,7 +15,7 @@ from scipy.stats import binned_statistic mpl.use("agg") -import lgdo.lh5_store as lh5 +import lgdo.lh5 as lh5 import matplotlib.pyplot as plt import numpy as np import pandas as pd diff --git a/src/pygama/pargen/energy_optimisation.py b/src/pygama/pargen/energy_optimisation.py index 24703fda2..1c34901d9 100644 --- a/src/pygama/pargen/energy_optimisation.py +++ b/src/pygama/pargen/energy_optimisation.py @@ -13,7 +13,7 @@ import sys from collections import namedtuple -import lgdo.lh5_store as lh5 +import lgdo.lh5 as lh5 import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np @@ -68,8 +68,8 @@ def run_optimisation( Number of events to run over """ grid = set_par_space(opt_config) - waveforms = sto.read_object(f"/raw/{wf_field}", file, idx=cuts, n_rows=n_events)[0] - baseline = sto.read_object("/raw/baseline", file, idx=cuts, n_rows=n_events)[0] + waveforms = sto.read(f"/raw/{wf_field}", file, idx=cuts, n_rows=n_events)[0] + baseline = sto.read("/raw/baseline", file, idx=cuts, n_rows=n_events)[0] tb_data = lh5.Table(col_dict={f"{wf_field}": waveforms, "baseline": baseline}) return opt.run_grid(tb_data, dsp_config, grid, fom, db_dict, **fom_kwargs) @@ -138,12 +138,8 @@ def form_dict(in_dict, length): fom_kwargs = fom_kwargs["fom_kwargs"] fom_kwargs = form_dict(fom_kwargs, len(grid)) sto = lh5.LH5Store() - waveforms = sto.read_object( - f"{lh5_path}/{wf_field}", file, idx=cuts, n_rows=n_events - )[0] - baseline = sto.read_object(f"{lh5_path}/baseline", file, idx=cuts, n_rows=n_events)[ - 0 - ] + waveforms = sto.read(f"{lh5_path}/{wf_field}", file, idx=cuts, n_rows=n_events)[0] + baseline = sto.read(f"{lh5_path}/baseline", file, idx=cuts, n_rows=n_events)[0] tb_data = lh5.Table(col_dict={f"{wf_field}": waveforms, "baseline": baseline}) return opt.run_grid_multiprocess_parallel( tb_data, @@ -999,9 +995,7 @@ def event_selection( idx_list = get_wf_indexes(sort_index, idx_list_lens) idxs = np.array(sorted(np.concatenate(masks))) - input_data = sto.read_object(f"{lh5_path}", raw_files, idx=idxs, n_rows=len(idxs))[ - 0 - ] + input_data = sto.read(f"{lh5_path}", raw_files, idx=idxs, n_rows=len(idxs))[0] if isinstance(dsp_config, str): with open(dsp_config) as r: diff --git a/src/pygama/pargen/extract_tau.py b/src/pygama/pargen/extract_tau.py index 72e357fd7..d35473715 100644 --- a/src/pygama/pargen/extract_tau.py +++ b/src/pygama/pargen/extract_tau.py @@ -15,7 +15,7 @@ mpl.use("agg") import lgdo -import lgdo.lh5_store as lh5 +import lgdo.lh5 as lh5 import matplotlib.pyplot as plt import numpy as np @@ -61,12 +61,10 @@ def load_data( cuts = np.where((df.daqenergy.values > threshold) & (~ids))[0] - waveforms = sto.read_object( - f"{lh5_path}/{wf_field}", raw_file, idx=cuts, n_rows=n_events - )[0] - baseline = sto.read_object( - f"{lh5_path}/baseline", raw_file, idx=cuts, n_rows=n_events - )[0] + waveforms = sto.read(f"{lh5_path}/{wf_field}", raw_file, idx=cuts, n_rows=n_events)[ + 0 + ] + baseline = sto.read(f"{lh5_path}/baseline", raw_file, idx=cuts, n_rows=n_events)[0] tb_data = lh5.Table(col_dict={f"{wf_field}": waveforms, "baseline": baseline}) return tb_data diff --git a/src/pygama/pargen/utils.py b/src/pygama/pargen/utils.py index 27f4af9ae..e58785e4e 100644 --- a/src/pygama/pargen/utils.py +++ b/src/pygama/pargen/utils.py @@ -69,7 +69,7 @@ def load_data( all_files = [] masks = np.array([], dtype=bool) for tstamp, tfiles in files.items(): - table = sto.read_object(lh5_path, tfiles)[0] + table = sto.read(lh5_path, tfiles)[0] if tstamp in cal_dict: file_df = table.eval(cal_dict[tstamp]).get_dataframe() else: @@ -95,7 +95,7 @@ def load_data( keys = [key.split("/")[-1] for key in keys] params = get_params(keys + list(cal_dict.keys()), params) - table = sto.read_object(lh5_path, files)[0] + table = sto.read(lh5_path, files)[0] df = table.eval(cal_dict).get_dataframe() for param in params: if param not in df: diff --git a/tests/configs/icpc-dsp-config.json b/tests/configs/icpc-dsp-config.json index 8536a31ec..28af29239 100644 --- a/tests/configs/icpc-dsp-config.json +++ b/tests/configs/icpc-dsp-config.json @@ -38,19 +38,19 @@ "processors": { "tp_min, tp_max, wf_min, wf_max": { "function": "min_max", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["waveform", "tp_min", "tp_max", "wf_min", "wf_max"], "unit": ["ns", "ns", "ADC", "ADC"] }, "wf_blsub": { "function": "bl_subtract", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["waveform", "baseline", "wf_blsub"], "unit": "ADC" }, "bl_mean , bl_std, bl_slope, bl_intercept": { "function": "linear_slope_fit", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": [ "wf_blsub[0:750]", "bl_mean", @@ -62,51 +62,65 @@ }, "wf_pz": { "function": "pole_zero", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_blsub", "db.pz.tau", "wf_pz"], "unit": "ADC", "defaults": { "db.pz.tau": "27460.5" } }, "pz_mean , pz_std, pz_slope, pz_intercept": { "function": "linear_slope_fit", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz[1500:]", "pz_mean", "pz_std", "pz_slope", "pz_intercept"], "unit": ["ADC", "ADC", "ADC", "ADC"] }, - "wf_t0_filter": { + "t0_kernel": { "function": "t0_filter", - "module": "pygama.dsp.processors", - "args": ["wf_pz", "wf_t0_filter(len(wf_pz), 'f', grid=wf_pz.grid)"], - "init_args": ["128*ns/wf_pz.period", "2*us/wf_pz.period"], + "module": "dspeed.processors", + "args": [ + "128*ns/wf_pz.period", + "2*us/wf_pz.period", + "t0_kernel(round((128*ns+2*us)/wf_pz.period), 'f')" + ], + "unit": "ADC" + }, + "wf_t0_filter": { + "function": "convolve_wf", + "module": "dspeed.processors", + "args": [ + "wf_pz", + "t0_kernel", + "'s'", + "wf_t0_filter(len(wf_pz), 'f', grid=wf_pz.grid)" + ], "unit": "ADC" }, "wf_atrap": { "function": "asym_trap_filter", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "128*ns", "4", "2*us", "wf_atrap"], "unit": "ADC" }, "conv_tmin ,tp_start, conv_min, conv_max": { "function": "min_max", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_t0_filter", "conv_tmin", "tp_start", "conv_min", "conv_max"], "unit": ["ns", "ns", "ADC", "ADC"] }, "tp_0_atrap": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_atrap", "bl_std", "tp_start", 0, "tp_0_atrap"], "unit": "ns" }, "tp_0_est": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_t0_filter", "bl_std", "tp_start", 0, "tp_0_est(unit=ns)"], "unit": "ns" }, "wf_trap": { "function": "trap_norm", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "db.ttrap.rise", "db.ttrap.flat", "wf_trap"], "unit": "ADC", "defaults": { "db.ttrap.rise": "10*us", "db.ttrap.flat": "3.008*us" } @@ -120,7 +134,7 @@ }, "wf_etrap": { "function": "trap_norm", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "db.etrap.rise", "db.etrap.flat", "wf_etrap"], "unit": "ADC", "defaults": { "db.etrap.rise": "10*us", "db.etrap.flat": "3.008*us" } @@ -134,10 +148,10 @@ }, "trapEftp": { "function": "fixed_time_pickoff", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": [ "wf_etrap", - "tp_0_est+db.etrap.rise+db.etrap.flat*db.etrap.sample", + "round(tp_0_est+db.etrap.rise+db.etrap.flat*db.etrap.sample, wf_etrap.grid)", "'l'", "trapEftp" ], @@ -148,23 +162,33 @@ "db.etrap.sample": "0.8" } }, - "wf_cusp": { + "cusp_kernel": { "function": "cusp_filter", - "module": "pygama.dsp.processors", - "args": ["wf_blsub", "wf_cusp(101, 'f')"], - "init_args": [ - "len(wf_blsub)-100", + "module": "dspeed.processors", + "args": [ "db.cusp.sigma/wf_blsub.period", "round(db.cusp.flat/wf_blsub.period)", - "db.pz.tau" + "db.pz.tau/wf_blsub.period", + "cusp_kernel(round(len(wf_blsub)-(33.6*us/wf_blsub.period)-(4.8*us/wf_blsub.period)), 'f')" ], "defaults": { "db.cusp.sigma": "20*us", "db.cusp.flat": "3*us", - "db.pz.tau": "27460.5" + "db.pz.tau": "450*us" }, "unit": "ADC" }, + "wf_cusp": { + "function": "fft_convolve_wf", + "module": "dspeed.processors", + "args": [ + "wf_blsub[:round(len(wf_blsub)-(33.6*us/wf_blsub.period))]", + "cusp_kernel", + "'v'", + "wf_cusp(round((4.8*us/wf_blsub.period)+1), 'f')" + ], + "unit": "ADC" + }, "cuspEmax": { "function": "amax", "module": "numpy", @@ -174,28 +198,38 @@ }, "cuspEftp": { "function": "fixed_time_pickoff", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_cusp", "db.cusp.sample", "'i'", "cuspEftp"], "unit": "ADC", "defaults": { "db.cusp.sample": "50" } }, - "wf_zac": { + "zac_kernel": { "function": "zac_filter", - "module": "pygama.dsp.processors", - "args": ["wf_blsub", "wf_zac(101, 'f')"], - "init_args": [ - "len(wf_blsub)-100", + "module": "dspeed.processors", + "args": [ "db.zac.sigma/wf_blsub.period", "round(db.zac.flat/wf_blsub.period)", - "db.pz.tau" + "db.pz.tau/wf_blsub.period", + "zac_kernel(round(len(wf_blsub)-(33.6*us/wf_blsub.period)-(4.8*us/wf_blsub.period)), 'f')" ], "defaults": { "db.zac.sigma": "20*us", "db.zac.flat": "3*us", - "db.pz.tau": "27460.5" + "db.pz.tau": "450*us" }, "unit": "ADC" }, + "wf_zac": { + "function": "fft_convolve_wf", + "module": "dspeed.processors", + "args": [ + "wf_blsub[:round(len(wf_blsub)-(33.6*us/wf_blsub.period))]", + "zac_kernel", + "'v'", + "wf_zac(round((4.8*us/wf_blsub.period)+1), 'f')" + ], + "unit": "ADC" + }, "zacEmax": { "function": "amax", "module": "numpy", @@ -205,74 +239,74 @@ }, "zacEftp": { "function": "fixed_time_pickoff", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_zac", "db.zac.sample", "'i'", "zacEftp"], "defaults": { "db.zac.sample": "50" }, "unit": "ADC" }, "tp_100": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax", "tp_0_est", 1, "tp_100"], "unit": "ns" }, "tp_99": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "0.99*trapTmax", "tp_0_est", 1, "tp_99"], "unit": "ns" }, "tp_95": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.95", "tp_99", 0, "tp_95"], "unit": "ns" }, "tp_90": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.9", "tp_95", 0, "tp_90"], "unit": "ns" }, "tp_80": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.8", "tp_90", 0, "tp_80"], "unit": "ns" }, "tp_50": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.5", "tp_80", 0, "tp_50"], "unit": "ns" }, "tp_20": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.2", "tp_50", 0, "tp_20"], "unit": "ns" }, "tp_10": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.1", "tp_20", 0, "tp_10"], "unit": "ns" }, "tp_01": { "function": "time_point_thresh", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "trapTmax*0.01", "tp_10", 0, "tp_01"], "unit": "ns" }, "wf_trap2": { "function": "trap_norm", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "4*us", "96*ns", "wf_trap2"], "unit": "ADC" }, "trapQftp": { "function": "fixed_time_pickoff", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_trap2", "tp_0_est + 8.096*us", "'l'", "trapQftp"], "unit": "ADC" }, @@ -290,31 +324,31 @@ }, "wf_le": { "function": "windower", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_pz", "tp_0_est", "wf_le(301, 'f')"], "unit": "ADC" }, "curr": { "function": "avg_current", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["wf_le", 1, "curr(len(wf_le)-1, 'f')"], "unit": "ADC/sample" }, "curr_up": { "function": "upsampler", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["curr", "16", "curr_up(4784, 'f')"], "unit": "ADC/sample" }, "curr_av": { "function": "moving_window_multi", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["curr_up", "48", 3, 0, "curr_av"], "unit": "ADC/sample" }, "aoe_t_min, tp_aoe_max, A_min, A_max": { "function": "min_max", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["curr_av", "aoe_t_min", "tp_aoe_max", "A_min", "A_max"], "unit": ["ns", "ns", "ADC/sample", "ADC/sample"] }, diff --git a/tests/configs/sipm-dplms-config.json b/tests/configs/sipm-dplms-config.json index cc7919e33..dd69bac0f 100644 --- a/tests/configs/sipm-dplms-config.json +++ b/tests/configs/sipm-dplms-config.json @@ -10,26 +10,26 @@ "processors": { "wf_gaus": { "function": "gaussian_filter1d", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.gaussian_filter1d", "args": ["waveform", "wf_gaus(len(waveform))"], "init_args": ["1", "4.0"], "unit": "ADC" }, "curr": { "function": "avg_current", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.moving_windows", "args": ["wf_gaus", 5, "curr(len(wf_gaus)-5)"], "unit": "ADC" }, "hist_weights , hist_borders": { "function": "histogram", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": ["curr", "hist_weights(100)", "hist_borders(101)"], "unit": ["none", "ADC"] }, "fwhm, idx_out_c, max_out": { "function": "histogram_stats", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": [ "hist_weights", "hist_borders", @@ -42,7 +42,7 @@ }, "vt_max_candidate_out, vt_min_out, n_max_out, n_min_out": { "function": "get_multi_local_extrema", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.get_multi_local_extrema", "args": [ "curr", 5, @@ -59,7 +59,7 @@ }, "trigger_pos, no_out": { "function": "peak_snr_threshold", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.peak_snr_threshold", "args": [ "curr", "vt_max_candidate_out", @@ -72,13 +72,13 @@ }, "energies": { "function": "multi_a_filter", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.multi_a_filter", "args": ["curr", "trigger_pos", "energies"], "unit": ["ADC"] }, "bl_mean , bl_std, bl_slope, bl_intercept": { "function": "linear_slope_fit", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": [ "waveform[:50]", "bl_mean", @@ -90,34 +90,45 @@ }, "wf_diff": { "function": "avg_current", - "module": "pygama.dsp.processors", + "module": "dspeed.processors", "args": ["waveform", 1, "wf_diff(len(waveform)-1)"], "unit": "ADC" }, - "wf_dplms": { + "dplms_kernel": { "function": "dplms_filter", - "module": "pygama.dsp.processors", - "args": ["wf_diff", "wf_dplms(len(wf_diff)-49, 'f')"], - "unit": "ADC", - "init_args": [ + "module": "dspeed.processors", + "args": [ "db.dplms.noise_matrix", "db.dplms.reference", - "50", "0.01", "1", "0", - "0" - ] + "0", + "dplms_kernel(50, 'f')" + ], + "unit": "ADC" + }, + "wf_dplms": { + "description": "convolve optimised cusp filter", + "function": "convolve_wf", + "module": "dspeed.processors", + "args": [ + "wf_diff", + "dplms_kernel", + "'s'", + "wf_dplms(len(wf_diff)-49, 'f')" + ], + "unit": "ADC" }, "h_weights , h_borders": { "function": "histogram", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": ["wf_dplms", "h_weights(100)", "h_borders(101)"], "unit": ["none", "ADC"] }, "fwhm_d, idx_out_d, max_out_d": { "function": "histogram_stats", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": [ "h_weights", "h_borders", @@ -130,7 +141,7 @@ }, "vt_max_candidate_out_d, vt_min_out_d, n_max_out_d, n_min_out_d": { "function": "get_multi_local_extrema", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.get_multi_local_extrema", "args": [ "wf_dplms", 10, @@ -145,7 +156,7 @@ }, "trigger_pos_dplms, no_out_d": { "function": "peak_snr_threshold", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.peak_snr_threshold", "args": [ "wf_dplms", "vt_max_candidate_out_d", @@ -158,7 +169,7 @@ }, "energies_dplms": { "function": "multi_a_filter", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.multi_a_filter", "args": ["wf_dplms", "trigger_pos_dplms", "energies_dplms"], "unit": ["ADC"] } diff --git a/tests/configs/sipm-dsp-config.json b/tests/configs/sipm-dsp-config.json index 8de5bd2e6..bb7878a5d 100644 --- a/tests/configs/sipm-dsp-config.json +++ b/tests/configs/sipm-dsp-config.json @@ -3,26 +3,26 @@ "processors": { "wf_gaus": { "function": "gaussian_filter1d", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.gaussian_filter1d", "args": ["waveform", "wf_gaus(len(waveform))"], "init_args": ["1", "4.0"], "unit": "ADC" }, "curr": { "function": "avg_current", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.moving_windows", "args": ["wf_gaus", 5, "curr(len(wf_gaus)-5)"], "unit": "ADC" }, "hist_weights , hist_borders": { "function": "histogram", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": ["curr", "hist_weights(100)", "hist_borders(101)"], "unit": ["none", "ADC"] }, "fwhm, idx_out_c, max_out": { "function": "histogram_stats", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.histogram", "args": [ "hist_weights", "hist_borders", @@ -35,7 +35,7 @@ }, "vt_max_candidate_out, vt_min_out, n_max_out, n_min_out": { "function": "get_multi_local_extrema", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.get_multi_local_extrema", "args": [ "curr", 5, @@ -52,7 +52,7 @@ }, "trigger_pos, no_out": { "function": "peak_snr_threshold", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.peak_snr_threshold", "args": [ "curr", "vt_max_candidate_out", @@ -65,7 +65,7 @@ }, "energies": { "function": "multi_a_filter", - "module": "pygama.dsp.processors", + "module": "dspeed.processors.multi_a_filter", "args": ["curr", "trigger_pos", "energies"], "unit": ["ADC"] } diff --git a/tests/evt/test_build_tcm.py b/tests/evt/test_build_tcm.py index 505196825..49296e9fe 100644 --- a/tests/evt/test_build_tcm.py +++ b/tests/evt/test_build_tcm.py @@ -15,7 +15,7 @@ def test_generate_tcm_cols(lgnd_test_data): store = LH5Store() coin_data = [] for tbl in tables: - ts, _ = store.read_object(f"{tbl}/raw/timestamp", f_raw) + ts, _ = store.read(f"{tbl}/raw/timestamp", f_raw) coin_data.append(ts) tcm_cols = evt.generate_tcm_cols( @@ -67,6 +67,6 @@ def test_build_tcm(lgnd_test_data, tmptestdir): ) assert os.path.exists(out_file) store = LH5Store() - obj, n_rows = store.read_object("hardware_tcm", out_file) + obj, n_rows = store.read("hardware_tcm", out_file) assert isinstance(obj, lgdo.Struct) assert list(obj.keys()) == ["cumulative_length", "array_id", "array_idx"] diff --git a/tests/hit/test_build_hit.py b/tests/hit/test_build_hit.py index 924928da6..8b8d39090 100644 --- a/tests/hit/test_build_hit.py +++ b/tests/hit/test_build_hit.py @@ -1,7 +1,7 @@ import os from pathlib import Path -import lgdo.lh5_store as store +import lgdo.lh5 as store import numpy as np import pytest from lgdo import LH5Store, ls @@ -94,7 +94,7 @@ def test_outputs_specification(dsp_test_file, tmptestdir): ) store = LH5Store() - obj, _ = store.read_object("/geds/hit", outfile) + obj, _ = store.read("/geds/hit", outfile) assert list(obj.keys()) == ["calE", "AoE", "A_max"] @@ -109,7 +109,7 @@ def test_aggregation_outputs(dsp_test_file, tmptestdir): ) sto = LH5Store() - obj, _ = sto.read_object("/geds/hit", outfile) + obj, _ = sto.read("/geds/hit", outfile) assert list(obj.keys()) == [ "is_valid_rt", "is_valid_t0",