From 83b953bb1834b873d2e9dd1ce264fd9e1b02708e Mon Sep 17 00:00:00 2001 From: Philipp Rouast Date: Wed, 13 Nov 2024 17:17:20 +1100 Subject: [PATCH 1/5] Distinguish between batch and burst mode; Implement burst mode for simple methods; Add new live example script --- examples/live.py | 177 +++++++++++++++++++++++ tests/conftest.py | 2 +- tests/test_buffer.py | 111 ++++++++++++++ tests/test_signal.py | 50 ++++++- tests/test_simple_rppg_method.py | 13 +- tests/test_ssd.py | 4 +- tests/test_vitallens.py | 9 +- vitallens/__init__.py | 2 +- vitallens/buffer.py | 183 ++++++++++++++++++++++++ vitallens/client.py | 51 ++++--- vitallens/enums.py | 31 ++++ vitallens/methods/chrom.py | 7 +- vitallens/methods/g.py | 7 +- vitallens/methods/pos.py | 7 +- vitallens/methods/rppg_method.py | 10 +- vitallens/methods/simple_rppg_method.py | 66 +++++---- vitallens/methods/vitallens.py | 73 ++++------ vitallens/signal.py | 73 +++++++++- vitallens/ssd.py | 20 +-- 19 files changed, 762 insertions(+), 134 deletions(-) create mode 100644 examples/live.py create mode 100644 tests/test_buffer.py create mode 100644 vitallens/buffer.py create mode 100644 vitallens/enums.py diff --git a/examples/live.py b/examples/live.py new file mode 100644 index 0000000..bfd4bbb --- /dev/null +++ b/examples/live.py @@ -0,0 +1,177 @@ +import argparse +import concurrent.futures +import cv2 +import numpy as np +from prpy.numpy.face import get_upper_body_roi_from_det +from prpy.numpy.signal import estimate_freq +import sys +import threading +import time +import warnings + +sys.path.append('../vitallens-python') +from vitallens import VitalLens, Mode, Method +from vitallens.buffer import SignalBuffer, MultiSignalBuffer + +def draw_roi(frame, roi): + roi = np.asarray(roi).astype(np.int32) + frame = cv2.rectangle(frame, (roi[0], roi[1]), (roi[2], roi[3]), (0, 255, 0), 1) + +def draw_signal(frame, roi, sig, sig_name, sig_conf_name, draw_area_tl_x, draw_area_tl_y, color): + def _draw(frame, vals, display_height, display_width, min_val, max_val, color, thickness): + height_mult = display_height/(max_val - min_val) + width_mult = display_width/(vals.shape[0] - 1) + p1 = (int(draw_area_tl_x), int(draw_area_tl_y + (max_val - vals[0]) * height_mult)) + for i, s in zip(range(1, len(vals)), vals[1:]): + p2 = (int(draw_area_tl_x + i * width_mult), int(draw_area_tl_y + (max_val - s) * height_mult)) + frame = cv2.line(frame, p1, p2, color, thickness) + p1 = p2 + # Derive dims from roi + display_height = (roi[3] - roi[1]) / 2.0 + display_width = (roi[2] - roi[0]) * 0.8 + # Draw signal + if sig_name in sig: + vals = np.asarray(sig[sig_name]) + min_val = np.min(vals) + max_val = np.max(vals) + if max_val - min_val == 0: + return frame + _draw(frame=frame, vals=vals, display_height=display_height, display_width=display_width, + min_val=min_val, max_val=max_val, color=color, thickness=2) + # Draw confidence + if sig_conf_name in sig: + vals = np.asarray(sig[sig_conf_name]) + _draw(frame=frame, vals=vals, display_height=display_height, display_width=display_width, + min_val=0., max_val=1., color=color, thickness=1) + +def draw_fps(frame, fps, text, draw_area_bl_x, draw_area_bl_y): + cv2.putText(frame, text='{}: {:.1f}'.format(text, fps), org=(draw_area_bl_x, draw_area_bl_y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.6, color=(0,255,0), thickness=1) + +def draw_vital(frame, sig, text, sig_name, fps, mult, color, draw_area_bl_x, draw_area_bl_y): + if sig_name in sig: + val = estimate_freq(x=sig[sig_name], f_s=fps, f_res=0.0167, method='periodogram') * mult + cv2.putText(frame, text='{}: {:.1f}'.format(text, val), org=(draw_area_bl_x, draw_area_bl_y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.6, color=color, thickness=1) + +class VitalLensRunnable: + def __init__(self, method, api_key): + self.active = threading.Event() + self.result = [] + self.vl = VitalLens(method=method, + mode=Mode.BURST, + api_key=api_key, + detect_faces=True, + estimate_running_vitals=True, + export_to_json=False) + def __call__(self, inputs, fps): + self.active.set() + self.result = self.vl(np.asarray(inputs), fps=fps) + self.active.clear() + +def run(args): + """TODO""" + cap = cv2.VideoCapture(0) + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + vl = VitalLensRunnable(method=args.method, api_key=args.api_key) + signal_buffer = MultiSignalBuffer(size=120, ndim=1, ignore_k=['face']) + fps_buffer = SignalBuffer(size=120, ndim=1, pad_val=np.nan) + frame_buffer = [] + # Sample frames from cv2 video stream attempting to achieve this framerate + target_fps = 30. + # Check if the webcam is opened correctly + if not cap.isOpened(): + raise IOError("Cannot open webcam") + # Read first frame to get dims + _, frame = cap.read() + height, width, _ = frame.shape + roi = None + i = 0 + t, p_t = time.time(), time.time() + fps, p_fps = 30.0, 30.0 + ds_factor = 1 + n_frames = 0 + signals = None + while True: + ret, frame = cap.read() + # Measure frequency + t_prev = t + t = time.time() + if not vl.active.is_set(): + # Process result if available + if len(vl.result) > 0: + # Results are available - fetch and reset + result = vl.result[0] + vl.result = [] + # Update the buffer + signals = signal_buffer.update({ + **{ + f"{key}_sig": value['value'] if 'value' in value else np.array(value['data']) + for key, value in result['vital_signs'].items() + }, + **{ + f"{key}_conf": value['confidence'] if isinstance(value['confidence'], np.ndarray) else np.array(value['confidence']) + for key, value in result['vital_signs'].items() + }, + 'face_conf': result['face']['confidence'], + }, dt=n_frames) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + # Measure actual effective sampling frequency at which neural net input was sampled + fps = np.nanmean(fps_buffer.update([(1./(t - t_prev))/ds_factor], dt=n_frames)) + roi = get_upper_body_roi_from_det(result['face']['coordinates'][-1], clip_dims=(width, height), cropped=True) + # Measure prediction frequency - how often predictions are made + p_t_prev = p_t + p_t = time.time() + p_fps = 1./(p_t - p_t_prev) + else: + # No results available + roi = None + signal_buffer.clear() + # Start next prediction + if len(frame_buffer) > 0: + n_frames = len(frame_buffer) + # print("New frames for prediction: {} @ pred_fps={:.2f} / ds_factor={}".format(n_frames, p_fps, ds_factor)) + future = executor.submit(vl, frame_buffer.copy(), fps) + frame_buffer.clear() + # Sample frames + if i % ds_factor == 0: + # Add current frame to the buffer (BGR -> RGB) + frame_buffer.append(frame[...,::-1]) + i += 1 + # Display + if roi is not None: + draw_roi(frame, roi) + draw_signal( + frame=frame, roi=roi, sig=signals, sig_name='ppg_waveform_sig', sig_conf_name='ppg_waveform_conf', + draw_area_tl_x=roi[2]+20, draw_area_tl_y=roi[1], color=(0, 0, 255)) + draw_signal( + frame=frame, roi=roi, sig=signals, sig_name='respiratory_waveform_sig', sig_conf_name='respiratory_waveform_conf', + draw_area_tl_x=roi[2]+20, draw_area_tl_y=int(roi[1]+(roi[3]-roi[1])/2.0), color=(255, 0, 0)) + draw_fps(frame, fps=fps, text="fps", draw_area_bl_x=roi[0], draw_area_bl_y=roi[3]+20) + draw_fps(frame, fps=p_fps, text="p_fps", draw_area_bl_x=int(roi[0]+0.4*(roi[2]-roi[0])), draw_area_bl_y=roi[3]+20) + draw_vital(frame, sig=signals, text="hr [bpm]", sig_name='ppg_waveform_sig', fps=fps, mult=60., color=(0,0,255), draw_area_bl_x=roi[2]+20, draw_area_bl_y=int(roi[1]+(roi[3]-roi[1])/2.0)) + draw_vital(frame, sig=signals, text="rr [rpm]", sig_name='respiratory_waveform_sig', fps=fps, mult=60., color=(255,0,0), draw_area_bl_x=roi[2]+20, draw_area_bl_y=roi[3]) + cv2.imshow('Live', frame) + c = cv2.waitKey(1) + if c == 27: + break + # Even out fps + dt_req = 1./target_fps - (time.time() - t) + if dt_req > 0: time.sleep(dt_req) + + cap.release() + cv2.destroyAllWindows() + +def method_type(name): + try: + return Method[name] + except KeyError: + raise argparse.ArgumentTypeError(f"{name} is not a valid Method") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--api_key', type=str, default='', help='Your API key. Get one for free at https://www.rouast.com/api.') + parser.add_argument('--method', type=method_type, default='VITALLENS', help='Choice of method (VITALLENS, POS, CHROM, or G)') + args = parser.parse_args() + run(args) diff --git a/tests/conftest.py b/tests/conftest.py index 766da65..963d586 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,7 +62,7 @@ def test_video_faces(request): test_video_ndarray = request.getfixturevalue('test_video_ndarray') test_video_fps = request.getfixturevalue('test_video_fps') boxes, _ = det(test_video_ndarray, - inputs_shape=test_video_ndarray.shape, + n_frames=test_video_ndarray.shape[0], fps=test_video_fps) boxes = (boxes * [test_video_ndarray.shape[2], test_video_ndarray.shape[1], test_video_ndarray.shape[2], test_video_ndarray.shape[1]]).astype(int) return boxes[:,0].astype(np.int64) diff --git a/tests/test_buffer.py b/tests/test_buffer.py new file mode 100644 index 0000000..b31a3b1 --- /dev/null +++ b/tests/test_buffer.py @@ -0,0 +1,111 @@ +# Copyright (c) 2024 Rouast Labs +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import numpy as np +import pytest + +import sys +sys.path.append('../vitallens-python') + +from vitallens.buffer import SignalBuffer, MultiSignalBuffer + +@pytest.mark.parametrize("pad_val", [0, -1]) +def test_signal_buffer(pad_val): + # 1 dim + buffer = SignalBuffer(size=8, ndim=1, pad_val=pad_val) + with pytest.raises(Exception): + buffer.get() + np.testing.assert_allclose( + buffer.update(signal=[.2, .4,], dt=2), + np.asarray([pad_val, pad_val, pad_val, pad_val, pad_val, pad_val, .2, .4])) + np.testing.assert_allclose( + buffer.update(signal=[.1, .3, .5, .6], dt=2), + np.asarray([pad_val, pad_val, pad_val, pad_val, .15, .35, .5, .6])) + np.testing.assert_allclose( + buffer.update(signal=[.6, .7], dt=1), + np.asarray([pad_val, pad_val, pad_val, .15, .35, .5, .6, .7])) + np.testing.assert_allclose( + buffer.update(signal=[.8, .9, .8, .7, .6], dt=4), + np.asarray([.35, .5, .6, .75, .9, .8, .7, .6])) + # 2 dim + buffer = SignalBuffer(size=4, ndim=2, pad_val=pad_val) + with pytest.raises(Exception): + buffer.get() + np.testing.assert_allclose( + buffer.update(signal=[[.1, .2,]], dt=1), + np.asarray([[pad_val, pad_val], [pad_val, pad_val], [pad_val, pad_val], [.1, .2]])) + np.testing.assert_allclose( + buffer.update(signal=[[.1, .3], [.5, .6]], dt=3), + np.asarray([[.1, .2], [pad_val, pad_val], [.1, .3], [.5, .6]])) + np.testing.assert_allclose( + buffer.update(signal=[[.3, .2], [.5, .6], [.5, .6]], dt=2), + np.asarray([[.1, .3], [.4, .4], [.5, .6], [.5, .6]])) + +@pytest.mark.parametrize("pad_val", [0, -1]) +def test_multi_signal_buffer(pad_val): + # 1 dim, 2 signals + buffer = MultiSignalBuffer(size=8, ndim=1, ignore_k=[], pad_val=pad_val) + with pytest.raises(Exception): + buffer.get() + with pytest.raises(Exception): + buffer.update(signals=[0., 1.]) + out = buffer.update( + signals={"a": [.2, .4,], "b": [.1, .2]}, dt=1) + np.testing.assert_allclose( + out["a"], + np.asarray([pad_val, pad_val, pad_val, pad_val, pad_val, pad_val, .2, .4])) + np.testing.assert_allclose( + out["b"], + np.asarray([pad_val, pad_val, pad_val, pad_val, pad_val, pad_val, .1, .2])) + out = buffer.update( + signals={"a": [.1, .3, .5, .6], "b": [.1, .4, .5]}, dt=2) + np.testing.assert_allclose( + out["a"], + np.asarray([pad_val, pad_val, pad_val, pad_val, .15, .35, .5, .6])) + np.testing.assert_allclose( + out["b"], + np.asarray([pad_val, pad_val, pad_val, pad_val, .1, .15, .4, .5])) + out = buffer.update( + signals={"a": [.6, .7], "b": [.3], "c": [.4, .8]}, dt=1) + np.testing.assert_allclose( + out["a"], + np.asarray([pad_val, pad_val, pad_val, .15, .35, .5, .6, .7])) + np.testing.assert_allclose( + out["b"], + np.asarray([pad_val, pad_val, pad_val, .1, .15, .4, .5, .3])) + np.testing.assert_allclose( + out["c"], + np.asarray([pad_val, pad_val, pad_val, pad_val, pad_val, pad_val, .4, .8])) + # 2 dim, 2 signals + buffer = MultiSignalBuffer(size=4, ndim=2, pad_val=pad_val, ignore_k=['c']) + out = buffer.update( + signals={"a": [[.2, .4,], [.1, .4]], "b": [[.1, .2], [.6, .6]]}, dt=1) + np.testing.assert_allclose( + out["a"], + np.asarray([[pad_val, pad_val], [pad_val, pad_val], [.2, .4], [.1, .4]])) + np.testing.assert_allclose( + out["b"], + np.asarray([[pad_val, pad_val], [pad_val, pad_val], [.1, .2], [.6, .6]])) + out = buffer.update( + signals={"a": [[.1, .1,], [.1, .1,], [.1, .1]], "c": [[.1], [.1]]}, dt=2) + assert len(out) == 1 + np.testing.assert_allclose( + out["a"], + np.asarray([[.2, .4], [.1, .25], [.1, .1], [.1, .1]])) diff --git a/tests/test_signal.py b/tests/test_signal.py index 81f5ecc..f600142 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -24,7 +24,7 @@ import sys sys.path.append('../vitallens-python') -from vitallens.signal import windowed_mean, windowed_freq, reassemble_from_windows +from vitallens.signal import windowed_mean, windowed_freq, reassemble_from_windows, assemble_results def test_windowed_mean(): x = np.asarray([0., 1., 2., 3., 4., 5., 6.]) @@ -60,4 +60,50 @@ def test_reassemble_from_windows(): np.testing.assert_equal( out_idxs, np.asarray([1, 3, 5, 7, 9, 11, 13])) - \ No newline at end of file + +@pytest.mark.parametrize("signals", [('ppg_waveform',), ('respiratory_waveform',), ('ppg_waveform','respiratory_waveform')]) +@pytest.mark.parametrize("can_provide_confidence", [True, False]) +@pytest.mark.parametrize("min_t_too_long", [True, False]) +def test_assemble_results(signals, can_provide_confidence, min_t_too_long): + sample_video_hr = 73.2 + sample_video_rr = 15. + sig_ppg_ir = [-0.30989337,-0.34880125,-0.51679734,-0.8264251,-1.16504221,-1.42803273,-1.50548274,-1.27397076,-0.75049456,-0.06296925,0.55622022,1.03613509,1.33531581,1.49509228,1.48178068,1.37909062,1.2243668,1.12657893,1.0408531,0.88764567,0.69828361,0.53705074,0.51951649,0.53721108,0.56165349,0.50940172,0.41704494,0.26956212,0.1083913,-0.01021207,-0.05248365,-0.05707812,0.03133779,0.11226076,0.22743949,0.11899808,-0.07210865,-0.30002545,-0.33543694,-0.43031631,-0.47661662,-0.59501978,-0.63097275,-0.74791876,-0.87790608,-1.05945451,-1.24667428,-1.49371626,-1.6371542,-1.71251337,-1.71386794,-1.75321702,-1.82464616,-1.83582955,-1.73761775,-1.64881252,-1.06284365,-0.2302026,0.87515657,1.56372653,1.93155176,2.02379647,1.98393698,1.87873691,1.74305679,1.48084475,1.19594038,0.90211362,0.73778882,0.6274904,0.56723345,0.47625067,0.37326869,0.24843488,0.11191973,-0.04886665,-0.24323905,-0.42655308,-0.58504122,-0.72104162,-0.80480045,-0.82562858,-0.50066783,-0.05626648,0.5518156,0.6590093,0.73628254,0.65119874,0.77386199,0.72584785,0.65558003,0.49750651,0.39234535,0.21854192,0.14756766,0.08148715,0.06862309,-0.05206853,-0.19597003,-0.38785157,-0.53376963,-0.72356122,-0.924077,-1.120926,-1.28434747,-1.43064954,-1.5675398,-1.43950115,-0.81664647,0.16796192,1.08697704,1.55443842,1.68625269,1.52211757,1.39033429,1.1453985,0.93370787,0.57538922,0.22143884,-0.08772584,-0.33665,-0.48391509,-0.61660534,-0.72096744,-0.87922138,-1.07761103,-1.28375948,-1.51790998,-1.68144848,-1.47123422,-1.10773505,-0.47217152,0.0946784,0.83295787,1.28496731,1.42196718,1.25495147,1.02965942,0.8611786] + sig_resp = [1.51216802,1.50955699,1.51529447,1.52743288,1.65720539,1.7819858,1.77276978,1.63947779,1.47796079,1.30538046,1.13795781,0.99811216,0.82452553,0.71412734,0.72596083,0.76797981,0.76781212,0.69270434,0.58791597,0.50698886,0.47142443,0.35067527,0.2118323,0.16459586,0.22390303,0.40910957,0.5626543,0.65655808,0.64031414,0.69680373,0.75727817,0.74380619,0.62069997,0.43008055,0.31596059,0.23553526,0.18816279,0.09700333,-0.08084037,-0.23348016,-0.33336491,-0.42039115,-0.56651639,-0.75424524,-0.94991853,-1.14872326,-1.28733549,-1.42709312,-1.55680766,-1.67107495,-1.73852152,-1.79410228,-1.83959645,-1.85642986,-1.87528841,-1.85077548,-1.83781359,-1.83267193,-1.85641745,-1.90721566,-1.95186978,-1.99866851,-2.03258962,-2.10955241,-2.13515795,-2.12067511,-2.04354335,-1.91670117,-1.73937042,-1.59369458,-1.40989926,-1.18404334,-0.97035116,-0.758191,-0.6093685,-0.45600976,-0.36745855,-0.25499029,-0.17721783,-0.08083689,-0.00516917,0.05230045,0.10007278,0.20457223,0.3421804,0.45500407,0.54078405,0.51011294,0.46511979,0.41234301,0.38203958,0.31548583,0.21042761,0.09049446,-0.03285828,-0.05941505,-0.03440629,0.00453256,0.03482003,0.02501604,0.05357855,0.0906964,0.14722884,0.20599569,0.23433949,0.24324672,0.25765686,0.27466344,0.25850446,0.28364076,0.34324296,0.44705979,0.50849328,0.55803365,0.57982195,0.58001413,0.61137475,0.68543343,0.72630508,0.64339287,0.56898251,0.53510479,0.54739451,0.63840937,0.6771955,0.64794777,0.62852041,0.5871531,0.63854356,0.6999562,0.76363539,0.76443195,0.73940662,0.70987721,0.61985124,0.61910382,0.56261351,0.46039093,0.37440005] + sig, train_signals, pred_signals = [], [], [] + for s in ('ppg_waveform','respiratory_waveform'): + if s in signals: + sig.append(sig_ppg_ir if s == 'ppg_waveform' else sig_resp) + train_signals.append('ppg_waveform' if s == 'ppg_waveform' else 'respiratory_waveform') + pred_signals.append('ppg_waveform' if s == 'ppg_waveform' else 'respiratory_waveform') + pred_signals.append('heart_rate' if s == 'ppg_waveform' else 'respiratory_rate') + sig = np.asarray(sig) + conf = np.ones_like(sig) + if can_provide_confidence: + live = np.asarray([0.01083279,0.005833,0.00820795,0.32068461,0.72727495,0.73995125,0.68828821,0.76356381,0.86223269,0.66235924,0.8745808,0.9736979,0.99009115,0.99616373,0.98721313,0.98010886,0.98385429,0.9717201,0.93585575,0.8679142,0.90865433,0.93440485,0.98587644,0.96389133,0.97187209,0.92145288,0.95829332,0.95605373,0.98511922,0.99141932,0.98829323,0.98694253,0.99081576,0.88806206,0.95761001,0.8804996,0.87235355,0.93236756,0.92263389,0.67303586,0.76512074,0.66042209,0.72830695,0.6841411,0.57755405,0.57206297,0.875808,0.8154847,0.91685879,0.91369033,0.87684751,0.90468764,0.88777065,0.84473473,0.8203187,0.82623559,0.93446434,0.989048,0.98233426,0.99410701,0.9945882,0.99369907,0.99628174,0.9965958,0.99624705,0.99346352,0.9840489,0.96598941,0.98651248,0.97259837,0.97288722,0.94726074,0.97161913,0.9443177,0.97780919,0.92675537,0.97956222,0.97669303,0.98266715,0.98055196,0.98956901,0.99485874,0.96439457,0.97913766,0.97569704,0.97491169,0.97939396,0.98797679,0.97168183,0.97424555,0.97713304,0.93872136,0.94148386,0.78307801,0.86659586,0.78297913,0.85506177,0.66137809,0.7253961,0.92985141,0.93096375,0.87163657,0.90946668,0.96046972,0.96768141,0.93454564,0.92034447,0.96902579,0.87022018,0.98491764,0.98937058,0.98426282,0.99290872,0.9882791,0.99606001,0.99252284,0.99147999,0.95046616,0.9427911,0.93811965,0.93727344,0.90602505,0.94663155,0.89995974,0.95777148,0.94477475,0.97495407,0.94638491,0.9741165,0.94961405,0.9778741,0.96273482,0.97291636,0.95187879,0.99234462,0.98897183,0.99798048,0.99894512,0.99634731]) + else: + live = np.ones((sig.shape[1],)) + fps = 30. + out_data, out_conf, out_live, out_note, out_live = assemble_results(sig=sig, + conf=conf, + live=live, + fps=fps, + train_sig_names=train_signals, + pred_signals=pred_signals, + method_name="test", + can_provide_confidence=True, + min_t_hr=8. if min_t_too_long else 2., + min_t_rr=8. if min_t_too_long else 4.) + if 'ppg_waveform' in signals: + np.testing.assert_allclose(out_data['ppg_waveform'], sig_ppg_ir) + 'ppg_waveform' in out_note['ppg_waveform'] + if min_t_too_long: + 'heart_rate' not in out_data + else: + np.testing.assert_allclose(out_data['heart_rate'], sample_video_hr, atol=0.5) + if 'respiratory_waveform' in signals: + np.testing.assert_allclose(out_data['respiratory_waveform'], sig_resp) + 'respiratory_waveform' in out_note['respiratory_waveform'] + if min_t_too_long: + 'respiratory_rate' not in out_data + else: + np.testing.assert_allclose(out_data['respiratory_rate'], sample_video_rr, atol=1.) diff --git a/tests/test_simple_rppg_method.py b/tests/test_simple_rppg_method.py index 6ca04a5..d4e92e2 100644 --- a/tests/test_simple_rppg_method.py +++ b/tests/test_simple_rppg_method.py @@ -24,6 +24,7 @@ import sys sys.path.append('../vitallens-python') +from vitallens.enums import Mode from vitallens.methods.chrom import CHROMRPPGMethod from vitallens.methods.g import GRPPGMethod from vitallens.methods.pos import POSRPPGMethod @@ -32,7 +33,7 @@ @pytest.mark.parametrize("override_fps_target", [None, 15]) def test_CHROMRPPGMethod(request, override_fps_target): config = load_config("chrom.yaml") - method = CHROMRPPGMethod(config) + method = CHROMRPPGMethod(config=config, mode=Mode.BATCH) res = method.algorithm(np.random.rand(100, 3), fps=30.) assert res.shape == (100,) res = method.pulse_filter(res, fps=30.) @@ -41,7 +42,7 @@ def test_CHROMRPPGMethod(request, override_fps_target): test_video_fps = request.getfixturevalue('test_video_fps') test_video_faces = request.getfixturevalue('test_video_faces') data, unit, conf, note, live = method( - frames=test_video_ndarray, faces=test_video_faces, + inputs=test_video_ndarray, faces=test_video_faces, fps=test_video_fps, override_fps_target=override_fps_target) assert all(key in data for key in method.signals) assert all(key in unit for key in method.signals) @@ -56,7 +57,7 @@ def test_CHROMRPPGMethod(request, override_fps_target): @pytest.mark.parametrize("override_fps_target", [None, 15]) def test_GRPPGMethod(request, override_fps_target): config = load_config("g.yaml") - method = GRPPGMethod(config) + method = GRPPGMethod(config=config, mode=Mode.BATCH) res = method.algorithm(np.random.rand(100, 3), fps=30.) assert res.shape == (100,) res = method.pulse_filter(res, fps=30.) @@ -65,7 +66,7 @@ def test_GRPPGMethod(request, override_fps_target): test_video_fps = request.getfixturevalue('test_video_fps') test_video_faces = request.getfixturevalue('test_video_faces') data, unit, conf, note, live = method( - frames=test_video_ndarray, faces=test_video_faces, + inputs=test_video_ndarray, faces=test_video_faces, fps=test_video_fps, override_fps_target=override_fps_target) assert all(key in data for key in method.signals) assert all(key in unit for key in method.signals) @@ -80,7 +81,7 @@ def test_GRPPGMethod(request, override_fps_target): @pytest.mark.parametrize("override_fps_target", [None, 15]) def test_POSRPPGMethod(request, override_fps_target): config = load_config("pos.yaml") - method = POSRPPGMethod(config) + method = POSRPPGMethod(config=config, mode=Mode.BATCH) res = method.algorithm(np.random.rand(100, 3), fps=30.) assert res.shape == (100,) res = method.pulse_filter(res, fps=30.) @@ -89,7 +90,7 @@ def test_POSRPPGMethod(request, override_fps_target): test_video_fps = request.getfixturevalue('test_video_fps') test_video_faces = request.getfixturevalue('test_video_faces') data, unit, conf, note, live = method( - frames=test_video_ndarray, faces=test_video_faces, + inputs=test_video_ndarray, faces=test_video_faces, fps=test_video_fps, override_fps_target=override_fps_target) assert all(key in data for key in method.signals) assert all(key in unit for key in method.signals) diff --git a/tests/test_ssd.py b/tests/test_ssd.py index c5d1468..4640215 100644 --- a/tests/test_ssd.py +++ b/tests/test_ssd.py @@ -141,13 +141,13 @@ def test_FaceDetector(request, file): test_video_shape = request.getfixturevalue('test_video_shape') test_video_fps = request.getfixturevalue('test_video_fps') boxes, info = det(inputs=test_video_path, - inputs_shape=test_video_shape, + n_frames=test_video_shape[0], fps=test_video_fps) else: test_video_ndarray = request.getfixturevalue('test_video_ndarray') test_video_fps = request.getfixturevalue('test_video_fps') boxes, info = det(inputs=test_video_ndarray, - inputs_shape=test_video_ndarray.shape, + n_frames=test_video_ndarray.shape[0], fps=test_video_fps) assert boxes.shape == (360, 1, 4) assert info.shape == (360, 1, 4) diff --git a/tests/test_vitallens.py b/tests/test_vitallens.py index 8e21617..13d6f82 100644 --- a/tests/test_vitallens.py +++ b/tests/test_vitallens.py @@ -30,6 +30,7 @@ sys.path.append('../vitallens-python') from vitallens.constants import API_MAX_FRAMES, API_MIN_FRAMES, API_URL +from vitallens.enums import Mode from vitallens.methods.vitallens import VitalLensRPPGMethod from vitallens.utils import load_config @@ -106,10 +107,12 @@ def test_VitalLensRPPGMethod_mock(mock_post, request, file, long, override_fps_t test_video_ndarray = request.getfixturevalue('test_video_ndarray') test_video_fps = request.getfixturevalue('test_video_fps') test_video_faces = request.getfixturevalue('test_video_faces') - method = VitalLensRPPGMethod(config, api_key=api_key) + method = VitalLensRPPGMethod(config=config, + mode=Mode.BATCH, + api_key=api_key) if file: data, unit, conf, note, live = method( - frames=test_video_path, faces=test_video_faces, + inputs=test_video_path, faces=test_video_faces, override_fps_target=override_fps_target, override_global_parse=override_global_parse) else: @@ -118,7 +121,7 @@ def test_VitalLensRPPGMethod_mock(mock_post, request, file, long, override_fps_t test_video_ndarray = np.repeat(test_video_ndarray, repeats=n_repeats, axis=0) test_video_faces = np.repeat(test_video_faces, repeats=n_repeats, axis=0) data, unit, conf, note, live = method( - frames=test_video_ndarray, faces=test_video_faces, + inputs=test_video_ndarray, faces=test_video_faces, fps=test_video_fps, override_fps_target=override_fps_target, override_global_parse=override_global_parse) assert all(key in data for key in method.signals) diff --git a/vitallens/__init__.py b/vitallens/__init__.py index 8a633ef..fa1d9f1 100644 --- a/vitallens/__init__.py +++ b/vitallens/__init__.py @@ -18,4 +18,4 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from .client import VitalLens, Method +from .client import VitalLens, Method, Mode diff --git a/vitallens/buffer.py b/vitallens/buffer.py new file mode 100644 index 0000000..451ff9b --- /dev/null +++ b/vitallens/buffer.py @@ -0,0 +1,183 @@ +# Copyright (c) 2024 Rouast Labs +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import numpy as np +from typing import Union +import warnings + +class SignalBuffer: + """A buffer for an arbitrary float signal. + - Builds the buffer up over time. + - Returns overlaps using mean. + """ + def __init__( + self, + size: int, + ndim: int, + pad_val: float = 0, + init_t: int = 0 + ): + """Initialise the signal buffer. + Args: + size: The temporal length of the buffer. Signals are pushed through in FIFO order. + ndim: The number of dims for the signal (e.g., ndim=1 means scalar signal with time dimension) + pad_val: Constant scalar to pad empty time steps with + init_t: Time for initialisation + """ + self.size = size + self.pad_val = pad_val + self.min_increment = 1 + self.ndim = ndim + # Buffer is a nested list with up to self.size lists + # Each element is a tuple + # - 0: t_start + # - 1: t_end + # - 2: signal + self.buffer = [] + self.t = init_t + self.out = None + def update( + self, + signal: Union[list, np.ndarray], + dt: int + ) -> np.ndarray: + """Update the buffer with new signal series and amount of time steps passed. + - Support arbitrary number of signal dims, but mostly intended for scalar series or one more dim + (e.g., RGB signal over time). + Args: + signal: The signal. list or ndarray, shape (n_frames, dim1, dim2, ...) + dt: The number of time steps passed. Scalar + Returns: + out: The signal of buffer size, with overlaps averaged. Shape (self.size, dim1, dim2, ...) + """ + if isinstance(signal, list): signal = np.asarray(signal) + if signal.size == 1 and self.ndim == 1: signal = np.full((dt,), fill_value=signal) + assert isinstance(signal, np.ndarray), "signal should be np.ndarray but is {}".format(type(signal)) + assert len(signal.shape) == self.ndim + assert len(signal) >= 1 + assert dt >= self.min_increment + # Initialise self.out if necessary + if self.out is None: + self.out = np.empty((self.size, self.size,) + signal.shape[1:], signal.dtype) + self.out[:] = np.nan + # Update self.t + self.t += dt + # Update self.buffer + self.buffer.append((self.t - signal.shape[0], self.t, signal)) + # Delete old buffer elements + i = 0 + while i < len(self.buffer): + if self.buffer[i][1] <= self.t - self.size: + self.buffer.pop(0) + else: + i += 1 + return self.get() + def get(self) -> np.ndarray: + """Get the series of current buffer contents, with overlaps averaged. + Returns: + out: The signal of buffer size, with overlaps averaged. Shape (self.size, dim1, dim2, ...) + """ + # No elements yet + assert self.t > 0, "Update at least once before calling get()" + # Assign buffer elements to self.out + for i in range(len(self.buffer)): + adj_t = self.t - self.size + adj_t_start = self.buffer[i][0] - adj_t + adj_t_end = self.buffer[i][1] - adj_t + outside = 0 if adj_t_start >= 0 else abs(adj_t_start) + self.out[i][adj_t_start+outside:adj_t_end] = self.buffer[i][2][outside:] + # Reduce via mean (ignore warnings due to nan) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + result = np.nanmean(self.out, axis=0) + # Replace np.nan with pad_val + return np.nan_to_num(result, nan=self.pad_val) + def clear(self): + """Clear the contents""" + self.buffer.clear() + self.t = 0 + self.out = None + +class MultiSignalBuffer: + """Manages a dict of SignalBuffer instances""" + def __init__( + self, + size: int, + ndim: int, + ignore_k: list, + pad_val: float = 0 + ): + """Initialise the multi signal buffer. + Args: + size: The temporal length of each buffer. Signals are pushed through in FIFO order. + ndim: The number of dims for each signal (e.g., ndim=1 means scalar signal with time dimension) + ignore_k: List of keys to ignore in update step + pad_val: Constant scalar to pad empty time steps with + """ + self.size = size + self.ndim = ndim + self.min_increment = 1 + self.ignore_k = ignore_k + self.pad_val = pad_val + self.signals = {} + self.t = 0 + def update( + self, + signals: dict, + dt: int + ) -> dict: + """Initialise or update each of the buffers corresponding to the entries in signals dict. + Args: + signals: Dictionary of signal updates. Each entry ndarray or list of shape (n_frames, dim1, dim2, ...) + dt: The number of time steps passed. Scalar + Returns: + out: Dictionary of buffered signals, with overlaps averaged. Each entry of shape (self.size, dim1, dim2, ...) + """ + assert isinstance(signals, dict) + result = {} + for k in signals: + if k in self.ignore_k: + continue + if k not in self.signals: + # Add k to self.signals + self.signals[k] = SignalBuffer( + size=self.size, ndim=self.ndim, pad_val=self.pad_val, init_t=self.t) + # Run update + result[k] = self.signals[k].update(signals[k], dt) + self.t += dt + return result + def get(self) -> dict: + """Get the series of current buffer contents, with overlaps averaged. + Returns: + out: Dictionary of buffered signals, with overlaps averaged. Each entry of shape (self.size, dim1, dim2, ...) + """ + assert self.t > 0, "Update at least once before calling get()" + result = {} + for k in self.signals: + if k in self.ignore_k: + continue + result[k] = self.signals[k].get() + return result + def clear(self): + """Clear the contents""" + for k in self.signals: + self.signals[k].clear() + self.signals = {} + \ No newline at end of file diff --git a/vitallens/client.py b/vitallens/client.py index 2ba1dff..873c1c4 100644 --- a/vitallens/client.py +++ b/vitallens/client.py @@ -19,7 +19,6 @@ # SOFTWARE. from datetime import datetime -from enum import IntEnum import json import logging import numpy as np @@ -31,6 +30,7 @@ from vitallens.constants import DISCLAIMER from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX, CALC_HR_WINDOW_SIZE from vitallens.constants import CALC_RR_MIN, CALC_RR_MAX, CALC_RR_WINDOW_SIZE +from vitallens.enums import Method, Mode from vitallens.methods.g import GRPPGMethod from vitallens.methods.chrom import CHROMRPPGMethod from vitallens.methods.pos import POSRPPGMethod @@ -39,18 +39,13 @@ from vitallens.ssd import FaceDetector from vitallens.utils import load_config, check_faces, convert_ndarray_to_list -class Method(IntEnum): - VITALLENS = 1 - G = 2 - CHROM = 3 - POS = 4 - logging.getLogger().setLevel("INFO") class VitalLens: def __init__( self, method: Method = Method.VITALLENS, + mode: Mode = Mode.BATCH, api_key: str = None, detect_faces: bool = True, estimate_running_vitals: bool = True, @@ -65,6 +60,7 @@ def __init__( Args: method: The rPPG method to be used for inference. + mode: Operate in batch or burst mode api_key: Usage key for the VitalLens API (required for Method.VITALLENS) detect_faces: `True` if faces need to be detected, otherwise `False`. estimate_running_vitals: Set `True` to compute running vitals (e.g., `running_heart_rate`). @@ -80,17 +76,18 @@ def __init__( # Load the config and model self.config = load_config(method.name.lower() + ".yaml") self.method = method + self.mode = mode if self.config['model'] == 'g': - self.rppg = GRPPGMethod(self.config) + self.rppg = GRPPGMethod(config=self.config, mode=mode) elif self.config['model'] == 'chrom': - self.rppg = CHROMRPPGMethod(self.config) + self.rppg = CHROMRPPGMethod(config=self.config, mode=mode) elif self.config['model'] == 'pos': - self.rppg = POSRPPGMethod(self.config) + self.rppg = POSRPPGMethod(config=self.config, mode=mode) elif self.config['model'] == 'vitallens': if self.api_key is None or self.api_key == '': raise ValueError("An API key is required to use Method.VITALLENS, but was not provided. " "Get one for free at https://www.rouast.com/api.") - self.rppg = VitalLensRPPGMethod(self.config, self.api_key) + self.rppg = VitalLensRPPGMethod(config=self.config, mode=mode, api_key=self.api_key) else: raise ValueError("Method {} not implemented!".format(self.config['model'])) self.detect_faces = detect_faces @@ -102,6 +99,7 @@ def __init__( self.face_detector = FaceDetector( max_faces=fdet_max_faces, fs=fdet_fs, score_threshold=fdet_score_threshold, iou_threshold=fdet_iou_threshold) + assert not (fdet_max_faces > 1 and mode == Mode.BURST), "burst mode only supported for one face" def __call__( self, video: Union[np.ndarray, str], @@ -173,6 +171,8 @@ def __call__( ] """ # Probe inputs + if self.mode == Mode.BURST and not isinstance(video, np.ndarray): + raise ValueError("Must provide `np.ndarray` inputs for burst mode.") inputs_shape, fps, _ = probe_image_inputs(video, fps=fps, allow_image=False) # TODO: Optimize performance of simple rPPG methods for long videos # Warning if using long video @@ -182,7 +182,10 @@ def __call__( _, height, width, _ = inputs_shape if self.detect_faces: # Detect faces - faces_rel, _ = self.face_detector(inputs=video, inputs_shape=inputs_shape, fps=fps) + if self.mode == Mode.BURST: + faces_rel, _ = self.face_detector(inputs=video[-1:], n_frames=1, fps=fps) + else: + faces_rel, _ = self.face_detector(inputs=video, n_frames=inputs_shape[0], fps=fps) # If no faces detected: return empty list if len(faces_rel) == 0: logging.warning("No faces to analyze") @@ -197,10 +200,11 @@ def __call__( results = [] for face in faces: # Run selected rPPG method - data, unit, conf, note, live = self.rppg( - frames=video, faces=face, fps=fps, - override_fps_target=override_fps_target, - override_global_parse=override_global_parse) + data, unit, conf, note, live = self.rppg(inputs=video, + faces=face, + fps=fps, + override_fps_target=override_fps_target, + override_global_parse=override_global_parse) # Parse face results face_result = {'face': { 'coordinates': face, @@ -210,12 +214,13 @@ def __call__( # Parse vital signs results vital_signs_results = {} for name in self.config['signals']: - vital_signs_results[name] = { - '{}'.format('data' if 'waveform' in name else 'value'): data[name], - 'unit': unit[name], - 'confidence': conf[name], - 'note': note[name] - } + if name in data and name in unit and name in conf and name in note: + vital_signs_results[name] = { + '{}'.format('data' if 'waveform' in name else 'value'): data[name], + 'unit': unit[name], + 'confidence': conf[name], + 'note': note[name] + } if self.estimate_running_vitals: try: if 'ppg_waveform' in self.config['signals']: @@ -247,7 +252,7 @@ def __call__( 'note': 'Estimate of the running respiratory rate using VitalLens, along with frame-wise confidences between 0 and 1.', } except ValueError as e: - logging.warning("Issue while computing running vitals: {}".format(e)) + logging.debug("Issue while computing running vitals: {}".format(e)) face_result['vital_signs'] = vital_signs_results face_result['message'] = DISCLAIMER results.append(face_result) diff --git a/vitallens/enums.py b/vitallens/enums.py new file mode 100644 index 0000000..9c190ea --- /dev/null +++ b/vitallens/enums.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024 Philipp Rouast +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from enum import IntEnum + +class Method(IntEnum): + VITALLENS = 1 + G = 2 + CHROM = 3 + POS = 4 + +class Mode(IntEnum): + BATCH = 1 + BURST = 2 diff --git a/vitallens/methods/chrom.py b/vitallens/methods/chrom.py index 0d0d265..17d9184 100644 --- a/vitallens/methods/chrom.py +++ b/vitallens/methods/chrom.py @@ -24,6 +24,7 @@ from prpy.numpy.signal import detrend, standardize, butter_bandpass, div0 from prpy.numpy.stride_tricks import window_view, reduce_window_view +from vitallens.enums import Mode from vitallens.methods.simple_rppg_method import SimpleRPPGMethod from vitallens.signal import detrend_lambda_for_hr_response @@ -31,14 +32,16 @@ class CHROMRPPGMethod(SimpleRPPGMethod): """The CHROM algorithm by De Haan and Jeanne (2013)""" def __init__( self, - config: dict + config: dict, + mode: Mode ): """Initialize the `CHROMRPPGMethod` Args: config: The configuration dict + mode: The operation mode """ - super(CHROMRPPGMethod, self).__init__(config=config) + super(CHROMRPPGMethod, self).__init__(config=config, mode=mode) def algorithm( self, rgb: np.ndarray, diff --git a/vitallens/methods/g.py b/vitallens/methods/g.py index 74913a3..a358661 100644 --- a/vitallens/methods/g.py +++ b/vitallens/methods/g.py @@ -21,6 +21,7 @@ import numpy as np from prpy.numpy.signal import detrend, moving_average, standardize +from vitallens.enums import Mode from vitallens.methods.simple_rppg_method import SimpleRPPGMethod from vitallens.signal import detrend_lambda_for_hr_response from vitallens.signal import moving_average_size_for_hr_response @@ -29,14 +30,16 @@ class GRPPGMethod(SimpleRPPGMethod): """The G algorithm by Verkruysse (2008)""" def __init__( self, - config: dict + config: dict, + mode: Mode ): """Initialize the `GRPPGMethod` Args: config: The configuration dict + mode: The operation mode """ - super(GRPPGMethod, self).__init__(config=config) + super(GRPPGMethod, self).__init__(config=config, mode=mode) def algorithm( self, rgb: np.ndarray, diff --git a/vitallens/methods/pos.py b/vitallens/methods/pos.py index 38f0bfe..5d56724 100644 --- a/vitallens/methods/pos.py +++ b/vitallens/methods/pos.py @@ -23,6 +23,7 @@ from prpy.numpy.signal import detrend, moving_average, standardize, div0 from prpy.numpy.stride_tricks import window_view, reduce_window_view +from vitallens.enums import Mode from vitallens.methods.simple_rppg_method import SimpleRPPGMethod from vitallens.signal import detrend_lambda_for_hr_response from vitallens.signal import moving_average_size_for_hr_response @@ -31,14 +32,16 @@ class POSRPPGMethod(SimpleRPPGMethod): """The POS algorithm by Wang et al. (2017)""" def __init__( self, - config: dict + config: dict, + mode: Mode ): """Initialize the `POSRPPGMethod` Args: config: The configuration dict + mode: The operation mode """ - super(POSRPPGMethod, self).__init__(config=config) + super(POSRPPGMethod, self).__init__(config=config, mode=mode) def algorithm( self, rgb: np.ndarray, diff --git a/vitallens/methods/rppg_method.py b/vitallens/methods/rppg_method.py index a32e95f..213df65 100644 --- a/vitallens/methods/rppg_method.py +++ b/vitallens/methods/rppg_method.py @@ -20,15 +20,23 @@ import abc +from vitallens.enums import Mode + class RPPGMethod(metaclass=abc.ABCMeta): """Abstract superclass for rPPG methods""" - def __init__(self, config: dict): + def __init__( + self, + config: dict, + mode: Mode + ): """Initialize the `RPPGMethod` Args: config: The configuration dict + mode: The operation mode """ self.fps_target = config['fps_target'] + self.op_mode = mode self.est_window_length = config['est_window_length'] self.est_window_overlap = config['est_window_overlap'] self.est_window_flexible = self.est_window_length == 0 diff --git a/vitallens/methods/simple_rppg_method.py b/vitallens/methods/simple_rppg_method.py index e5f9f15..441d692 100644 --- a/vitallens/methods/simple_rppg_method.py +++ b/vitallens/methods/simple_rppg_method.py @@ -20,31 +20,36 @@ import abc import numpy as np -from prpy.constants import SECONDS_PER_MINUTE from prpy.numpy.face import get_roi_from_det from prpy.numpy.image import reduce_roi, parse_image_inputs -from prpy.numpy.signal import interpolate_cubic_spline, estimate_freq +from prpy.numpy.signal import interpolate_cubic_spline from typing import Union, Tuple -from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX +from vitallens.buffer import SignalBuffer +from vitallens.enums import Mode from vitallens.methods.rppg_method import RPPGMethod +from vitallens.signal import assemble_results from vitallens.utils import merge_faces class SimpleRPPGMethod(RPPGMethod): """A simple rPPG method using a handcrafted algorithm based on RGB signal trace""" def __init__( self, - config: dict + config: dict, + mode: Mode ): """Initialize the `SimpleRPPGMethod` Args: config: The configuration dict + mode: The operation mode """ - super(SimpleRPPGMethod, self).__init__(config=config) + super(SimpleRPPGMethod, self).__init__(config=config, mode=mode) self.model = config['model'] self.roi_method = config['roi_method'] self.signals = config['signals'] + if mode == Mode.BURST: + self.buffer = SignalBuffer(size=self.est_window_length, ndim=2) @abc.abstractmethod def algorithm( self, @@ -62,7 +67,7 @@ def pulse_filter(self, pass def __call__( self, - frames: Union[np.ndarray, str], # TODO: Rename to `video` + inputs: Union[np.ndarray, str], faces: np.ndarray, fps: float, override_fps_target: float = None, @@ -71,7 +76,7 @@ def __call__( """Estimate pulse signal from video frames using the subclass algorithm. Args: - frames: The video to analyze. Either a np.ndarray of shape (n_frames, h, w, 3) + inputs: The video to analyze. Either a np.ndarray of shape (n_frames, h, w, 3) in unscaled uint8 RGB format, or a path to a video file. faces: The face detection boxes as np.int64. Shape (n_frames, 4) in form (x0, y0, x1, y1) fps: The rate at which video was sampled. @@ -90,7 +95,7 @@ def __call__( faces = faces - [u_roi[0], u_roi[1], u_roi[0], u_roi[1]] # Parse the inputs frames_ds, fps, inputs_shape, ds_factor, _ = parse_image_inputs( - inputs=frames, fps=fps, roi=u_roi, target_size=None, + inputs=inputs, fps=fps, roi=u_roi, target_size=None, target_fps=override_fps_target if override_fps_target is not None else self.fps_target, preserve_aspect_ratio=False, library='prpy', scale_algorithm='bilinear', trim=None, allow_image=False, videodims=True) @@ -99,32 +104,31 @@ def __call__( assert frames_ds.shape[0] == faces_ds.shape[0], "Need same number of frames as face detections" fps_ds = fps*1.0/ds_factor # Extract rgb signal (n_frames_ds, 3) - roi_ds = np.asarray([get_roi_from_det(f, roi_method=self.roi_method) for f in faces_ds], dtype=np.int64) # roi for each frame (n, 4) - rgb_ds = reduce_roi(video=frames_ds, roi=roi_ds) + if self.op_mode == Mode.BATCH: + roi_ds = np.asarray([get_roi_from_det(f, roi_method=self.roi_method) for f in faces_ds], dtype=np.int64) # roi for each frame (n, 4) + rgb_ds = reduce_roi(video=frames_ds, roi=roi_ds) + else: + # Use the last face detection for cropping (n_frames, 3) + rgb_ds = reduce_roi(video=frames_ds, roi=np.asarray(get_roi_from_det(faces_ds[-1], roi_method=self.roi_method), dtype=np.int64)) + # Push to buffer and get buffered vals (pred_window_length, 3) + rgb_ds = self.buffer.update(rgb_ds, dt=inputs.shape[0]) # Perform rppg algorithm step (n_frames_ds,) sig_ds = self.algorithm(rgb_ds, fps_ds) # Interpolate to original sampling rate (n_frames,) sig = interpolate_cubic_spline( x=np.arange(inputs_shape[0])[0::ds_factor], y=sig_ds, xs=np.arange(inputs_shape[0]), axis=1) - # Filter (n_frames,) + # Filter and add dim (1, n_frames) sig = self.pulse_filter(sig, fps) - # Estimate HR - hr = estimate_freq( - sig, f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, - f_range=(CALC_HR_MIN/SECONDS_PER_MINUTE, CALC_HR_MAX/SECONDS_PER_MINUTE), - method='periodogram') * SECONDS_PER_MINUTE - # Assemble results - data, unit, conf, note = {}, {}, {}, {} - for name in self.signals: - if name == 'heart_rate': - data[name] = hr - unit[name] = 'bpm' - conf[name] = 1.0 - note[name] = 'Estimate of the global heart rate using {} method. This method is not capable of providing a confidence estimate, hence returning 1.'.format(self.model) - elif name == 'ppg_waveform': - data[name] = sig - unit[name] = 'unitless' - conf[name] = np.ones_like(sig) - note[name] = 'Estimate of the ppg waveform using {} method. This method is not capable of providing a confidence estimate, hence returning 1.'.format(self.model) - # Return results - return data, unit, conf, note, np.ones_like(sig) + sig = np.expand_dims(sig, axis=0) + # Simple rPPG method cannot specify confidence or live. Set to always 1. + conf = np.ones_like(sig) + live = np.ones_like(sig[0]) + # Assemble and return the results + return assemble_results(sig=sig, + conf=conf, + live=live, + fps=fps, + train_sig_names=['ppg_waveform'], + pred_signals=self.signals, + method_name=self.model, + can_provide_confidence=False) diff --git a/vitallens/methods/vitallens.py b/vitallens/methods/vitallens.py index 4f7aa8c..9fbc3fb 100644 --- a/vitallens/methods/vitallens.py +++ b/vitallens/methods/vitallens.py @@ -22,11 +22,10 @@ import concurrent.futures import math import numpy as np -from prpy.constants import SECONDS_PER_MINUTE from prpy.numpy.face import get_roi_from_det from prpy.numpy.image import probe_image_inputs, parse_image_inputs from prpy.numpy.signal import detrend, moving_average, standardize -from prpy.numpy.signal import interpolate_cubic_spline, estimate_freq +from prpy.numpy.signal import interpolate_cubic_spline from prpy.numpy.utils import enough_memory_for_ndarray import json import logging @@ -34,12 +33,12 @@ from typing import Union, Tuple from vitallens.constants import API_MAX_FRAMES, API_URL, API_OVERLAP -from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX, CALC_RR_MIN, CALC_RR_MAX +from vitallens.enums import Mode from vitallens.errors import VitalLensAPIKeyError, VitalLensAPIQuotaExceededError, VitalLensAPIError from vitallens.methods.rppg_method import RPPGMethod from vitallens.signal import detrend_lambda_for_hr_response, detrend_lambda_for_rr_response from vitallens.signal import moving_average_size_for_hr_response, moving_average_size_for_rr_response -from vitallens.signal import reassemble_from_windows +from vitallens.signal import reassemble_from_windows, assemble_results from vitallens.utils import check_faces_in_roi class VitalLensRPPGMethod(RPPGMethod): @@ -47,22 +46,25 @@ class VitalLensRPPGMethod(RPPGMethod): def __init__( self, config: dict, + mode: Mode, api_key: str ): """Initialize the `VitalLensRPPGMethod` Args: config: The configuration dict + mode: The operation mode api_key: The API key """ - super(VitalLensRPPGMethod, self).__init__(config=config) + super(VitalLensRPPGMethod, self).__init__(config=config, mode=mode) self.api_key = api_key + self.model = config['model'] self.input_size = config['input_size'] self.roi_method = config['roi_method'] self.signals = config['signals'] def __call__( self, - frames: Union[np.ndarray, str], + inputs: Union[np.ndarray, str], faces: np.ndarray, fps: float = None, override_fps_target: float = None, @@ -71,7 +73,7 @@ def __call__( """Estimate vitals from video frames using the VitalLens API. Args: - frames: The video to analyze. Either a np.ndarray of shape (n_frames, h, w, 3) + inputs: The video to analyze. Either a np.ndarray of shape (n_frames, h, w, 3) in unscaled uint8 RGB format, or a path to a video file. faces: The face detection boxes as np.int64. Shape (n_frames, 4) in form (x0, y0, x1, y1) fps: The rate at which video was sampled. @@ -86,7 +88,7 @@ def __call__( - out_note: An explanatory note for each signal. - live: The face live confidence. Shape (n_frames,) """ - inputs_shape, fps, video_issues = probe_image_inputs(frames, fps=fps) + inputs_shape, fps, video_issues = probe_image_inputs(inputs, fps=fps) # Check the number of frames to be processed inputs_n = inputs_shape[0] fps_target = override_fps_target if override_fps_target is not None else self.fps_target @@ -100,14 +102,16 @@ def __call__( global_roi = get_roi_from_det( global_face, roi_method=self.roi_method, clip_dims=(inputs_shape[2], inputs_shape[1])) global_faces_in_roi = check_faces_in_roi(faces=faces, roi=global_roi, percentage_required_inside_roi=(0.6, 1.0)) - global_parse = isinstance(frames, str) and video_fits_in_memory and (video_issues or global_faces_in_roi) + global_parse = isinstance(inputs, str) and video_fits_in_memory and (video_issues or global_faces_in_roi) if override_global_parse is not None: global_parse = override_global_parse if global_parse: # Parse entire video for inference globally frames, _, _, _, idxs = parse_image_inputs( - inputs=frames, fps=fps, roi=global_roi, target_size=self.input_size, target_fps=fps_target, + inputs=inputs, fps=fps, roi=global_roi, target_size=self.input_size, target_fps=fps_target, preserve_aspect_ratio=False, library='prpy', scale_algorithm='bilinear', trim=None, allow_image=False, videodims=True) + else: + frames = inputs # Longer videos are split up with small overlaps n_splits = 1 if expected_ds_n <= API_MAX_FRAMES else math.ceil((expected_ds_n - API_MAX_FRAMES) / (API_MAX_FRAMES - API_OVERLAP)) + 1 split_len = expected_ds_n if n_splits == 1 else math.ceil((inputs_n + (n_splits-1) * API_OVERLAP * expected_ds_factor) / n_splits) @@ -127,51 +131,24 @@ def __call__( sig_ds, idxs = reassemble_from_windows(x=sig_results, idxs=idxs_results) conf_ds, _ = reassemble_from_windows(x=conf_results, idxs=idxs_results) live_ds = reassemble_from_windows(x=np.asarray(live_results)[:,np.newaxis], idxs=idxs_results)[0][0] - # Interpolate to original sampling rate (n_frames,) + # Interpolate to original sampling rate sig = interpolate_cubic_spline( x=idxs, y=sig_ds, xs=np.arange(inputs_n), axis=1) conf = interpolate_cubic_spline( x=idxs, y=conf_ds, xs=np.arange(inputs_n), axis=1) live = interpolate_cubic_spline( x=idxs, y=live_ds, xs=np.arange(inputs_n), axis=0) - # Filter (n_frames,) + # Filter (2, n_frames) sig = np.asarray([self.postprocess(p, fps, type=name) for p, name in zip(sig, ['ppg', 'resp'])]) - # Estimate summary vitals - hr = estimate_freq( - sig[0], f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, - f_range=(CALC_HR_MIN/SECONDS_PER_MINUTE, CALC_HR_MAX/SECONDS_PER_MINUTE), - method='periodogram') * SECONDS_PER_MINUTE - rr = estimate_freq( - sig[1], f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, - f_range=(CALC_RR_MIN/SECONDS_PER_MINUTE, CALC_RR_MAX/SECONDS_PER_MINUTE), - method='periodogram') * SECONDS_PER_MINUTE - # Confidences - hr_conf = float(np.mean(conf[0])) - rr_conf = float(np.mean(conf[1])) - # Assemble results - out_data, out_unit, out_conf, out_note = {}, {}, {}, {} - for name in self.signals: - if name == 'heart_rate': - out_data[name] = hr - out_unit[name] = 'bpm' - out_conf[name] = hr_conf - out_note[name] = 'Estimate of the global heart rate using VitalLens, along with a confidence level between 0 and 1.' - elif name == 'respiratory_rate': - out_data[name] = rr - out_unit[name] = 'bpm' - out_conf[name] = rr_conf - out_note[name] = 'Estimate of the global respiratory rate using VitalLens, along with a confidence level between 0 and 1.' - elif name == 'ppg_waveform': - out_data[name] = sig[0] - out_unit[name] = 'unitless' - out_conf[name] = conf[0] - out_note[name] = 'Estimate of the ppg waveform using VitalLens, along with frame-wise confidences between 0 and 1.' - elif name == 'respiratory_waveform': - out_data[name] = sig[1] - out_unit[name] = 'unitless' - out_conf[name] = conf[1] - out_note[name] = 'Estimate of the respiratory waveform using VitalLens, along with frame-wise confidences between 0 and 1.' - return out_data, out_unit, out_conf, out_note, live + # Assemble and return the results + return assemble_results(sig=sig, + conf=conf, + live=live, + fps=fps, + train_sig_names=['ppg_waveform', 'respiratory_waveform'], + pred_signals=self.signals, + method_name=self.model, + can_provide_confidence=True) def process_api_batch( self, batch: int, diff --git a/vitallens/signal.py b/vitallens/signal.py index f11526a..3ae50fa 100644 --- a/vitallens/signal.py +++ b/vitallens/signal.py @@ -24,7 +24,7 @@ from prpy.numpy.stride_tricks import window_view, resolve_1d_window_view from typing import Tuple, Union -from vitallens.constants import CALC_HR_MAX, CALC_RR_MAX +from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX, CALC_RR_MIN, CALC_RR_MAX def moving_average_size_for_hr_response( f_s: Union[float, int] @@ -179,3 +179,74 @@ def reassemble_from_windows( result = x.reshape(x.shape[0], -1)[:,mask] idxs = idxs.flatten()[mask] return result, idxs + +def assemble_results( + sig: np.ndarray, + conf: np.ndarray, + live: np.ndarray, + fps: float, + train_sig_names: list, + pred_signals: list, + method_name: str, + can_provide_confidence: bool = True, + min_t_hr: float = 2., + min_t_rr: float = 4. + ) -> Tuple[dict, dict, dict, dict, np.ndarray]: + """Assemble rPPG method results in the format expected by the API. + Args: + sig: The estimated signals. Shape (n_sig, n_frames) + conf: The estimation confidence. Shape (n_sig, n_frames) + live: The liveness confidence. Shape (n_frames,) + fps: The sampling rate + train_sig_names: The train signal names of the method + pred_signals: The pred signals specs of the method + method_name: The name of the method + can_provide_confidence: Whether the method can provide a confidence estimate + min_t_hr: Minimum amount of time signal to estimate hr + min_t_rr: Minimum amount of time signal to estimate rr + Returns: + Tuple of + - out_data: The estimated data/value for each signal. + - out_unit: The estimation unit for each signal. + - out_conf: The estimation confidence for each signal. + - out_note: An explanatory note for each signal. + - live: The face live confidence. Shape (n_frames,) + """ + # Infer the signal length in seconds + sig_t = sig.shape[1] / fps + # Get the names of signals model outputs + out_data, out_unit, out_conf, out_note = {}, {}, {}, {} + confidence_note_scalar = ', along with a confidence level between 0 and 1.' if can_provide_confidence else '. This method is not capable of providing a confidence estimate, hence returning 1.' + confidence_note_data = ', along with frame-wise confidences between 0 and 1.' if can_provide_confidence else '. This method is not capable of providing a confidence estimate, hence returning 1.' + for name in pred_signals: + if name == 'heart_rate' and 'ppg_waveform' in train_sig_names and sig_t > min_t_hr: + ppg_ir_idx = train_sig_names.index('ppg_waveform') + out_data[name] = estimate_freq( + sig[ppg_ir_idx], f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, + f_range=(CALC_HR_MIN/SECONDS_PER_MINUTE, CALC_HR_MAX/SECONDS_PER_MINUTE), + method='periodogram') * SECONDS_PER_MINUTE + out_unit[name] = 'bpm' + out_conf[name] = float(np.mean(conf[ppg_ir_idx])) + out_note[name] = f'Estimate of the global heart rate using {method_name}{confidence_note_scalar}' + elif name == 'respiratory_rate' and 'respiratory_waveform' in train_sig_names and sig_t > min_t_rr: + resp_idx = train_sig_names.index('respiratory_waveform') + out_data[name] = estimate_freq( + sig[resp_idx], f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, + f_range=(CALC_RR_MIN/SECONDS_PER_MINUTE, CALC_RR_MAX/SECONDS_PER_MINUTE), + method='periodogram') * SECONDS_PER_MINUTE + out_unit[name] = 'bpm' + out_conf[name] = float(np.mean(conf[resp_idx])) + out_note[name] = f'Estimate of the global respiratory rate using {method_name}{confidence_note_scalar}' + elif name == 'ppg_waveform': + ppg_ir_idx = train_sig_names.index('ppg_waveform') + out_data[name] = sig[ppg_ir_idx] + out_unit[name] = 'unitless' + out_conf[name] = conf[ppg_ir_idx] + out_note[name] = f'Estimate of the ppg waveform using {method_name}{confidence_note_data}' + elif name == 'respiratory_waveform': + resp_idx = train_sig_names.index('respiratory_waveform') + out_data[name] = sig[resp_idx] + out_unit[name] = 'unitless' + out_conf[name] = conf[resp_idx] + out_note[name] = f'Estimate of the respiratory waveform using {method_name}{confidence_note_data}' + return out_data, out_unit, out_conf, out_note, live diff --git a/vitallens/ssd.py b/vitallens/ssd.py index d095fe9..91f6d97 100644 --- a/vitallens/ssd.py +++ b/vitallens/ssd.py @@ -115,7 +115,6 @@ def enforce_temporal_consistency( """ # Make sure that enough frames are present if n_frames == 1: - logging.warning("Ignoring enforce_consistency since n_frames=={}".format(n_frames)) return boxes, info # Determine the maximum number of detections in any frame max_det_faces = int(np.max(np.sum(info[...,2], axis=-1))) @@ -212,7 +211,7 @@ def __init__( def __call__( self, inputs: Tuple[np.ndarray, str], - inputs_shape: Tuple[tuple, float], + n_frames: int, fps: float ) -> Tuple[np.ndarray, np.ndarray]: """Run inference. @@ -220,7 +219,7 @@ def __call__( Args: inputs: The video to analyze. Either a np.ndarray of shape (n_frames, h, w, 3) with a sequence of frames in unscaled uint8 RGB format, or a path to a video file. - inputs_shape: The shape of the input video as (n_frames, h, w, 3) + n_frames: The number of input video frames fps: Sampling frequency of the input video. Returns: Tuple of @@ -228,7 +227,6 @@ def __call__( - info: Tuple (idx, scanned, scan_found_face, interp_valid, confidence) (n_frames, n_faces, 5) """ # Determine number of batches - n_frames = inputs_shape[0] n_batches = math.ceil((n_frames / (fps / self.fs)) / MAX_SCAN_FRAMES) if n_batches > 1: logging.info("Running face detection in {} batches...".format(n_batches)) @@ -236,10 +234,14 @@ def __call__( offsets_lengths = [(i[0], len(i)) for i in np.array_split(np.arange(n_frames), n_batches)] # Process in batches results = [self.scan_batch(inputs=inputs, batch=i, n_batches=n_batches, start=int(s), end=int(s+l), fps=fps) for i, (s, l) in enumerate(offsets_lengths)] - boxes = np.concatenate([r[0] for r in results], axis=0) - classes = np.concatenate([r[1] for r in results], axis=0) - scan_idxs = np.concatenate([r[2] for r in results], axis=0) - scan_every = int(np.max(np.diff(scan_idxs))) + if len(results) == 1: + boxes, classes, scan_idxs = results[0] + scan_idxs = np.asarray(scan_idxs) + else: + boxes = np.concatenate([r[0] for r in results], axis=0) + classes = np.concatenate([r[1] for r in results], axis=0) + scan_idxs = np.concatenate([r[2] for r in results], axis=0) + scan_every = 1 if len(scan_idxs) == 1 else int(np.max(np.diff(scan_idxs))) n_frames_scan = boxes.shape[0] # Non-max suppression idxs, num_valid = nms(boxes=boxes, @@ -256,7 +258,7 @@ def __call__( logging.warning("No faces found") return [], [] # Assort info: idx, scanned, scan_found_face, confidence - idxs = np.repeat(scan_idxs[:,np.newaxis], max_valid, axis=1)[...,np.newaxis] + idxs = np.repeat(scan_idxs[...,np.newaxis], max_valid, axis=1)[...,np.newaxis] scanned = np.ones((n_frames_scan, max_valid, 1), dtype=np.int32) scan_found_face = np.where(classes[...,1:2] < self.score_threshold, np.zeros([n_frames_scan, max_valid, 1], dtype=np.int32), scanned) info = np.r_['2', idxs, scanned, scan_found_face, classes[...,1:2]] From a69114d8e36aa22cdcddb8bb4c562c4a00a3545b Mon Sep 17 00:00:00 2001 From: Philipp Rouast Date: Wed, 13 Nov 2024 19:17:17 +1100 Subject: [PATCH 2/5] Support Method.VITALLENS for burst mode --- examples/live.py | 5 ++--- vitallens/client.py | 9 +++++++-- vitallens/methods/rppg_method.py | 5 +++++ vitallens/methods/simple_rppg_method.py | 4 ++++ vitallens/methods/vitallens.py | 14 ++++++++++++-- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/examples/live.py b/examples/live.py index bfd4bbb..2b2635b 100644 --- a/examples/live.py +++ b/examples/live.py @@ -12,6 +12,7 @@ sys.path.append('../vitallens-python') from vitallens import VitalLens, Mode, Method from vitallens.buffer import SignalBuffer, MultiSignalBuffer +from vitallens.constants import API_MIN_FRAMES def draw_roi(frame, roi): roi = np.asarray(roi).astype(np.int32) @@ -70,7 +71,6 @@ def __call__(self, inputs, fps): self.active.clear() def run(args): - """TODO""" cap = cv2.VideoCapture(0) executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) vl = VitalLensRunnable(method=args.method, api_key=args.api_key) @@ -129,9 +129,8 @@ def run(args): roi = None signal_buffer.clear() # Start next prediction - if len(frame_buffer) > 0: + if len(frame_buffer) >= (API_MIN_FRAMES if args.method == Method.VITALLENS else 1): n_frames = len(frame_buffer) - # print("New frames for prediction: {} @ pred_fps={:.2f} / ds_factor={}".format(n_frames, p_fps, ds_factor)) future = executor.submit(vl, frame_buffer.copy(), fps) frame_buffer.clear() # Sample frames diff --git a/vitallens/client.py b/vitallens/client.py index 873c1c4..0e1eb1c 100644 --- a/vitallens/client.py +++ b/vitallens/client.py @@ -27,7 +27,7 @@ from prpy.numpy.image import probe_image_inputs from typing import Union -from vitallens.constants import DISCLAIMER +from vitallens.constants import DISCLAIMER, API_MAX_FRAMES from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX, CALC_HR_WINDOW_SIZE from vitallens.constants import CALC_RR_MIN, CALC_RR_MAX, CALC_RR_WINDOW_SIZE from vitallens.enums import Method, Mode @@ -173,6 +173,8 @@ def __call__( # Probe inputs if self.mode == Mode.BURST and not isinstance(video, np.ndarray): raise ValueError("Must provide `np.ndarray` inputs for burst mode.") + if self.mode == Mode.BURST and video.shape[0] > API_MAX_FRAMES: + raise ValueError(f"Maximum number of frames in burst mode is {API_MAX_FRAMES}, but received {video.shape[0]}.") inputs_shape, fps, _ = probe_image_inputs(video, fps=fps, allow_image=False) # TODO: Optimize performance of simple rPPG methods for long videos # Warning if using long video @@ -195,7 +197,7 @@ def __call__( # Face axis first faces = np.transpose(faces, (1, 0, 2)) # Check if the faces are valid - faces = check_faces(faces, inputs_shape) + faces = check_faces(faces, inputs_shape) # Run separately for each face results = [] for face in faces: @@ -263,3 +265,6 @@ def __call__( with open(os.path.join(self.export_dir, export_filename), 'w') as f: json.dump(convert_ndarray_to_list(results), f, indent=4) return results + def reset(self): + """Reset""" + self.rppg.reset() diff --git a/vitallens/methods/rppg_method.py b/vitallens/methods/rppg_method.py index 213df65..b6566e2 100644 --- a/vitallens/methods/rppg_method.py +++ b/vitallens/methods/rppg_method.py @@ -44,3 +44,8 @@ def __init__( def __call__(self, frames, faces, fps, override_fps_target, override_global_parse): """Run inference. Abstract method to be implemented in subclasses.""" pass + @abc.abstractmethod + def reset(self): + """Reset. Abstract method to be implemented in subclasses.""" + pass + \ No newline at end of file diff --git a/vitallens/methods/simple_rppg_method.py b/vitallens/methods/simple_rppg_method.py index 441d692..683d43e 100644 --- a/vitallens/methods/simple_rppg_method.py +++ b/vitallens/methods/simple_rppg_method.py @@ -132,3 +132,7 @@ def __call__( pred_signals=self.signals, method_name=self.model, can_provide_confidence=False) + def reset(self): + """Reset""" + if self.op_mode == Mode.BURST: + self.buffer.clear() diff --git a/vitallens/methods/vitallens.py b/vitallens/methods/vitallens.py index 9fbc3fb..888663b 100644 --- a/vitallens/methods/vitallens.py +++ b/vitallens/methods/vitallens.py @@ -62,6 +62,8 @@ def __init__( self.input_size = config['input_size'] self.roi_method = config['roi_method'] self.signals = config['signals'] + if mode == Mode.BURST: + self.state = None def __call__( self, inputs: Union[np.ndarray, str], @@ -92,9 +94,9 @@ def __call__( # Check the number of frames to be processed inputs_n = inputs_shape[0] fps_target = override_fps_target if override_fps_target is not None else self.fps_target - expected_ds_factor = round(fps / fps_target) + expected_ds_factor = max(round(fps / fps_target), 1) expected_ds_n = math.ceil(inputs_n / expected_ds_factor) - # Check if we can parse the video globally + # Check if we should parse the video globally video_fits_in_memory = enough_memory_for_ndarray( shape=(expected_ds_n, self.input_size, self.input_size, 3), dtype=np.uint8, max_fraction_of_available_memory_to_use=0.1) @@ -220,6 +222,8 @@ def process_api_batch( # Prepare API header and payload headers = {"x-api-key": self.api_key} payload = {"video": base64.b64encode(frames_ds.tobytes()).decode('utf-8')} + if self.op_mode == Mode.BURST and self.state is not None: + payload["state"] = base64.b64encode(self.state.astype(np.float32).tobytes()).decode('utf-8') # Ask API to process video response = requests.post(API_URL, headers=headers, json=payload) response_body = json.loads(response.text) @@ -244,6 +248,8 @@ def process_api_batch( np.asarray(response_body["vital_signs"]["respiratory_waveform"]["confidence"]), ], axis=0) live_ds = np.asarray(response_body["face"]["confidence"]) + if self.op_mode == Mode.BURST: + self.state = np.asarray(response_body["state"]["data"], dtype=np.float32) idxs = np.asarray(idxs) return sig_ds, conf_ds, live_ds, idxs def postprocess( @@ -283,3 +289,7 @@ def postprocess( # Return assert sig.shape == (n_frames,) return sig + def reset(self): + """Reset""" + if self.op_mode == Mode.BURST: + self.state = None From 15e7130233d3b855d1b7bf9e0453e4056d21b894 Mon Sep 17 00:00:00 2001 From: Philipp Rouast Date: Thu, 14 Nov 2024 12:23:22 +1100 Subject: [PATCH 3/5] Fix test --- examples/live.py | 13 ++++++++----- tests/test_vitallens.py | 18 ++++++++++-------- vitallens/client.py | 4 ++-- vitallens/configs/vitallens.yaml | 2 ++ vitallens/methods/rppg_method.py | 1 + vitallens/methods/vitallens.py | 32 ++++++++++++++++++++++++++------ 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/examples/live.py b/examples/live.py index 2b2635b..5b014f0 100644 --- a/examples/live.py +++ b/examples/live.py @@ -2,6 +2,7 @@ import concurrent.futures import cv2 import numpy as np +from prpy.constants import SECONDS_PER_MINUTE from prpy.numpy.face import get_upper_body_roi_from_det from prpy.numpy.signal import estimate_freq import sys @@ -13,6 +14,7 @@ from vitallens import VitalLens, Mode, Method from vitallens.buffer import SignalBuffer, MultiSignalBuffer from vitallens.constants import API_MIN_FRAMES +from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX, CALC_RR_MIN, CALC_RR_MAX def draw_roi(frame, roi): roi = np.asarray(roi).astype(np.int32) @@ -49,9 +51,10 @@ def draw_fps(frame, fps, text, draw_area_bl_x, draw_area_bl_y): cv2.putText(frame, text='{}: {:.1f}'.format(text, fps), org=(draw_area_bl_x, draw_area_bl_y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.6, color=(0,255,0), thickness=1) -def draw_vital(frame, sig, text, sig_name, fps, mult, color, draw_area_bl_x, draw_area_bl_y): +def draw_vital(frame, sig, text, sig_name, fps, color, draw_area_bl_x, draw_area_bl_y): if sig_name in sig: - val = estimate_freq(x=sig[sig_name], f_s=fps, f_res=0.0167, method='periodogram') * mult + f_range = (CALC_HR_MIN/SECONDS_PER_MINUTE, CALC_HR_MAX/SECONDS_PER_MINUTE) if 'heart' in sig_name else (CALC_RR_MIN/SECONDS_PER_MINUTE, CALC_RR_MAX/SECONDS_PER_MINUTE) + val = estimate_freq(x=sig[sig_name], f_s=fps, f_res=0.1/SECONDS_PER_MINUTE, f_range=f_range, method='periodogram') * SECONDS_PER_MINUTE cv2.putText(frame, text='{}: {:.1f}'.format(text, val), org=(draw_area_bl_x, draw_area_bl_y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.6, color=color, thickness=1) @@ -131,7 +134,7 @@ def run(args): # Start next prediction if len(frame_buffer) >= (API_MIN_FRAMES if args.method == Method.VITALLENS else 1): n_frames = len(frame_buffer) - future = executor.submit(vl, frame_buffer.copy(), fps) + executor.submit(vl, frame_buffer.copy(), fps) frame_buffer.clear() # Sample frames if i % ds_factor == 0: @@ -149,8 +152,8 @@ def run(args): draw_area_tl_x=roi[2]+20, draw_area_tl_y=int(roi[1]+(roi[3]-roi[1])/2.0), color=(255, 0, 0)) draw_fps(frame, fps=fps, text="fps", draw_area_bl_x=roi[0], draw_area_bl_y=roi[3]+20) draw_fps(frame, fps=p_fps, text="p_fps", draw_area_bl_x=int(roi[0]+0.4*(roi[2]-roi[0])), draw_area_bl_y=roi[3]+20) - draw_vital(frame, sig=signals, text="hr [bpm]", sig_name='ppg_waveform_sig', fps=fps, mult=60., color=(0,0,255), draw_area_bl_x=roi[2]+20, draw_area_bl_y=int(roi[1]+(roi[3]-roi[1])/2.0)) - draw_vital(frame, sig=signals, text="rr [rpm]", sig_name='respiratory_waveform_sig', fps=fps, mult=60., color=(255,0,0), draw_area_bl_x=roi[2]+20, draw_area_bl_y=roi[3]) + draw_vital(frame, sig=signals, text="hr [bpm]", sig_name='ppg_waveform_sig', fps=fps, color=(0,0,255), draw_area_bl_x=roi[2]+20, draw_area_bl_y=int(roi[1]+(roi[3]-roi[1])/2.0)) + draw_vital(frame, sig=signals, text="rr [rpm]", sig_name='respiratory_waveform_sig', fps=fps, color=(255,0,0), draw_area_bl_x=roi[2]+20, draw_area_bl_y=roi[3]) cv2.imshow('Live', frame) c = cv2.waitKey(1) if c == 27: diff --git a/tests/test_vitallens.py b/tests/test_vitallens.py index 13d6f82..2b5531a 100644 --- a/tests/test_vitallens.py +++ b/tests/test_vitallens.py @@ -135,7 +135,8 @@ def test_VitalLensRPPGMethod_mock(mock_post, request, file, long, override_fps_t assert live.shape == (test_video_ndarray.shape[0],) @pytest.mark.parametrize("process_signals", [True, False]) -def test_VitalLens_API_valid_response(request, process_signals): +@pytest.mark.parametrize("n_frames", [16, 250]) +def test_VitalLens_API_valid_response(request, process_signals, n_frames): config = load_config("vitallens.yaml") api_key = request.getfixturevalue('test_dev_api_key') test_video_ndarray = request.getfixturevalue('test_video_ndarray') @@ -145,7 +146,7 @@ def test_VitalLens_API_valid_response(request, process_signals): inputs=test_video_ndarray, fps=test_video_fps, target_size=config['input_size'], roi=test_video_faces[0].tolist(), library='prpy', scale_algorithm='bilinear') headers = {"x-api-key": api_key} - payload = {"video": base64.b64encode(frames[:16].tobytes()).decode('utf-8')} + payload = {"video": base64.b64encode(frames[:n_frames].tobytes()).decode('utf-8')} if process_signals: payload['fps'] = str(30) response = requests.post(API_URL, headers=headers, json=payload) response_body = json.loads(response.text) @@ -157,13 +158,14 @@ def test_VitalLens_API_valid_response(request, process_signals): ppg_waveform_conf = np.asarray(response_body["vital_signs"]["ppg_waveform"]["confidence"]) resp_waveform_data = np.asarray(response_body["vital_signs"]["respiratory_waveform"]["data"]) resp_waveform_conf = np.asarray(response_body["vital_signs"]["respiratory_waveform"]["confidence"]) - assert ppg_waveform_data.shape == (16,) - assert ppg_waveform_conf.shape == (16,) - assert resp_waveform_data.shape == (16,) - assert resp_waveform_conf.shape == (16,) - assert all((key in vital_signs) if process_signals else (key not in vital_signs) for key in ["heart_rate", "respiratory_rate"]) + assert ppg_waveform_data.shape == (n_frames,) + assert ppg_waveform_conf.shape == (n_frames,) + assert resp_waveform_data.shape == (n_frames,) + assert resp_waveform_conf.shape == (n_frames,) + t = n_frames/test_video_fps + assert all((key in vital_signs) if (process_signals and t > 8.) else (key not in vital_signs) for key in ["heart_rate", "respiratory_rate"]) live = np.asarray(response_body["face"]["confidence"]) - assert live.shape == (16,) + assert live.shape == (n_frames,) state = np.asarray(response_body["state"]["data"]) assert state.shape == (2, 128) diff --git a/vitallens/client.py b/vitallens/client.py index 0e1eb1c..94790e0 100644 --- a/vitallens/client.py +++ b/vitallens/client.py @@ -173,8 +173,8 @@ def __call__( # Probe inputs if self.mode == Mode.BURST and not isinstance(video, np.ndarray): raise ValueError("Must provide `np.ndarray` inputs for burst mode.") - if self.mode == Mode.BURST and video.shape[0] > API_MAX_FRAMES: - raise ValueError(f"Maximum number of frames in burst mode is {API_MAX_FRAMES}, but received {video.shape[0]}.") + if self.mode == Mode.BURST and video.shape[0] > (API_MAX_FRAMES - self.rppg.n_inputs + 1): + raise ValueError(f"Maximum number of frames in burst mode is {API_MAX_FRAMES - self.rppg.n_inputs + 1}, but received {video.shape[0]}.") inputs_shape, fps, _ = probe_image_inputs(video, fps=fps, allow_image=False) # TODO: Optimize performance of simple rPPG methods for long videos # Warning if using long video diff --git a/vitallens/configs/vitallens.yaml b/vitallens/configs/vitallens.yaml index 88c6610..06fdd18 100644 --- a/vitallens/configs/vitallens.yaml +++ b/vitallens/configs/vitallens.yaml @@ -27,6 +27,8 @@ model: 'vitallens' # Size of the input input_size: 40 +# Number of inputs +n_inputs: 4 # List estimated signals signals: ['heart_rate', 'respiratory_rate', 'ppg_waveform', 'respiratory_waveform'] diff --git a/vitallens/methods/rppg_method.py b/vitallens/methods/rppg_method.py index b6566e2..4d85743 100644 --- a/vitallens/methods/rppg_method.py +++ b/vitallens/methods/rppg_method.py @@ -37,6 +37,7 @@ def __init__( """ self.fps_target = config['fps_target'] self.op_mode = mode + self.n_inputs = 1 self.est_window_length = config['est_window_length'] self.est_window_overlap = config['est_window_overlap'] self.est_window_flexible = self.est_window_length == 0 diff --git a/vitallens/methods/vitallens.py b/vitallens/methods/vitallens.py index 888663b..0e0917c 100644 --- a/vitallens/methods/vitallens.py +++ b/vitallens/methods/vitallens.py @@ -60,10 +60,12 @@ def __init__( self.api_key = api_key self.model = config['model'] self.input_size = config['input_size'] + self.n_inputs = config['n_inputs'] self.roi_method = config['roi_method'] self.signals = config['signals'] if mode == Mode.BURST: self.state = None + self.input_buffer = None def __call__( self, inputs: Union[np.ndarray, str], @@ -140,8 +142,9 @@ def __call__( x=idxs, y=conf_ds, xs=np.arange(inputs_n), axis=1) live = interpolate_cubic_spline( x=idxs, y=live_ds, xs=np.arange(inputs_n), axis=0) - # Filter (2, n_frames) - sig = np.asarray([self.postprocess(p, fps, type=name) for p, name in zip(sig, ['ppg', 'resp'])]) + # Filter only in batch mode (2, n_frames) + if self.op_mode == Mode.BATCH: + sig = np.asarray([self.postprocess(p, fps, type=name) for p, name in zip(sig, ['ppg', 'resp'])]) # Assemble and return the results return assemble_results(sig=sig, conf=conf, @@ -209,6 +212,16 @@ def process_api_batch( else: idxs = list(range(0, inputs_shape[0], ds_factor)) else: + # Buffer inputs for burst mode + if self.op_mode == Mode.BURST: + # Inputs in burst mode are always np.ndarray + if self.state is not None: + # State has been initialized + assert self.input_buffer is not None + if inputs.shape[1:] != self.input_buffer.shape[1:]: + raise ValueError("In burst mode, input dimensions must be consistent.") + inputs = np.concatenate([self.input_buffer, inputs], axis=0) + self.input_buffer = inputs[-(self.n_inputs-1):] # Inputs have not been parsed globally. Parse the inputs frames_ds, _, _, ds_factor, idxs = parse_image_inputs( inputs=inputs, fps=fps, roi=roi, target_size=self.input_size, target_fps=fps_target, @@ -216,14 +229,21 @@ def process_api_batch( trim=(start, end) if start is not None and end is not None else None, allow_image=False, videodims=True) # Make sure we have the correct number of frames + idxs = np.asarray(idxs) expected_n = math.ceil(((end-start) if start is not None and end is not None else inputs_shape[0]) / ds_factor) - if frames_ds.shape[0] != expected_n or len(idxs) != expected_n: + if (self.op_mode == Mode.BURST and self.state is not None): expected_n += (self.n_inputs - 1) + if frames_ds.shape[0] != expected_n or idxs.shape[0] != expected_n: raise ValueError("Unexpected number of frames returned. Try to set `override_global_parse` to `True` or `False`.") # Prepare API header and payload headers = {"x-api-key": self.api_key} payload = {"video": base64.b64encode(frames_ds.tobytes()).decode('utf-8')} - if self.op_mode == Mode.BURST and self.state is not None: - payload["state"] = base64.b64encode(self.state.astype(np.float32).tobytes()).decode('utf-8') + if self.op_mode == Mode.BURST: + if self.state is not None: + # State and frame buffer have been initialized + assert self.input_buffer is not None + payload["state"] = base64.b64encode(self.state.astype(np.float32).tobytes()).decode('utf-8') + # Adjust idxs + idxs = idxs[3:] - 3 # Ask API to process video response = requests.post(API_URL, headers=headers, json=payload) response_body = json.loads(response.text) @@ -250,7 +270,6 @@ def process_api_batch( live_ds = np.asarray(response_body["face"]["confidence"]) if self.op_mode == Mode.BURST: self.state = np.asarray(response_body["state"]["data"], dtype=np.float32) - idxs = np.asarray(idxs) return sig_ds, conf_ds, live_ds, idxs def postprocess( self, @@ -293,3 +312,4 @@ def reset(self): """Reset""" if self.op_mode == Mode.BURST: self.state = None + self.input_buffer = None From d36f665fe30cb47805ca4ed7cc5c53d3e2e4463d Mon Sep 17 00:00:00 2001 From: Philipp Rouast Date: Thu, 14 Nov 2024 12:53:38 +1100 Subject: [PATCH 4/5] Update README --- README.md | 23 +++++++++++++++++++++-- examples/README.md | 23 +++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2a54a1c..dbb69e5 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ It can be configured using the following parameters: | Parameter | Description | Default | |-------------------------|------------------------------------------------------------------------------------|--------------------| | method | Inference method. {`Method.VITALLENS`, `Method.POS`, `Method.CHROM` or `Method.G`} | `Method.VITALLENS` | +| mode | Operation mode. {`Mode.BATCH` for indep. videos or `Mode.BURST` for video stream} | `Mode.BATCH` | | api_key | Usage key for the VitalLens API (required for `Method.VITALLENS`) | `None` | | detect_faces | `True` if faces need to be detected, otherwise `False`. | `True` | | estimate_running_vitals | Set `True` to compute running vitals (e.g., `running_heart_rate`). | `True` | @@ -66,7 +67,8 @@ It can be configured using the following parameters: | export_dir | The directory to which json files are written. | `.` | Once instantiated, `vitallens.VitalLens` can be called to estimate vitals. -This can also be configured using the following parameters: +In `Mode.BATCH` calls are assumed to be working on independent videos, whereas in `Mode.BURST` we expect the subsequent calls to pass the next frames of the same video (stream) as `np.ndarray`. +Calls are configured using the following parameters: | Parameter | Description | Default | |---------------------|---------------------------------------------------------------------------------------|---------| @@ -148,9 +150,24 @@ If the video is long enough and `estimate_running_vitals=True`, the results addi ] ``` +## Example: Live test with webcam in real-time + +Test `vitallens` in real-time with your webcam using the script `examples/live.py`. +This uses `Mode.BURST` to update results continuously (approx. every 2 seconds for `Method.VITALLENS`). +Some options are available: + +- `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`) +- `api_key`: Pass your API Key. Required if using `method=VITALLENS`. + +May need to install requirements first: `pip install opencv-python` + +``` +python examples/live.py --method=VITALLENS --api_key=YOUR_API_KEY +``` + ### Example: Compare results with gold-standard labels using our example script -There is an example Python script in `examples/test.py` which lets you run vitals estimation and plot the predictions against ground truth labels recorded with gold-standard medical equipment. +There is an example Python script in `examples/test.py` which uses `Mode.BATCH` to run vitals estimation and plot the predictions against ground truth labels recorded with gold-standard medical equipment. Some options are available: - `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`) @@ -158,6 +175,8 @@ Some options are available: - `vitals_path`: Path to gold-standard vitals (Default: `examples/sample_vitals_1.csv`) - `api_key`: Pass your API Key. Required if using `method=VITALLENS`. +May need to install requirements first: `pip install matplotlib pandas` + For example, to reproduce the results from the banner image on the [VitalLens API Webpage](https://www.rouast.com/api/): ``` diff --git a/examples/README.md b/examples/README.md index cc48136..21d9aa5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,21 @@ -# Sample videos with ground truth labels +# Example ways to test `vitallens` + +## Live test with webcam in real-time + +Test `vitallens` in real-time with your webcam using the script `live.py`. +This uses `Mode.BURST` to update results continuously (approx. every 2 seconds for `Method.VITALLENS`). +Some options are available: + +- `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`) +- `api_key`: Pass your API Key. Required if using `method=VITALLENS`. + +May need to install requirements first: `pip install opencv-python` + +``` +python examples/live.py --method=VITALLENS --api_key=YOUR_API_KEY +``` + +## Sample videos with ground truth labels In this folder, you can find two sample videos to test `vitallens` on. Each video has ground truth labels recorded with gold-standard medical equipment. @@ -6,7 +23,7 @@ Each video has ground truth labels recorded with gold-standard medical equipment - `sample_video_1.mp4` which has ground truth labels for PPG Waveform (`ppg`), ECG Waveform (`ecg`), Respiratory Waveform (`resp`), Blood Pressure (`sbp` and `dbp`), Blood Oxygen (`spo2`), Heart Rate (`hr_ecg` - derived from ECG and `hr_ppg` - derived from PPG), Heart Rate Variability (`hrv_sdnn_ecg`), and Respiratory Rate (`rr`). - `sample_video_2.mp4` which has ground truth labels for PPG Waveform (`ppg`). This sample is kindly provided by the [VitalVideos](http://vitalvideos.org) dataset. -There is a test script in `test.py` which lets you run vitals estimation and plot the predictions against the ground truth labels. +There is a test script in `test.py` which uses `Mode.BATCH` to run vitals estimation and plot the predictions against the ground truth labels. This uses `vitallens.Mode.BATCH` mode. Some options are available: - `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`) @@ -14,6 +31,8 @@ Some options are available: - `vitals_path`: Path to gold-standard vitals (Default: `examples/sample_vitals_1.csv`) - `api_key`: Pass your API Key. Required if using `method=VITALLENS`. +May need to install requirements first: `pip install matplotlib pandas` + For example, to reproduce the results from the banner image on the [VitalLens API Webpage](https://www.rouast.com/api/): ``` From 7a062e451725e3a35821d0cd4b221c0480c9fc7c Mon Sep 17 00:00:00 2001 From: Philipp Rouast Date: Thu, 14 Nov 2024 12:55:56 +1100 Subject: [PATCH 5/5] README formatting --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dbb69e5..2bb0240 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,9 @@ If the video is long enough and `estimate_running_vitals=True`, the results addi ] ``` -## Example: Live test with webcam in real-time +## Examples to get started + +### Live test with webcam in real-time Test `vitallens` in real-time with your webcam using the script `examples/live.py`. This uses `Mode.BURST` to update results continuously (approx. every 2 seconds for `Method.VITALLENS`). @@ -165,7 +167,7 @@ May need to install requirements first: `pip install opencv-python` python examples/live.py --method=VITALLENS --api_key=YOUR_API_KEY ``` -### Example: Compare results with gold-standard labels using our example script +### Compare results with gold-standard labels using our example script There is an example Python script in `examples/test.py` which uses `Mode.BATCH` to run vitals estimation and plot the predictions against ground truth labels recorded with gold-standard medical equipment. Some options are available: @@ -185,7 +187,7 @@ python examples/test.py --method=VITALLENS --video_path=examples/sample_video_2. This sample is kindly provided by the [VitalVideos](http://vitalvideos.org) dataset. -### Example: Use VitalLens API to estimate vitals from a video file +### Use VitalLens API to estimate vitals from a video file ```python from vitallens import VitalLens, Method @@ -194,7 +196,7 @@ vl = VitalLens(method=Method.VITALLENS, api_key="YOUR_API_KEY") result = vl("video.mp4") ``` -### Example: Use POS method on an `np.ndarray` of video frames +### Use POS method on an `np.ndarray` of video frames ```python from vitallens import VitalLens, Method @@ -205,7 +207,7 @@ vl = VitalLens(method=Method.POS) result = vl(my_video_arr, fps=my_video_fps) ``` -### Example: Run example script with Docker +### Run example script with Docker If you encounter issues installing `vitallens-python` dependencies directly, you can use our Docker image, which contains all necessary tools and libraries. This docker image is set up to execute the example Python script in `examples/test.py` for you.