Merge pull request #7 from Rouast-Labs/running-vitals

Running vitals, JSON export, and lighter build
Rouast-Labs · Jul 21, 2024 · 1bf997e · 1bf997e
2 parents 92b6728 + 2be0ee6
commit 1bf997e
Show file tree

Hide file tree

Showing 16 changed files with 517 additions and 101 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,6 +3,8 @@ recursive-include vitallens/configs *
 recursive-include vitallens/methods *
 recursive-include vitallens/models *
 recursive-include tests *
+exclude examples/*.mp4
+exclude examples/*.csv
 global-exclude .DS_Store
 prune **/__pycache__ 
 prune .github/workflows
diff --git a/README.md b/README.md
@@ -54,23 +54,27 @@ pip install ./vitallens-python
 To start using `vitallens-python`, first create an instance of `vitallens.VitalLens`. 
 It can be configured using the following parameters:
 
-| Parameter      | Description                                                                        | Default            |
-|----------------|------------------------------------------------------------------------------------|--------------------|
-| method         | Inference method. {`Method.VITALLENS`, `Method.POS`, `Method.CHROM` or `Method.G`} | `Method.VITALLENS` |
-| api_key        | Usage key for the VitalLens API (required for `Method.VITALLENS`)                  | `None`             |
-| detect_faces   | `True` if faces need to be detected, otherwise `False`.                            | `True`             |
-| fdet_max_faces | The maximum number of faces to detect (if necessary).                              | `2`                |
-| fdet_fs        | Frequency [Hz] at which faces should be scanned - otherwise linearly interpolated. | `1.0`              |
+| Parameter               | Description                                                                        | Default            |
+|-------------------------|------------------------------------------------------------------------------------|--------------------|
+| method                  | Inference method. {`Method.VITALLENS`, `Method.POS`, `Method.CHROM` or `Method.G`} | `Method.VITALLENS` |
+| api_key                 | Usage key for the VitalLens API (required for `Method.VITALLENS`)                  | `None`             |
+| detect_faces            | `True` if faces need to be detected, otherwise `False`.                            | `True`             |
+| estimate_running_vitals | Set `True` to compute running vitals (e.g., `running_heart_rate`).                 | `True`             |
+| fdet_max_faces          | The maximum number of faces to detect (if necessary).                              | `1`                |
+| fdet_fs                 | Frequency [Hz] at which faces should be scanned - otherwise linearly interpolated. | `1.0`              |
+| export_to_json          | If `True`, write results to a json file.                                           | `True`             |
+| export_dir              | The directory to which json files are written.                                     | `.`                |
 
 Once instantiated, `vitallens.VitalLens` can be called to estimate vitals.
 This can also be configured using the following parameters:
 
 | Parameter           | Description                                                                           | Default |
 |---------------------|---------------------------------------------------------------------------------------|---------|
-| video               | The video to analyze. Either a path to a video file or `np.ndarray`. [More info here.](https://github.com/Rouast-Labs/vitallens-python/blob/ddcf48f29a2765fd98a7029c0f10075a33e44247/vitallens/client.py#L98)    |         |
-| faces               | Face detections. Ignored unless `detect_faces=False`. [More info here.](https://github.com/Rouast-Labs/vitallens-python/blob/ddcf48f29a2765fd98a7029c0f10075a33e44247/vitallens/client.py#L101) | `None`  |
+| video               | The video to analyze. Either a path to a video file or `np.ndarray`. [More info here.](https://github.com/Rouast-Labs/vitallens-python/raw/main/vitallens/client.py#L114)    |         |
+| faces               | Face detections. Ignored unless `detect_faces=False`. [More info here.](https://github.com/Rouast-Labs/vitallens-python/raw/main/vitallens/client.py#L117) | `None`  |
 | fps                 | Sampling frequency of the input video. Required if video is `np.ndarray`.             | `None`  |
 | override_fps_target | Target frequency for inference (optional - use methods's default otherwise).          | `None`  |
+| export_filename     | Filename for json export if applicable.                                               | `None`  |
 
 The estimation results are returned as a `list`. It contains a `dict` for each distinct face, with the following structure:
 
@@ -84,13 +88,13 @@ The estimation results are returned as a `list`. It contains a `dict` for each d
     },
     'vital_signs': {
       'heart_rate': {
-          'value': <Estimated value as float scalar>,
-          'unit': <Value unit>,
-          'confidence': <Estimation confidence as float scalar>,
-          'note': <Explanatory note>
-        },
+        'value': <Estimated global value as float scalar>,
+        'unit': <Value unit>,
+        'confidence': <Estimation confidence as float scalar>,
+        'note': <Explanatory note>
+      },
       'respiratory_rate': {
-        'value': <Estimated value as float scalar>,
+        'value': <Estimated global value as float scalar>,
         'unit': <Value unit>,
         'confidence': <Estimation confidence as float scalar>,
         'note': <Explanatory note>
@@ -117,6 +121,51 @@ The estimation results are returned as a `list`. It contains a `dict` for each d
   ]
 ```
 
+If the video is long enough and `estimate_running_vitals=True`, the results additionally contain running vitals:
+
+```
+[
+  {
+    ...
+    'vital_signs': {
+      ...
+      'running_heart_rate': {
+        'data': <Estimated value for each frame as np.ndarray of shape (n_frames,)>,
+        'unit': <Value unit>,
+        'confidence': <Estimation confidence for each frame as np.ndarray of shape (n_frames,)>,
+        'note': <Explanatory note>
+      },
+      'running_respiratory_rate': {
+        'data': <Estimated value for each frame as np.ndarray of shape (n_frames,)>,
+        'unit': <Value unit>,
+        'confidence': <Estimation confidence for each frame as np.ndarray of shape (n_frames,)>,
+        'note': <Explanatory note>
+      }
+    }
+  ...
+  },
+  ...
+]
+```
+
+### Example: Compare results with gold-standard labels using our example script
+
+There is an example Python script in `examples/test.py` which lets you run vitals estimation and plot the predictions against ground truth labels recorded with gold-standard medical equipment.
+Some options are available:
+
+- `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`)
+- `video_path`: Path to video (Default: `examples/sample_video_1.mp4`)
+- `vitals_path`: Path to gold-standard vitals (Default: `examples/sample_vitals_1.csv`)
+- `api_key`: Pass your API Key. Required if using `method=VITALLENS`.
+
+For example, to reproduce the results from the banner image on the [VitalLens API Webpage](https://www.rouast.com/api/):
+
+```
+python examples/test.py --method=VITALLENS --video_path=examples/sample_video_2.mp4 --vitals_path=examples/sample_vitals_2.csv --api_key=YOUR_API_KEY
+```
+
+This sample is kindly provided by the [VitalVideos](http://vitalvideos.org) dataset.
+
 ### Example: Use VitalLens API to estimate vitals from a video file
 
 ```python

diff --git a/examples/README.md b/examples/README.md
@@ -0,0 +1,21 @@
+# Sample videos with ground truth labels
+
+In this folder, you can find two sample videos to test `vitallens` on.
+Each video has ground truth labels recorded with gold-standard medical equipment.
+
+- `sample_video_1.mp4` which has ground truth labels for PPG Waveform (`ppg`), ECG Waveform (`ecg`), Respiratory Waveform (`resp`), Blood Pressure (`sbp` and `dbp`), Blood Oxygen (`spo2`), Heart Rate (`hr_ecg` - derived from ECG and `hr_ppg` - derived from PPG), Heart Rate Variability (`hrv_sdnn_ecg`), and Respiratory Rate (`rr`).
+- `sample_video_2.mp4` which has ground truth labels for PPG Waveform (`ppg`). This sample is kindly provided by the [VitalVideos](http://vitalvideos.org) dataset.
+
+There is a test script in `test.py` which lets you run vitals estimation and plot the predictions against the ground truth labels.
+Some options are available:
+
+- `method`: Choose from [`VITALLENS`, `POS`, `G`, `CHROM`] (Default: `VITALLENS`)
+- `video_path`: Path to video (Default: `examples/sample_video_1.mp4`)
+- `vitals_path`: Path to gold-standard vitals (Default: `examples/sample_vitals_1.csv`)
+- `api_key`: Pass your API Key. Required if using `method=VITALLENS`.
+
+For example, to reproduce the results from the banner image on the [VitalLens API Webpage](https://www.rouast.com/api/):
+
+```
+python examples/test.py --method=VITALLENS --video_path=examples/sample_video_2.mp4 --vitals_path=examples/sample_vitals_2.csv --api_key=YOUR_API_KEY
+```
diff --git a/examples/test.py b/examples/test.py
@@ -4,12 +4,16 @@
 import matplotlib.pyplot as plt
 import os
 import pandas as pd
+from prpy.constants import SECONDS_PER_MINUTE
 from prpy.ffmpeg.probe import probe_video
 from prpy.ffmpeg.readwrite import read_video_from_path
 from prpy.helpers import str2bool
 from prpy.numpy.signal import estimate_freq
 import timeit
 from vitallens import VitalLens, Method
+from vitallens.utils import download_file
+from vitallens.constants import CALC_HR_MIN, CALC_HR_MAX
+from vitallens.constants import CALC_RR_MIN, CALC_RR_MAX
 
 COLOR_GT = '#000000'
 METHOD_COLORS = {
@@ -18,8 +22,21 @@
   Method.CHROM: '#4ceaff',
   Method.POS: '#23b031'
 }
+SAMPLE_VIDEO_URLS = {
+  'examples/sample_video_1.mp4': 'https://github.com/Rouast-Labs/vitallens-python/raw/main/examples/sample_video_1.mp4',
+  'examples/sample_video_2.mp4': 'https://github.com/Rouast-Labs/vitallens-python/raw/main/examples/sample_video_2.mp4',
+}
+SAMPLE_VITALS_URLS = {
+  'examples/sample_vitals_1.csv': 'https://github.com/Rouast-Labs/vitallens-python/raw/main/examples/sample_vitals_1.csv',
+  'examples/sample_vitals_2.csv': 'https://github.com/Rouast-Labs/vitallens-python/raw/main/examples/sample_vitals_2.csv',
+}
 
 def run(args=None):
+  # Download sample data if necessary
+  if args.video_path in SAMPLE_VIDEO_URLS.keys() and not os.path.exists(args.video_path):
+    download_file(url=SAMPLE_VIDEO_URLS[args.video_path], dest=args.video_path)
+  if args.vitals_path in SAMPLE_VITALS_URLS.keys() and not os.path.exists(args.vitals_path):
+    download_file(url=SAMPLE_VITALS_URLS[args.vitals_path], dest=args.vitals_path)
   # Get ground truth vitals
   vitals = pd.read_csv(args.vitals_path) if os.path.exists(args.vitals_path) else []
   ppg_gt = vitals['ppg'] if 'ppg' in vitals else None 
@@ -48,10 +65,10 @@ def run(args=None):
     fig, ax1 = plt.subplots(1, figsize=(12, 6))
   fig.suptitle('Vital signs estimated from {} using {} in {:.2f} ms'.format(args.video_path, args.method.name, time_ms))
   if "ppg_waveform" in vital_signs and ppg_gt is not None:
-    hr_gt = estimate_freq(ppg_gt, f_s=fps, f_res=0.005, f_range=(40./60., 240./60.), method='periodogram') * 60.
+    hr_gt = estimate_freq(ppg_gt, f_s=fps, f_res=0.005, f_range=(CALC_HR_MIN/SECONDS_PER_MINUTE, CALC_HR_MAX/SECONDS_PER_MINUTE), method='periodogram') * SECONDS_PER_MINUTE
     ax1.plot(ppg_gt, color=COLOR_GT, label='PPG Waveform Ground Truth -> HR: {:.1f} bpm'.format(hr_gt))
   if "respiratory_waveform" in vital_signs and resp_gt is not None:
-    rr_gt = estimate_freq(resp_gt, f_s=fps, f_res=0.005, f_range=(4./60., 90./60.), method='periodogram') * 60.
+    rr_gt = estimate_freq(resp_gt, f_s=fps, f_res=0.005, f_range=(CALC_RR_MIN/SECONDS_PER_MINUTE, CALC_RR_MAX/SECONDS_PER_MINUTE), method='periodogram') * SECONDS_PER_MINUTE
     ax2.plot(resp_gt, color=COLOR_GT, label='Respiratory Waveform Ground Truth -> RR: {:.1f} bpm'.format(rr_gt))
   if "ppg_waveform" in vital_signs:
     ax1.plot(vital_signs['ppg_waveform']['data'], color=METHOD_COLORS[args.method], label='PPG Waveform Estimate -> HR: {:.1f} bpm ({:.0f}% confidence)'.format(

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -28,9 +28,14 @@
 sys.path.append('../vitallens-python')
 
 from vitallens.ssd import FaceDetector
+from vitallens.utils import download_file
 
+TEST_VIDEO_URL = "https://github.com/Rouast-Labs/vitallens-python/raw/main/examples/sample_video_2.mp4"
 TEST_VIDEO_PATH = "examples/sample_video_2.mp4"
 
+# Download the test video before running any tests
+download_file(TEST_VIDEO_URL, TEST_VIDEO_PATH)
+
 @pytest.fixture(scope='session')
 def test_video_path():
   return TEST_VIDEO_PATH

diff --git a/tests/test_client.py b/tests/test_client.py
@@ -18,7 +18,9 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import json
 import numpy as np
+import os
 import pytest
 
 import sys
@@ -29,29 +31,42 @@
 @pytest.mark.parametrize("method", [Method.G, Method.CHROM, Method.POS])
 @pytest.mark.parametrize("detect_faces", [True, False])
 @pytest.mark.parametrize("file", [True, False])
-def test_VitalLens(request, method, detect_faces, file):
-  vl = VitalLens(method=method, detect_faces=detect_faces)
+@pytest.mark.parametrize("export", [True, False])
+def test_VitalLens(request, method, detect_faces, file, export):
+  vl = VitalLens(method=method, detect_faces=detect_faces, export_to_json=export)
   if file:
     test_video_path = request.getfixturevalue('test_video_path')
-    result = vl(test_video_path, faces = None if detect_faces else [247, 57, 440, 334])
+    result = vl(test_video_path, faces = None if detect_faces else [247, 57, 440, 334], export_filename="test")
   else:
     test_video_ndarray = request.getfixturevalue('test_video_ndarray')
     test_video_fps = request.getfixturevalue('test_video_fps')
-    result = vl(test_video_ndarray, fps=test_video_fps, faces = None if detect_faces else [247, 57, 440, 334])
+    result = vl(test_video_ndarray, fps=test_video_fps, faces = None if detect_faces else [247, 57, 440, 334], export_filename="test")
   assert len(result) == 1
   assert result[0]['face']['coordinates'].shape == (360, 4)
   assert result[0]['face']['confidence'].shape == (360,)
   assert result[0]['vital_signs']['ppg_waveform']['data'].shape == (360,)
   assert result[0]['vital_signs']['ppg_waveform']['confidence'].shape == (360,)
   np.testing.assert_allclose(result[0]['vital_signs']['heart_rate']['value'], 60, atol=10)
   assert result[0]['vital_signs']['heart_rate']['confidence'] == 1.0
+  if export:
+    test_json_path = os.path.join("test.json")
+    assert os.path.exists(test_json_path)
+    with open(test_json_path, 'r') as f:
+      data = json.load(f)
+    assert np.asarray(data[0]['face']['coordinates']).shape == (360, 4)
+    assert np.asarray(data[0]['face']['confidence']).shape == (360,)
+    assert np.asarray(data[0]['vital_signs']['ppg_waveform']['data']).shape == (360,)
+    assert np.asarray(data[0]['vital_signs']['ppg_waveform']['confidence']).shape == (360,)
+    np.testing.assert_allclose(data[0]['vital_signs']['heart_rate']['value'], 60, atol=10)
+    assert data[0]['vital_signs']['heart_rate']['confidence'] == 1.0
+    os.remove(test_json_path)
 
 def test_VitalLens_API(request):
   api_key = request.getfixturevalue('test_dev_api_key')
-  vl = VitalLens(method=Method.VITALLENS, api_key=api_key, detect_faces=True)
+  vl = VitalLens(method=Method.VITALLENS, api_key=api_key, detect_faces=True, export_to_json=False)
   test_video_ndarray = request.getfixturevalue('test_video_ndarray')
   test_video_fps = request.getfixturevalue('test_video_fps')
-  result = vl(test_video_ndarray, fps=test_video_fps, faces=None)
+  result = vl(test_video_ndarray, fps=test_video_fps, faces=None, export_filename="test")
   assert len(result) == 1
   assert result[0]['face']['coordinates'].shape == (360, 4)
   assert result[0]['vital_signs']['ppg_waveform']['data'].shape == (360,)
@@ -62,3 +77,4 @@ def test_VitalLens_API(request):
   np.testing.assert_allclose(result[0]['vital_signs']['heart_rate']['confidence'], 1.0, atol=0.1)
   np.testing.assert_allclose(result[0]['vital_signs']['respiratory_rate']['value'], 13.5, atol=0.5)
   np.testing.assert_allclose(result[0]['vital_signs']['respiratory_rate']['confidence'], 1.0, atol=0.1)
+  assert not os.path.exists("test.json")
diff --git a/tests/test_signal.py b/tests/test_signal.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2024 Philipp Rouast
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import numpy as np
+import pytest
+
+import sys
+sys.path.append('../vitallens-python')
+
+from vitallens.signal import windowed_mean, windowed_freq, reassemble_from_windows
+
+def test_windowed_mean():
+  x = np.asarray([0., 1., 2., 3., 4., 5., 6.])
+  y = np.asarray([1., 1., 2., 3., 4., 5., 5.])
+  out_y = windowed_mean(x=x, window_size=3, overlap=1)
+  np.testing.assert_equal(
+    out_y,
+    y)
+
+@pytest.mark.parametrize("num", [100, 1000])
+@pytest.mark.parametrize("freq", [2.35, 4.89, 13.55])
+@pytest.mark.parametrize("window_size", [10, 20])
+def test_estimate_freq_periodogram(num, freq, window_size):
+  # Test data
+  x = np.linspace(0, freq * 2 * np.pi, num=num)
+  np.random.seed(0)
+  y = 100 * np.sin(x) + np.random.normal(scale=8, size=num)
+  # Check a default use case with axis=-1
+  np.testing.assert_allclose(
+    windowed_freq(x=y, window_size=window_size, overlap=window_size//2, f_s=len(x), f_range=(max(freq-2,1),freq+2), f_res=0.05),
+    np.full((num,), fill_value=freq),
+    rtol=1)
+
+def test_reassemble_from_windows():
+  x = np.array([[[2.0, 4.0, 6.0, 8.0, 10.0], [7.0, 1.0, 10.0, 12.0, 18.0]],
+                [[2.0, 3.0, 4.0, 5.0, 6.0], [7.0, 8.0, 9.0, 10.0, 11.0]]], dtype=np.float32).transpose(1, 0, 2)
+  idxs = np.array([[1, 3, 5, 7, 9], [5, 6, 9, 11, 13]], dtype=np.int64)
+  out_x, out_idxs = reassemble_from_windows(x=x, idxs=idxs)
+  np.testing.assert_equal(
+    out_x,
+    np.asarray([[2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 18.0],
+                [2.0, 3.0, 4.0, 5.0, 6.0, 10.0, 11.0]]))
+  np.testing.assert_equal(
+    out_idxs,
+    np.asarray([1, 3, 5, 7, 9, 11, 13]))
+