Merge pull request #255 from gipert/refactor

Address some testing and packaging issues

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "actions/*"

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
         python -m pip install .
     - name: Run unit tests
       run: |
-        pytest tests
+        pytest
 
   # test-in-legend-container:
   #   name: Test pygama in LEGEND container
@@ -48,7 +48,28 @@ jobs:
   #   - name: Install and run unit tests
   #     run: |
   #       python -m pip install .
-  #       pytest tests
+  #       pytest
+
+  test-coverage:
+    name: Calculate and upload test coverage
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Generate Report
+        run: |
+          .github/workflows/get-dependencies.sh
+          python -m pip install .
+          python -m pip install coverage
+          coverage run -m pytest
+      - name: Upload Coverage to codecov.io
+        uses: codecov/codecov-action@v2
 
   deploy-docs:
     name: Deploy documentation on legend-exp.github.io/pygama

.pre-commit-config.yaml

@@ -0,0 +1,82 @@
+ci:
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+
+exclude: ^attic
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: "v4.2.0"
+  hooks:
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: check-executables-have-shebangs
+  - id: check-merge-conflict
+  - id: check-symlinks
+  - id: check-yaml
+  - id: check-json
+  - id: check-toml
+  - id: check-docstring-first
+  - id: debug-statements
+  - id: end-of-file-fixer
+  - id: forbid-new-submodules
+  - id: mixed-line-ending
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/asottile/setup-cfg-fmt
+  rev: "v1.20.1"
+  hooks:
+  - id: setup-cfg-fmt
+
+- repo: https://github.com/PyCQA/isort
+  rev: "5.10.1"
+  hooks:
+  - id: isort
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: "v2.32.0"
+  hooks:
+  - id: pyupgrade
+    args: ["--py36-plus"]
+
+# - repo: https://github.com/psf/black
+#   rev: "22.3.0"
+#   hooks:
+#   - id: black
+
+# - repo: https://github.com/PyCQA/flake8
+#   rev: "4.0.1"
+#   hooks:
+#   - id: flake8
+#     additional_dependencies: [flake8-bugbear, flake8-print]
+
+- repo: https://github.com/kynan/nbstripout
+  rev: "0.5.0"
+  hooks:
+    - id: nbstripout
+      args: ["--strip-empty-cells",
+             "--extra-keys", "metadata.kernelspec metadata.language_info"]
+
+- repo: https://github.com/mgedmin/check-manifest
+  rev: "0.48"
+  hooks:
+  - id: check-manifest
+    stages: [manual]
+
+- repo: https://github.com/codespell-project/codespell
+  rev: "v2.1.0"
+  hooks:
+  - id: codespell
+    args: ["-L", "hist,gaus,nd,ans,crate,nin,puls,spms"]
+
+- repo: https://github.com/shellcheck-py/shellcheck-py
+  rev: "v0.8.0.4"
+  hooks:
+  - id: shellcheck
+
+- repo: https://github.com/pre-commit/pygrep-hooks
+  rev: "v1.9.0"
+  hooks:
+  - id: rst-backticks
+  - id: rst-directive-colons
+  - id: rst-inline-touching-normal

LICENSE

@@ -198,4 +198,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

README.md

@@ -1,9 +1,9 @@
 # pygama
 
 ![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/legend-exp/pygama?logo=git)
-![Documentation (main)](https://img.shields.io/badge/documentation-online-purple?logo=readthedocs&link=https%3A%2F%2Flegend-exp.github.io%2Fpygama)
-![GitHub Workflow Status (main)](https://img.shields.io/github/workflow/status/legend-exp/pygama/pygama/main?label=main%20branch&logo=github)
-![GitHub Workflow Status (dev)](https://img.shields.io/github/workflow/status/legend-exp/pygama/pygama/dev?label=dev%20branch&logo=github)
+![Documentation](https://img.shields.io/badge/documentation-online-purple?logo=readthedocs&link=https%3A%2F%2Flegend-exp.github.io%2Fpygama)
+![GitHub Workflow Status](https://img.shields.io/github/workflow/status/legend-exp/pygama/pygama/main?label=main%20branch&logo=github)
+![Codecov](https://img.shields.io/codecov/c/github/legend-exp/pygama?logo=codecov)
 ![GitHub issues](https://img.shields.io/github/issues/legend-exp/pygama?logo=github)
 ![GitHub pull requests](https://img.shields.io/github/issues-pr/legend-exp/pygama?logo=github)
 ![License](https://img.shields.io/github/license/legend-exp/pygama)
@@ -15,3 +15,4 @@
 * optimizing DSP routines and tuning associated analysis parameters
 * generating and selecting high-level event data for further analysis
 
+Check out the [online documentation](https://legend-exp.github.io/pygama).

codecov.yml

@@ -0,0 +1,16 @@
+codecov:
+  require_ci_to_pass: true
+
+coverage:
+  status:
+    project:
+      default:
+        target: 20%
+    patch: false
+
+github_checks:
+    annotations: false
+
+ignore:
+  - tests
+  - docs

docs/source/contributing.rst

@@ -8,7 +8,23 @@ that do not comply to the following directives will be rejected.
 Code style
 ----------
 
-*In progress...*
+A set of `pre-commit <https://pre-commit.com>`_ hooks is configured to make
+sure that *pygama* coherently follows standard coding style conventions. The
+pre-commit tool is able to identify common style problems and automatically fix
+them, wherever possible. Configured hooks are listed in the
+``.pre-commit-config.yaml`` file at the project root folder. They are run
+remotely on the GitHub repository through the `pre-commit bot
+<https://pre-commit.ci>`_, but can also be run locally before submitting a
+pull request (recommended):
+
+.. code-block:: console
+
+   $ pip install pre-commit      # required
+   $ pre-commit run --all-files  # analyse the source code and fix it wherever possible
+   $ pre-commit install          # install a Git pre-commit hook (optional)
+
+For a more comprehensive guide, check out the `Scikit-HEP documentation about
+code style <https://scikit-hep.org/developer/style>`_.
 
 Testing
 -------
@@ -22,11 +38,11 @@ Testing
   overview.
 * *pygama* tests belong to three categories:
 
-  :micro-tests: Should ensure the correct behaviour of each function
+  :unit tests: Should ensure the correct behaviour of each function
       independently, possibly without relying on other *pygama* methods. The
-      existence of these microscopic tests makes it possible to promptly
-      identify and fix the source of a bug. An example of this are tests for
-      each single DSP processor
+      existence of these micro-tests makes it possible to promptly identify and
+      fix the source of a bug. An example of this are tests for each single DSP
+      processor
 
   :integration tests: Should ensure that independent parts of the code base
       work well together and are integrated in a cohesive framework. An example
@@ -40,10 +56,19 @@ Testing
 * Unit tests are automatically run for every push event and pull request to the
   remote Git repository on a remote server (currently handled by GitHub
   actions). Every pull request must pass all tests before being approved for
-  merging
+  merging.
 * Additionally, pull request authors are required to provide tests with
   sufficient code coverage for every proposed change or addition. If necessary,
-  high-level functional tests should be updated.
+  high-level functional tests should be updated. We currently rely on
+  `codecov.io <https://app.codecov.io/gh/legend-exp/pygama>`_ to keep track of
+  the test coverage. To generate a local coverage report (recommended before
+  submitting pull requests), the ``coverage`` Python package is needed:
+
+  .. code-block:: console
+
+    $ pip install coverage
+    $ coverage run -m pytest
+    $ coverage report
 
 Documentation
 -------------

pygama/dsp/_processors/Wiener_filter.py

@@ -1,13 +1,14 @@
 import numpy as np
-from pygama import lh5
 from numba import guvectorize
+
+from pygama import lh5
 from pygama.dsp.errors import DSPFatal
 
 
 def Wiener_filter(file_name_array):
     """
     Apply a Wiener filter to the waveform.  Note that this convolution is performed in the frequency domain
-    
+
     Initialization Parameters
     -------------------------
     file_name_array : string
@@ -19,8 +20,8 @@ def Wiener_filter(file_name_array):
     fft_w_in : array-like
            The fourier transform input waveform
     fft_w_out: array-like
-           The filtered waveform, in the frequency domain     
-           
+           The filtered waveform, in the frequency domain
+
     Processing Chain Example
     ------------------------
     "wf_wiener": {
@@ -32,69 +33,69 @@ def Wiener_filter(file_name_array):
         "init_args": ["/path/to/file/wiener.lh5"]
     }
     """
-    
+
     sto = lh5.Store()
 
     # Check that the file is valid and the data is in the correct format
 
     try: file_name_array[0]
     except: raise DSPFatal('init_args must be an array with the filename')
-    
+
     file_name = file_name_array[0]
-     
+
     try: f = sto.gimme_file(file_name, 'r')
-    except: raise DSPFatal('File must be a valid lh5 file') 
+    except: raise DSPFatal('File must be a valid lh5 file')
 
     if 'spms/processed/superpulse' not in f:
         raise DSPFatal('lh5 file must have \'spms/processed/superpulse\' as a group')
-        
+
     if 'spms/processed/noise_wf' not in f:
         raise DSPFatal('lh5 file must have \'spms/processed/noise_wf\' as a group')
-        
-    # Read in the data 
-    
+
+    # Read in the data
+
     superpulse, _ = sto.read_object('spms/processed/superpulse', file_name)
     superpulse = superpulse.nda
-    
+
     noise_wf, _ = sto.read_object('spms/processed/noise_wf', file_name)
     noise_wf = noise_wf.nda
- 
+
     # Now check that the data are valid
-    
+
     if len(superpulse) <= 0:
         raise DSPFatal('The length of the filter must be positive')
-        
-    if len(superpulse) != len(noise_wf): 
+
+    if len(superpulse) != len(noise_wf):
         raise DSPFatal('The length of the superpulse must be equal to the length of the noise waveform')
 
     if np.argmax(superpulse) <= 0 or np.argmax(superpulse) > len(superpulse):
         raise DSPFatal('The index of the maximum of the superpulse must occur within the waveform')
-    
-    # Transform these to the frequency domain to eventually create the Wiener filter 
-    
+
+    # Transform these to the frequency domain to eventually create the Wiener filter
+
     fft_superpulse = np.fft.fft(superpulse)
     fft_noise_wf = np.fft.fft(noise_wf)
-    
-    # Create the point spread function for the detector's response 
-    
-    def PSF(superpulse, fft_superpulse):  
-        
+
+    # Create the point spread function for the detector's response
+
+    def PSF(superpulse, fft_superpulse):
+
         delta = np.zeros_like(superpulse)
         arg_max = np.argmax(superpulse)
         delta[arg_max] = np.amax(superpulse)
 
         return fft_superpulse/np.fft.fft(delta)
-    
-    # Now create the Wiener filter in the frequency domain 
-    
+
+    # Now create the Wiener filter in the frequency domain
+
     fft_PSF = PSF(superpulse, fft_superpulse)
     PSD_noise_wf = fft_noise_wf*np.conj(fft_noise_wf)
     PSD_superpulse = fft_superpulse*np.conj(fft_superpulse)
 
-    w_filter = (np.conj(fft_PSF))/((fft_PSF*np.conj(fft_PSF))+(PSD_noise_wf/PSD_superpulse)) 
-    
-    # Create a factory function that performs the convolution with the Wiener filter, the output is still in the frequency domain 
-    
+    w_filter = (np.conj(fft_PSF))/((fft_PSF*np.conj(fft_PSF))+(PSD_noise_wf/PSD_superpulse))
+
+    # Create a factory function that performs the convolution with the Wiener filter, the output is still in the frequency domain
+
     @guvectorize(["void(complex64[:], complex64[:])",
                   "void(complex128[:], complex128[:])"],
                  "(n)->(n)", forceobj=True)
@@ -109,4 +110,4 @@ def Wiener_out(fft_w_in, fft_w_out):
 
         fft_w_out[:] = fft_w_in * w_filter
 
-    return Wiener_out
+    return Wiener_out

pygama/dsp/_processors/bl_subtract.py

@@ -1,7 +1,9 @@
 import numpy as np
 from numba import guvectorize
+
 from pygama.dsp.errors import DSPFatal
 
+
 @guvectorize(["void(float32[:], float32, float32[:])",
               "void(float64[:], float64, float64[:])"],
              "(n),()->(n)", nopython=True, cache=True)