Skip to content

Commit

Permalink
Merge pull request #139 from esi-neuroscience/acme_amputation
Browse files Browse the repository at this point in the history
Acme amputation. Unanesthetized.
  • Loading branch information
pantaray authored Oct 27, 2021
2 parents 7aa9187 + 67b7f48 commit 1f5972f
Show file tree
Hide file tree
Showing 40 changed files with 1,699 additions and 1,546 deletions.
5 changes: 3 additions & 2 deletions .github/ISSUE_TEMPLATE/syncopy-feature-request.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,18 @@ about: Suggest an idea for Syncopy
title: ''
labels: ''
assignees: ''
labels: 'Feature Request'

---

**Is your feature request related to a problem? Please elaborate.**
Please provide a clear and concise description of the problem. If possible/applicable, please consider providing a minimal complete working example illustrating the problem.
Please provide a clear and concise description of the problem. If possible/applicable, please consider providing a minimal complete working example illustrating the problem.

**Do you have a solution in mind? Please elaborate**
Please provide a clear and concise description of what you want to happen.

**Not really a problem per se, more like a missing feature? Please elaborate**
Please provide a description of what kind of functionality you want to see in Syncopy. If possible/applicable, please consider writing a few lines of (pseudo-)code to illustrate your thoughts.
Please provide a description of what kind of functionality you want to see in Syncopy. If possible/applicable, please consider writing a few lines of (pseudo-)code to illustrate your thoughts.

**Additional context**
Please add any other context about the feature request here.
Expand Down
99 changes: 62 additions & 37 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,46 +1,87 @@
stages:
- test
- tox
- slurm
- upload
- deploy

clusterTests:
stage: test
intellinux:
stage: tox
allow_failure: true
tags:
- slurm
only:
- master
- dev
variables:
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose --ignore=syncopy/acme"
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose"
GIT_FETCH_EXTRA_FLAGS: --tags
NO_SLURM: "true"
script:
- source $HOME/miniconda3/etc/profile.d/conda.sh
- conda update --yes conda
- source /opt/conda/etc/profile.d/conda.sh
- conda env update -f syncopy.yml --prune
- conda activate syncopy
- srun -p DEV --mem=8000m -c 4 tox -r
- tox

windowsTests:
stage: test
powerlinux:
stage: tox
allow_failure: true
tags:
- power
only:
- master
- dev
variables:
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose"
GIT_FETCH_EXTRA_FLAGS: --tags
script:
- source /opt/conda/etc/profile.d/conda.sh
- conda env update -f syncopy.yml --prune
- conda activate syncopy
- tox -p 0

intelwin:
stage: tox
allow_failure: true
tags:
- windows10
only:
- master
- dev
variables:
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose --ignore=syncopy/acme"
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose"
GIT_FETCH_EXTRA_FLAGS: --tags
script:
- conda env update -f syncopy.yml --prune
- conda.bat activate syncopy
- tox

slurmtest:
stage: slurm
allow_failure: true
tags:
- slurm
only:
- master
- dev
variables:
PYTEST_ADDOPTS: "--color=yes --tb=short --verbose"
GIT_FETCH_EXTRA_FLAGS: --tags
script:
- python -m pip install --upgrade --user pip
- python -m pip install --upgrade --user tox
- tox -r
- source /opt/conda/etc/profile.d/conda.sh
- conda env update -f syncopy.yml --prune
- conda activate syncopy
- export PYTHONPATH=$CI_PROJECT_DIR
- srun -p DEV --mem=8000m -c 4 pytest

deploy-to-testpypi:
pypitest:
stage: upload
only:
- master
- tags
tags:
- deploy
variables:
GIT_FETCH_EXTRA_FLAGS: --tags
script:
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda update --yes conda
Expand All @@ -51,23 +92,24 @@ deploy-to-testpypi:
- python setup.py sdist bdist_wheel
- tarname="$(basename -- $(ls dist/*.tar.gz) .tar.gz)"
- version=$(sed -e 's/esi-syncopy-\(.*\)/\1/' <<< "$tarname")
- twine upload --repository testpypi dist/*
- twine upload --repository testpypi --config-file=~/.esipypirc dist/*
- conda create --yes --name piptest python=3.8
- conda activate piptest
- conda install --yes pip
- pip --no-cache-dir install --extra-index-url https://testpypi.python.org/pypi esi-syncopy==$version
- pip --no-cache-dir install --extra-index-url https://test.pypi.org/simple esi-syncopy==$version
- python -c "import syncopy as spy"
- conda deactivate
- conda remove --yes --name piptest --all

deploy-pypi:
when: manual
pypideploy:
stage: deploy
when: manual
only:
- master
- tags
tags:
- deploy
variables:
GIT_FETCH_EXTRA_FLAGS: --tags
script:
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda update --yes conda
Expand All @@ -76,21 +118,4 @@ deploy-pypi:
- conda install --yes twine keyring rfc3986
- conda update --yes twine keyring rfc3986
- python setup.py sdist bdist_wheel
- twine upload --cert /etc/ssl/certs/ca-certificates.crt dist/*

docs:
stage: deploy
tags:
- docs
only:
- dev
- master
- tags
script:
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda update --yes conda
- conda env update -f syncopy.yml --prune
- conda activate syncopy
- cd doc
- make html
- rsync -av -e "ssh" --delete build/html/* root@monitor:/var/www/html/syncopy
- twine upload --config-file=~/.esipypirc dist/*
4 changes: 0 additions & 4 deletions .gitmodules

This file was deleted.

2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ install:
- python setup.py -q install
# command to run tests
script:
- pytest -v --ignore=syncopy/acme
- pytest -v
161 changes: 161 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
# Changelog of SyNCoPy
All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### NEW
- Made SyNCoPy PEP 517 compliant: added pyproject.toml and modified setup.py
accordingly
- Added IBM POWER testing pipeline (via dedicated GitLab Runner)
- Added Superlet spectral estimation method

### CHANGED

### REMOVED
- Retired tox in `slurmtest` CI pipeline in favor of a "simple" pytest testing
session due to file-locking problems of tox environments on NFS mounts

### DEPRECATED
- Removed ACME from source repository: the submodule setup proved to be too
unreliable and hard to maintain. ACME is now an optional (but recommended)
dependency of SyNCoPy

### FIXED
- Stream-lined GitLab Runner setup: use cluster-wide conda instead of local
installations (that differ slightly across runners) and leverage `tox-conda`
to fetch pre-built dependencies

## [v0.1b2] - 2020-01-15
Housekeeping and maintenance release

### NEW
- Included ACME as SyNCoPy submodule: all ESI-HPC cluster specific code has
been migrated to the new ACME package, see https://github.com/esi-neuroscience/acme
- Better late than never: added this CHANGELOG file

### CHANGED
- Modified GitLab CI Pipeline Setup + version handling: use `setuptools_scm`
to populate `spy.__version__` instead of hard-coding a version string
in the package `__init__.py`; this makes test-uploads to PyPI-Test infinitely
easier since `setuptools_scm` takes care of generating non-conflicting
package versions.
- Modified packaging setup and adapted modular layout to account for new
submodule ACME

### REMOVED
- Deleted ESI-specific `dask_helpers.py` module (migrated to ACME)

### DEPRECATED
- Cleaned up dependencies: removed all `jupyter`-packages from depencency
list to not cause (unnecessary) conflicts in existing Python environments

### FIXED
- Repaired CI pipelines
- Repaired h5py version mismatch: pin SyNCoPy to `hypy` versions greater than
2.9 but less than 3.x
- Pin SyNCoPy to Python 3.8.x (Python 3.9 currently triggers too many dependency
conflicts)

## [v0.1b1] - 2020-10-23
First public pre-release of SyNCoPy on PyPI and GitHub.

### NEW
- Included `selectdata` as a `computeFunction` that uses the parallelization
framework in `ComputationalRoutine` to perform arbitrary data-selection tasks
(including but not limited to unordered lists, repetitions and removals).
- Included time-frequency analysis routines `mtmconvol` and `wavelet`
- Added plotting functionality: functions `singlepanelplot` and `multiplanelplot`
allow quick visual inspection of `AnalogData` and `SpectralData` objects
- Added support to process multiple SyNCoPy objects in a single meta-function
call (all decorators have been modified accordingly)
- Introduced standardized warning messages via new class `SPYWarning`
- Included (more or less) extensive developer docs
- Added Travis CI and included badges on GitHub landing page
- New convenience scripts to ease developing/testing
- New conda.yml file + script for consolidating conda/pip requirements: all
of SyNCoPy's dependencies are now collected in `syncopy.yml`, the respective
pip-specific requirements.txt and requirements-test.txt files are generated
on the fly by a new function `conda2pip` that relies on ruamel.yaml (new
required dependency for building SyNCoPy)
- New GitLab CI directive for uploading SyNCoPy to PyPI
- Included GitHub templates for new issues/pull requests
- SyNCoPy docu is now hosted on readthedocs (re-directed from syncopy.org)
- New logo + icon

### CHANGED
- Made `cluster_cleanup` more robust (works with `LocalCluster` objects now)
- Made data-parser more feature-rich: check for emptiness, parse non-data
datasets etc.
- Made `generate_artificial_data` more robust: change usage of random number
seed to allow persistent comparisons across testing runs
- Updated CI dependencies (SyNCoPy now requires NumPy 1.18 and Matplotlib 3.3.x)

### REMOVED
- All *.py-file headers have been removed
- Removed examples sub-module from main package (examples will be part of a
separate repo)

### DEPRECATED
- Wiped all hand-crafted array-matching routines; use `best_match` instead
- Do not use `pbr` in the build system any more; rely instead on up-to-date
setuptools functionality
- Retired memory map support and raw binary data reading routines

### FIXED
- Improved temporary storage handling so that dask workers that import the
package do not repeat all temp-dir I/O tests (and potentially run into
dead-locks or race conditions)

## [v0.1a1] - 2019-10-14
Preview alpha release of SyNCoPy for first ESI-internal tryout workshop.

### NEW
- Added routines `esi_cluster_setup` and `cluster_cleanup` to facilitate
using SLURM from within SyNCoPy
- Included new `FauxTrial` class and `_preview_trial` class methods to
permit quick and performant compute dry-runs
- Included a `select` keyword to allow for in-place selections that are applied
on the fly in any meta-function via a new decorator. The heavy lifting is performed
by a new `Selector` class
- Re-worked the `specest` package: `mtmfft` is now fully functional
- Overhauled HTML documentation

### CHANGED
- New layout of SyNCoPy objects on disk: introduction of Spy-containers supporting
multiple datasets/objects within the same folder
- First working implementation of `spy.load` and `spy.save`
- Use dask bags instead of arrays in parallelization engine to permit more
flexible distribution of data across workers
- Re-worked `trialdefinition` mechanics: attach the full `trialdefinition` array
to objects and fetch relevant information on the fly: `BaseData._trialdefinition`
unifies `sampleinfo`, `t0` and `trialinfo` and calls `definetrial`

### REMOVED
- Removed `dimlabels` property

### DEPRECATED
- Retired Dask arrays in `ComputationalRoutine`; use dask bags instead

### FIXED
- Flipped sign of offsets in `trialdefinition` to be compatible w/FieldTrip
- Enforced PEP8 compliance
- Cleaned up constructor of `BaseData` to prohibit accessing uninitialized attributes

## [v0.1a0] - 2019-07-20
Internal pre-alpha release of SyNCoPy. Prototypes of data format, user-interface
and parallelization framework are in place.

### NEW
- Class structure is laid out, meta-functions are present but mostly place-holders
- Support FieldTrip-style calling syntax via `cfg` "structures" (the keys of which are
"unwrapped" by a corresponding decorator)
- Preliminary I/O capabilities implemented, objects can be written/read
to/from HDF5
- First prototype of parallelization framework based on Dask
- Custom traceback that is enabeld as soon as SyNCoPy is imported: do not
spill hundreds of lines to STDOUT, instead highlight most probable cause
of error and explain how to get to full traceback if wanted
- Basic session management to ensure concurrent SyNCoPy sessions only access
their own data
14 changes: 12 additions & 2 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ message: 'If you use this software, please cite it based on metadata found in th
Invasive Electrophysiological Data. Computational Intelligence and Neuroscience,
vol. 2011, Article ID 156869, 9 pages, 2011. doi:10.1155/2011/156869.'
authors:
- affiliation: Ernst Strüngmann Institute for Neuroscience in Cooperation with Max
Planck Society
family-names: Mönke
given-names: Gregor
orcid: https://orcid.org/0000-0002-3521-715X
- affiliation: Ernst Strüngmann Institute for Neuroscience in Cooperation with Max
Planck Society
family-names: Fuertinger
Expand All @@ -17,6 +22,11 @@ authors:
family-names: Schmiedt
given-names: Joscha
orcid: https://orcid.org/0000-0001-6233-1866
- affiliation: Ernst Strüngmann Institute for Neuroscience in Cooperation with Max
Planck Society
family-names: Fries
given-names: Pascal
orcid: https://orcid.org/0000-0002-4270-1468
license: BSD-3-Clause
keywords:
- large-scale electrophysiology
Expand All @@ -27,5 +37,5 @@ keywords:
- spectral-methods
- brain
repository-code: https://github.com/esi-neuroscience/syncopy
version: 0.1b2.dev74+gc436466.d20210806
date-released: '2021-08-06'
version: 0.1b2.dev125
date-released: '2021-10-20'
3 changes: 1 addition & 2 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
exclude .gitignore
exclude .gitlab-ci.yml
exclude .gitmodules
exclude MANIFEST.in
exclude .readthedocs.yml
exclude .travis.yml
recursive-exclude .github *
recursive-exclude doc *
recursive-include syncopy *.py
Loading

0 comments on commit 1f5972f

Please sign in to comment.