From dffb4b78d54fb1cca3b610dce0d58f179d425e10 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 25 Aug 2021 16:10:00 +0200 Subject: [PATCH 001/109] WIP: Cross spectral density + coherence - initial backend commit Changes to be committed: new file: csd.py --- syncopy/connectivity/csd.py | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 syncopy/connectivity/csd.py diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py new file mode 100644 index 000000000..ec4f4735a --- /dev/null +++ b/syncopy/connectivity/csd.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Cross Spectral Densities and Coherency +# + +# Builtin/3rd party package imports +import numpy as np +from scipy import signal +import itertools + +# syncopy imports +from syncopy.specest.mtmfft import mtmfft + + +def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): + + """ + Cross spectral density (CSD) estimate between all channels + of the input data. First all the individual Fourier transforms + are calculated via a (multi-)tapered FFT, then the pairwise + coherence is calculated. Averaging over tapers is done implicitly. + Output consists of all (nChannels x nChannels-1)/2 different CSD estimates + aranged in a symmetric fashion (CSD_ij == CSD_ji). The elements on the + main diagonal (CSD_ii) are the auto-spectra. + + If normalization is required (`norm=True`) the respective coherencies + are returned. + + See also + -------- + mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` + (Multi-)tapered Fourier analysis + + """ + + nSamples, nChannels = data_arr.shape + + # has shape (nTapers x nFreq x nChannels) + specs, freqs = mtmfft(data_arr, samplerate, taper, taperopt) + + # has shape (nChannels x nChannels x nFreq) + output = np.zeros((nChannels, nChannels, freqs.size)) + + for i in range(nChannels): + for j in range(i, nChannels): + output[i, j, :] = np.real(specs[0, :, i] * specs[0, :, j].conj()) + output[j, i, :] = output[i, j, :] + + # there is probably a more efficient way + if norm: + for i in range(nChannels): + for j in range(i, nChannels): + output[i, j, :] = output[i, j, :] / np.sqrt( + output[i, i, :] * output[j, j, :] + ) + + return output, freqs From fdb4a23c53b3e93859686252a040be96c669e3be Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 26 Aug 2021 13:38:37 +0200 Subject: [PATCH 002/109] WIP: CSD - taper averaging Changes to be committed: modified: syncopy/connectivity/csd.py --- syncopy/connectivity/csd.py | 42 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index ec4f4735a..ce2e675dd 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -19,7 +19,7 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): of the input data. First all the individual Fourier transforms are calculated via a (multi-)tapered FFT, then the pairwise coherence is calculated. Averaging over tapers is done implicitly. - Output consists of all (nChannels x nChannels-1)/2 different CSD estimates + Output consists of all (nChannels x nChannels+1)/2 different CSD estimates aranged in a symmetric fashion (CSD_ij == CSD_ji). The elements on the main diagonal (CSD_ii) are the auto-spectra. @@ -41,17 +41,35 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): # has shape (nChannels x nChannels x nFreq) output = np.zeros((nChannels, nChannels, freqs.size)) - for i in range(nChannels): - for j in range(i, nChannels): - output[i, j, :] = np.real(specs[0, :, i] * specs[0, :, j].conj()) - output[j, i, :] = output[i, j, :] - - # there is probably a more efficient way + # somewhat vectorized - not really fast :/ if norm: + # main diagonal: auto spectrum for each taper and averaging + diag = np.multiply(specs, specs.conj()).mean(axis=0) + # output[range(nChannels), range(nChannels), :] = np.real(diag.T) + diag = np.real(diag).T + for i in range(nChannels): - for j in range(i, nChannels): - output[i, j, :] = output[i, j, :] / np.sqrt( - output[i, i, :] * output[j, j, :] - ) + idx = slice(i, nChannels) + row = np.multiply(specs[..., np.tile(i, nChannels - i)], + specs.conj()[..., idx]) + + # normalization + denom = np.multiply(np.tile(diag[i], ((nChannels - i), 1)), diag[i:]) + row = row.mean(axis=0).T / np.sqrt(denom) + output[i, i:, ...] = np.real(row) + + else: + for i in range(nChannels): + idx = slice(i, nChannels) + row = np.multiply(specs[..., np.tile(i, nChannels - i)], specs.conj()[..., idx]) + output[i, i:, ...] = np.real(row.mean(axis=0).T) + + return output # , freqs + - return output, freqs +# dummy input +a = np.ones((10, 3)) * np.arange(1,4) +# dummy mtmfft result +b = np.arange(1, 4) * np.ones((2,10,3)).astype('complex') +# dummt csd matrix +c = np.ones((5, 5, 10)) From ec3ca97ccc6765a98b12919ed650b01b4801b74b Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 30 Aug 2021 20:27:56 +0200 Subject: [PATCH 003/109] WIP: Vectorization of cross spectral density - so far nothing is faster than naive looping :/ Changes to be committed: modified: syncopy/connectivity/csd.py --- syncopy/connectivity/csd.py | 42 +++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index ce2e675dd..c41f8002f 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -12,7 +12,7 @@ from syncopy.specest.mtmfft import mtmfft -def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): +def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False, naive=True): """ Cross spectral density (CSD) estimate between all channels @@ -40,7 +40,7 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): # has shape (nChannels x nChannels x nFreq) output = np.zeros((nChannels, nChannels, freqs.size)) - + # somewhat vectorized - not really fast :/ if norm: # main diagonal: auto spectrum for each taper and averaging @@ -58,18 +58,42 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): row = row.mean(axis=0).T / np.sqrt(denom) output[i, i:, ...] = np.real(row) - else: + elif naive: for i in range(nChannels): - idx = slice(i, nChannels) - row = np.multiply(specs[..., np.tile(i, nChannels - i)], specs.conj()[..., idx]) - output[i, i:, ...] = np.real(row.mean(axis=0).T) - - return output # , freqs + for j in range(i, nChannels): + output[i, j] = np.real(specs[:, :, i] * specs[:, :, j].conj()).mean(axis=0) + output[j, i] = output[i, j] + + else: + # build single taper array + chan_idx = np.arange(nChannels) + idx1, idx2 = np.meshgrid(chan_idx, chan_idx) + output = np.real(specs[:, :, idx1] * specs[:, :, idx2].conj()).mean(axis=0).transpose(2,1,0) + return output # dummy input a = np.ones((10, 3)) * np.arange(1,4) +abig = np.ones((100, 50)) * np.arange(1,51) # dummy mtmfft result -b = np.arange(1, 4) * np.ones((2,10,3)).astype('complex') +# b = np.arange(1, 4) * np.ones((2,10,3)).astype('complex') # dummt csd matrix c = np.ones((5, 5, 10)) + +def vectorized(arr): + nSamples, nChannels = arr.shape + r = np.multiply.outer(arr,arr.T).reshape(nSamples, nChannels**2 ,nSamples).diagonal(axis1=0, axis2=2) + + return r.reshape((nChannels, nChannels, nSamples)) + +def arr_loop(arr): + nChannels = arr.shape[1] + output = np.zeros((nChannels, nChannels, arr.shape[0])) + + for i in range(nChannels): + for j in range(i, nChannels): + output[i, j] = np.real(arr[:, i] * arr[:, j]) + output[j, i] = output[i, j] + + return output + From 5c0e6993b3ae9c8ac91af5aa4aa1ec4b985bf186 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 30 Aug 2021 21:11:37 +0200 Subject: [PATCH 004/109] WIP: Vectorized CSD success - broadcasting magic to take outer products only along necessary axis Changes to be committed: modified: syncopy/connectivity/csd.py --- syncopy/connectivity/csd.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index c41f8002f..aa801d202 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -65,26 +65,28 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False, naive=True) output[j, i] = output[i, j] else: - # build single taper array - chan_idx = np.arange(nChannels) - idx1, idx2 = np.meshgrid(chan_idx, chan_idx) - output = np.real(specs[:, :, idx1] * specs[:, :, idx2].conj()).mean(axis=0).transpose(2,1,0) + # outer product along channel axis + output = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() + # average tapers + output = np.real(output.mean(axis=0).T) + return output # dummy input a = np.ones((10, 3)) * np.arange(1,4) -abig = np.ones((100, 50)) * np.arange(1,51) +abig = np.ones((100, 500)) * np.arange(1,501) # dummy mtmfft result # b = np.arange(1, 4) * np.ones((2,10,3)).astype('complex') # dummt csd matrix c = np.ones((5, 5, 10)) +# for timing + def vectorized(arr): nSamples, nChannels = arr.shape - r = np.multiply.outer(arr,arr.T).reshape(nSamples, nChannels**2 ,nSamples).diagonal(axis1=0, axis2=2) - return r.reshape((nChannels, nChannels, nSamples)) + return (arr[:, np.newaxis, :] * arr[:, :, np.newaxis]).T def arr_loop(arr): nChannels = arr.shape[1] From 8bab75133b865faff361e3bd047057128121625f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 1 Sep 2021 15:22:59 +0200 Subject: [PATCH 005/109] NEW: Vectorized CSDs and Coherencies Changes to be committed: modified: csd.py --- devtest_backend.py | 205 ------------------------------------ syncopy/connectivity/csd.py | 93 ++++++---------- 2 files changed, 31 insertions(+), 267 deletions(-) delete mode 100644 devtest_backend.py diff --git a/devtest_backend.py b/devtest_backend.py deleted file mode 100644 index f202ade61..000000000 --- a/devtest_backend.py +++ /dev/null @@ -1,205 +0,0 @@ -''' This is a temporary development file ''' - -import numpy as np -import matplotlib.pyplot as ppl - -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser -from syncopy.specest.wavelet import get_optimal_wavelet_scales, wavelet -from syncopy.specest.superlet import SuperletTransform, MorletSL, cwtSL, _get_superlet_support, superlet, compute_adaptive_order, scale_from_period -from syncopy.specest.wavelets import Morlet -from scipy.signal import fftconvolve - - -def gen_superlet_testdata(freqs=[20, 40, 60], - cycles=11, fs=1000, - eps = 0): - - ''' - Harmonic superposition of multiple - few-cycle oscillations akin to the - example of Figure 3 in Moca et al. 2021 NatComm - ''' - - signal = [] - for freq in freqs: - - # 10 cycles of f1 - tvec = np.arange(cycles / freq, step=1 / fs) - - harmonic = np.cos(2 * np.pi * freq * tvec) - f_neighbor = np.cos(2 * np.pi * (freq + 10) * tvec) - packet = harmonic + f_neighbor - - # 2 cycles time neighbor - delta_t = np.zeros(int(2 / freq * fs)) - - # 5 cycles break - pad = np.zeros(int(5 / freq * fs)) - - signal.extend([pad, packet, delta_t, harmonic]) - - # stack the packets together with some padding - signal.append(pad) - signal = np.concatenate(signal) - - # additive white noise - if eps > 0: - signal = np.random.randn(len(signal)) * eps + signal - - return signal - - -# test the Wavelet transform -fs = 1000 -s1 = 1 * gen_superlet_testdata(fs=fs, eps=0) # 20Hz, 40Hz and 60Hz -data = np.c_[3*s1, 50*s1] -preselect = np.ones(len(s1), dtype=bool) -preselect2 = np.ones((len(s1), 2), dtype=bool) -pads = 0 - -ts = np.arange(-50,50) -morletTC = Morlet() -morletSL = MorletSL(c_i=30) - -# frequencies to look at, 10th freq is around 20Hz -freqs = np.linspace(1, 100, 50) # up to 100Hz -scalesTC = morletTC.scale_from_period(1 / freqs) -# scales are cycle independent! -scalesSL = scale_from_period(1 / freqs) - -# automatic diadic scales -ssTC = get_optimal_wavelet_scales(Morlet().scale_from_period, len(s1), 1/fs) -ssSL = get_optimal_wavelet_scales(scale_from_period, len(s1), 1/fs) -# a multiplicative Superlet - a set of Morlets, order 1 - 30 -c_1 = 1 -cycles = c_1 * np.arange(1, 31) -sl = [MorletSL(c) for c in cycles] - -res = wavelet(data, - preselect, - preselect, - pads, - pads, - samplerate=fs, - # toi='some', - output_fmt="pow", - scales=scalesTC, - wav=Morlet(), - noCompute=False) - - -# unit impulse -# data = np.zeros(500) -# data[248:252] = 1 -spec = superlet(s1, samplerate=fs, scales=scalesSL, - order_max=10, - order_min=5, - adaptive=False) -spec2 = superlet(data, samplerate=fs, scales=scalesSL, order_max=20, adaptive=False) - -# nc = superlet(data, samplerate=fs, scales=scalesSL, order_max=30) - - -def do_slt(data, scales=scalesSL, **slkwargs): - - if scales is None: - scales = get_optimal_wavelet_scales(scale_from_period, - len(data[:, 0]), - 1 / fs) - - spec = superlet(data, samplerate=fs, - scales=scales, - **slkwargs) - - print(spec.max(),spec.shape) - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - ppl.imshow(np.abs(spec[...,0]), cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return spec - - -def show_MorletSL(morletSL, scale): - - cycle = morletSL.c_i - ts = _get_superlet_support(scale, 1/fs, cycle) - ppl.plot(ts, MorletSL(cycle)(ts, scale)) - - -def show_MorletTC(morletTC, scale): - - M = 10 * scale * fs - # times to use, centred at zero - ts = np.arange((-M + 1) / 2.0, (M + 1) / 2.0) / fs - ppl.plot(ts, morletTC(ts, scale)) - - -def do_superlet_cwt(data, wav, scales=None): - - if scales is None: - scales = get_optimal_wavelet_scales(scale_from_period, len(data[:,0]), 1/fs) - - res = cwtSL(data, - wav, - scales=scales, - dt=1 / fs) - - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - channel=0 - ppl.imshow(np.abs(res[:,:, channel]), cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return res.T - - -def do_normal_cwt(data, wav, scales=None): - - if scales is None: - scales = get_optimal_wavelet_scales(wav.scale_from_period, - len(data[:,0]), - 1/fs) - res = wavelet(data, - preselect, - preselect, - pads, - pads, - samplerate=fs, - # toi='some', - output_fmt="pow", - scales=scales, - wav=wav) - - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - ppl.imshow(res[:, 0, :, 0].T, cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return res[:, 0, :, :].T - -# do_cwt(morletSL) - - -def screen_CWT(w0s= [5, 8, 12]): - for w0 in w0s: - morletTC = Morlet(w0) - scales = _get_optimal_wavelet_scales(morletTC, len(s1), 1/fs) - res = wavelet(s1[:, np.newaxis], - preselect, - preselect, - pads, - pads, - samplerate=fs, - toi=np.array([1,2]), - scales=scales, - wav=morletTC) - - ppl.figure() - ppl.imshow(res[:, 0, :, 0].T, cmap='plasma', aspect='auto') diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index aa801d202..4ca6300ac 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- # -# Cross Spectral Densities and Coherency +# Cross Spectral Density and Coherency # # Builtin/3rd party package imports import numpy as np -from scipy import signal -import itertools # syncopy imports from syncopy.specest.mtmfft import mtmfft -def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False, naive=True): +def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): """ Cross spectral density (CSD) estimate between all channels @@ -26,76 +24,47 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False, naive=True) If normalization is required (`norm=True`) the respective coherencies are returned. + Parameters + ---------- + data_arr : (K,N) :class:`numpy.ndarray` + Uniformly sampled multi-channel time-series data + The 1st dimension is interpreted as the time axis, + columns represent individual channels. + norm : bool + Set to `True` to normalize the cross spectra + + Returns + ------- + CSD_ij : (N, N, M) :class:`numpy.ndarray` + Cross spectral densities or coherencies if `norm=True`. + `M = K // 2 + 1` is the number of Fourier frequency bins, + `N` corresponds to number of input channels. + See also -------- mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` (Multi-)tapered Fourier analysis """ - nSamples, nChannels = data_arr.shape # has shape (nTapers x nFreq x nChannels) specs, freqs = mtmfft(data_arr, samplerate, taper, taperopt) - # has shape (nChannels x nChannels x nFreq) - output = np.zeros((nChannels, nChannels, freqs.size)) + # outer product along channel axes + # has shape (nTapers x nFreq x nChannels x nChannels) + CSD_ij = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() + + # average tapers and transpose: + # now has shape (nChannels x nChannels x nFreq) + CSD_ij = np.real(CSD_ij.mean(axis=0).T) - # somewhat vectorized - not really fast :/ if norm: # main diagonal: auto spectrum for each taper and averaging - diag = np.multiply(specs, specs.conj()).mean(axis=0) - # output[range(nChannels), range(nChannels), :] = np.real(diag.T) - diag = np.real(diag).T - - for i in range(nChannels): - idx = slice(i, nChannels) - row = np.multiply(specs[..., np.tile(i, nChannels - i)], - specs.conj()[..., idx]) - - # normalization - denom = np.multiply(np.tile(diag[i], ((nChannels - i), 1)), diag[i:]) - row = row.mean(axis=0).T / np.sqrt(denom) - output[i, i:, ...] = np.real(row) + # has shape (nChannels x nFreq) + diag = CSD_ij.diagonal() + # get the needed product pairs of the autospectra + Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T + CSD_ij = CSD_ij / Ciijj - elif naive: - for i in range(nChannels): - for j in range(i, nChannels): - output[i, j] = np.real(specs[:, :, i] * specs[:, :, j].conj()).mean(axis=0) - output[j, i] = output[i, j] - - else: - # outer product along channel axis - output = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() - # average tapers - output = np.real(output.mean(axis=0).T) - - return output - - -# dummy input -a = np.ones((10, 3)) * np.arange(1,4) -abig = np.ones((100, 500)) * np.arange(1,501) -# dummy mtmfft result -# b = np.arange(1, 4) * np.ones((2,10,3)).astype('complex') -# dummt csd matrix -c = np.ones((5, 5, 10)) - -# for timing - -def vectorized(arr): - nSamples, nChannels = arr.shape - - return (arr[:, np.newaxis, :] * arr[:, :, np.newaxis]).T - -def arr_loop(arr): - nChannels = arr.shape[1] - output = np.zeros((nChannels, nChannels, arr.shape[0])) - - for i in range(nChannels): - for j in range(i, nChannels): - output[i, j] = np.real(arr[:, i] * arr[:, j]) - output[j, i] = output[i, j] - - return output - + return CSD_ij, freqs From 452461f6a98168362c2544d9a9a0b482862197d4 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 1 Sep 2021 16:03:12 +0200 Subject: [PATCH 006/109] WIP: CSD backend test Changes to be committed: modified: csd.py new file: ../tests/backend/test_coherence.py --- syncopy/connectivity/csd.py | 5 ++- syncopy/tests/backend/test_coherence.py | 43 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 syncopy/tests/backend/test_coherence.py diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index 4ca6300ac..5650b1e35 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -40,6 +40,9 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): `M = K // 2 + 1` is the number of Fourier frequency bins, `N` corresponds to number of input channels. + freqs : (M,) :class:`numpy.ndarray` + The Fourier frequencies + See also -------- mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` @@ -60,7 +63,7 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): CSD_ij = np.real(CSD_ij.mean(axis=0).T) if norm: - # main diagonal: auto spectrum for each taper and averaging + # main diagonal: the auto spectra # has shape (nChannels x nFreq) diag = CSD_ij.diagonal() # get the needed product pairs of the autospectra diff --git a/syncopy/tests/backend/test_coherence.py b/syncopy/tests/backend/test_coherence.py new file mode 100644 index 000000000..34d88c5a6 --- /dev/null +++ b/syncopy/tests/backend/test_coherence.py @@ -0,0 +1,43 @@ +import numpy as np +import matplotlib.pyplot as ppl + +from syncopy.connectivity import csd + + +def gen_testdata(): + + ''' + Superposition of harmonics with + distinct phase relationships between channel. + + Every channel has a 30Hz and a pi/2 shifted 80Hz band, + plus an additional channel specific shift: + Channel1 : 0 + Channel2 : pi/2 + Channel3 : pi + + So the coherencies should be: + C_12 = 0, C_23 = 0, C_13 = -1 + ''' + + fs = 1000 # sampling frequency + nSamples = fs # for integer Fourier freq bins + nChannels = 3 + tvec = np.arange(nSamples) / fs + omegas = np.array([30, 80]) * 2 * np.pi + phase_shifts = np.array([0, np.pi / 2, np.pi]) + + data = np.zeros((nSamples, nChannels)) + + for i, pshift in enumerate(phase_shifts): + sig = 2 * np.cos(omegas[0] * tvec + pshift) + sig += np.cos(omegas[1] * tvec + pshift + np.pi / 2) + + data[:, i] = sig + + return data + + +data = gen_testdata() + + From d213e9f8a03e452ab5266b19e29c3d68f0b422b4 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 1 Sep 2021 17:01:12 +0200 Subject: [PATCH 007/109] WIP: CSD backend test Changes to be committed: modified: ../tests/backend/test_coherence.py --- syncopy/tests/backend/test_coherence.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syncopy/tests/backend/test_coherence.py b/syncopy/tests/backend/test_coherence.py index 34d88c5a6..f55d27acb 100644 --- a/syncopy/tests/backend/test_coherence.py +++ b/syncopy/tests/backend/test_coherence.py @@ -4,12 +4,14 @@ from syncopy.connectivity import csd -def gen_testdata(): +def gen_testdata(eps=0.3): ''' Superposition of harmonics with distinct phase relationships between channel. + Add some white noise to avoid almost 0 frequency bins. + Every channel has a 30Hz and a pi/2 shifted 80Hz band, plus an additional channel specific shift: Channel1 : 0 @@ -33,7 +35,7 @@ def gen_testdata(): sig = 2 * np.cos(omegas[0] * tvec + pshift) sig += np.cos(omegas[1] * tvec + pshift + np.pi / 2) - data[:, i] = sig + data[:, i] = sig + eps * np.random.randn(nSamples) return data From 7a193eb9ecf0c5dc3f4ccb1ef1f3fa2a6712423a Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 3 Sep 2021 10:25:09 +0200 Subject: [PATCH 008/109] WIP: CSD - Coherence estimator - we need an ensemble average for the coherence estimator - multiple possibilities: Welch, multi-taper or multiple realisations - let's first think math/strategy before continuing the implementation --- syncopy/connectivity/csd.py | 4 ++-- syncopy/tests/backend/test_coherence.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/syncopy/connectivity/csd.py b/syncopy/connectivity/csd.py index 5650b1e35..43723a35c 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/connectivity/csd.py @@ -60,7 +60,7 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): # average tapers and transpose: # now has shape (nChannels x nChannels x nFreq) - CSD_ij = np.real(CSD_ij.mean(axis=0).T) + CSD_ij = CSD_ij.mean(axis=0).T if norm: # main diagonal: the auto spectra @@ -70,4 +70,4 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CSD_ij = CSD_ij / Ciijj - return CSD_ij, freqs + return CSD_ij, specs diff --git a/syncopy/tests/backend/test_coherence.py b/syncopy/tests/backend/test_coherence.py index f55d27acb..01519213d 100644 --- a/syncopy/tests/backend/test_coherence.py +++ b/syncopy/tests/backend/test_coherence.py @@ -41,5 +41,5 @@ def gen_testdata(eps=0.3): data = gen_testdata() - +data2 = np.random.randn(1000,3) From 31327085781251ee752f18d3ba7834296fea986a Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 7 Oct 2021 16:53:39 +0200 Subject: [PATCH 009/109] WIP : csd explorations Changes to be committed: new file: dev_csd.py deleted: devtest_frontend.py renamed: syncopy/connectivity/csd.py -> syncopy/specest/csd.py --- dev_csd.py | 51 ++++++++++++++ devtest_frontend.py | 90 ------------------------ syncopy/{connectivity => specest}/csd.py | 14 ++-- 3 files changed, 59 insertions(+), 96 deletions(-) create mode 100644 dev_csd.py delete mode 100644 devtest_frontend.py rename syncopy/{connectivity => specest}/csd.py (90%) diff --git a/dev_csd.py b/dev_csd.py new file mode 100644 index 000000000..627da79f0 --- /dev/null +++ b/dev_csd.py @@ -0,0 +1,51 @@ +import numpy as np +from scipy.signal import csd as sci_csd +from syncopy.specest import csd +import matplotlib.pyplot as ppl + + +# white noise ensemble +nSamples = 10000 +fs = 1000 +nChannels = 5 +data1 = np.random.randn(nSamples, nChannels) + +x1 = data1[:, 0] +y1 = data1[:, 1] + + +def sci_est(x, y, nper, norm=False): + freqs1, csd1 = sci_csd(x, y, fs, window='bartlett', nperseg=nper) + freqs2, csd2 = sci_csd(x, y, fs, window='bartlett', nperseg=nSamples) + + if norm: + # WIP.. + auto1 = sci_csd(x, x, fs, window='bartlett', nperseg=nper) + auto1 *= sci_csd(y, y, fs, window='bartlett', nperseg=nper) + + auto2 = sci_csd(x, y, fs, window='bartlett', nperseg=nSamples) + auto2 *= sci_csd(y, y, fs, window='bartlett', nperseg=nSamples) + + csd1 = csd1 / np.sqrt(auto1 * auto2) + + return (freqs1, np.abs(csd1)), (freqs2, np.abs(csd2)) + + +freqs, CSD, specs = csd.csd(data1, fs, 'bartlett', norm=False) + +# harmonics +tvec = np.arange(nSamples) / fs +omegas = np.array([30, 80]) * 2 * np.pi +phase_shifts = np.array([0, np.pi / 2, np.pi]) + +data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] +data2 = np.array(data2).T +data2 = data2 + np.random.randn(nSamples, 3) * .3 + +# x1 = data2[:, 0] +# y1 = data2[:, 1] + +# freqs2, csd1 = sci_csd(x1, y1, fs, window='bartlett', nperseg=10000) +# freqs2, csd2 = sci_csd(y1, y1, fs, window='bartlett', nperseg=10000) + +# freqs, CSD, specs = csd.csd(data2, fs, 'bartlett', norm=False) diff --git a/devtest_frontend.py b/devtest_frontend.py deleted file mode 100644 index 340864b21..000000000 --- a/devtest_frontend.py +++ /dev/null @@ -1,90 +0,0 @@ -''' This is a temporary development file ''' - -import numpy as np -import matplotlib.pyplot as ppl -from scipy import signal - -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser -from syncopy.datatype import padding -from syncopy.shared.tools import get_defaults -from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning - -from syncopy.specest import freqanalysis -from syncopy.tests.misc import generate_artificial_data - -from syncopy.specest.compRoutines import ( - SuperletTransform, - WaveletTransform, - MultiTaperFFTConvol -) - - -tdat = generate_artificial_data(inmemory=True) - -toi_ival = [-1, 2] -toi_eqd = np.arange(-1, 2, step=0.01) -tSteps = np.diff(toi_eqd) -toi_neqd = [0, 0.1, 0.15, 0.2, 0.24, 0.28, 0.3, 0.35, 1, 2] -# toi_ival = 'all' -# toi_ival = None -# foi = np.logspace(-1, 2.6, 50) -foi = np.linspace(0, 45, 46) -foi = None - -# pad = 'relative' -# pad = 'absolute' -pad = 'maxlen' -padlength = 4000 -# prepadlength = 150 -# postpadlength = 150 - - -r_mtmc = freqanalysis(tdat, method="mtmconvol", - toi=toi_eqd, - # toi='all', - # toi=0.95, - t_ftimwin=.7, - output='pow', - taper='hann', - nTaper=10, - tapsmofrq=5, - keeptapers=False, - pad=pad, - foilim=[0, 50]) -print(r_mtmc.trials[0].shape) - -r_mtm = freqanalysis(tdat, method="mtmfft", - toi=toi_ival, - t_ftimwin=1.5, - output='pow', - taper='dpss', - nTaper=10, - tapsmofrq=None, - keeptapers=False, - pad=pad, - padlength=padlength, - foi=foi) - - -# test classical wavelet analysis -r_wav = freqanalysis( - tdat, method="wavelet", - # toi=toi_ival, - toi='all', - wav=None, - order=4, - output='abs', - foilim=[0.1,50], - #foi=foi, - adaptive=True -) - -# test superlet analysis -r_sup = freqanalysis(tdat, method="superlet", toi=toi_ival, - order_max=30, output='abs', - order_min=1, - c_1 = 3, - foi=foi, - adaptive=True, - wav="Paul") - diff --git a/syncopy/connectivity/csd.py b/syncopy/specest/csd.py similarity index 90% rename from syncopy/connectivity/csd.py rename to syncopy/specest/csd.py index 43723a35c..34015156c 100644 --- a/syncopy/connectivity/csd.py +++ b/syncopy/specest/csd.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Cross Spectral Density and Coherency +# Cross Spectral Density # # Builtin/3rd party package imports @@ -16,14 +16,11 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): Cross spectral density (CSD) estimate between all channels of the input data. First all the individual Fourier transforms are calculated via a (multi-)tapered FFT, then the pairwise - coherence is calculated. Averaging over tapers is done implicitly. + cross-spectra are calculated. Averaging over tapers is done implicitly. Output consists of all (nChannels x nChannels+1)/2 different CSD estimates aranged in a symmetric fashion (CSD_ij == CSD_ji). The elements on the main diagonal (CSD_ii) are the auto-spectra. - If normalization is required (`norm=True`) the respective coherencies - are returned. - Parameters ---------- data_arr : (K,N) :class:`numpy.ndarray` @@ -70,4 +67,9 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CSD_ij = CSD_ij / Ciijj - return CSD_ij, specs + return freqs, CSD_ij, specs + + +# white noise ensemble + + From 7ece7a2ccc2da7a1c6ea6a23d0091f5d5763a393 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 8 Oct 2021 17:07:28 +0200 Subject: [PATCH 010/109] WIP: CF for the single trial cross-spectra - got a working minimal signature - no pure backend Changes to be committed: modified: ../../dev_csd.py new file: single_trial_compRoutines.py modified: ../specest/csd.py --- dev_csd.py | 18 +- .../connectivity/single_trial_compRoutines.py | 198 ++++++++++++++++++ syncopy/specest/csd.py | 21 +- 3 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 syncopy/connectivity/single_trial_compRoutines.py diff --git a/dev_csd.py b/dev_csd.py index 627da79f0..447c3a45c 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -1,6 +1,6 @@ import numpy as np from scipy.signal import csd as sci_csd -from syncopy.specest import csd +from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF import matplotlib.pyplot as ppl @@ -31,7 +31,10 @@ def sci_est(x, y, nper, norm=False): return (freqs1, np.abs(csd1)), (freqs2, np.abs(csd2)) -freqs, CSD, specs = csd.csd(data1, fs, 'bartlett', norm=False) +freqs, CS = cross_spectra_cF(data1, fs, taper='bartlett') +# freqs, CS2, specs = cross_spectra(data1, fs, 'dpss', +# taperopt={'Kmax' : 60, 'NW' : 14}) + # harmonics tvec = np.arange(nSamples) / fs @@ -40,12 +43,11 @@ def sci_est(x, y, nper, norm=False): data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] data2 = np.array(data2).T -data2 = data2 + np.random.randn(nSamples, 3) * .3 +data2 = data2 + np.random.randn(nSamples, 3) * 1 -# x1 = data2[:, 0] -# y1 = data2[:, 1] +x2 = data2[:, 0] +y2 = data2[:, 1] -# freqs2, csd1 = sci_csd(x1, y1, fs, window='bartlett', nperseg=10000) -# freqs2, csd2 = sci_csd(y1, y1, fs, window='bartlett', nperseg=10000) +freqs, CS = cross_spectra_cF(data2, fs, taper='bartlett') +freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : 12, 'NW' : 4}) -# freqs, CSD, specs = csd.csd(data2, fs, 'bartlett', norm=False) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py new file mode 100644 index 000000000..52a9a76c4 --- /dev/null +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# +# computeFunctions and -Routines to calculate +# single trial measures needed for the averaged +# measures like cross spectral densities +# + +# Builtin/3rd party package imports +import numpy as np + +# syncopy imports +from syncopy.specest.mtmfft import mtmfft +from syncopy.specest.const_def import spectralDTypes +from syncopy.shared.errors import SPYWarning +from syncopy.datatype import padding +from syncopy.shared.tools import best_match +from syncopy.shared.computational_routine import ComputationalRoutine +from syncopy.shared.kwarg_decorators import unwrap_io + + +def cross_spectra_cF(trl_dat, + samplerate=1, + foi=None, + padding_opt={}, + taper="hann", + taperopt={}, + polyremoval=False, + timeAxis=0, + noCompute=False): + + """ + Single trial Fourier cross spectra estimates between all channels + of the input data. First all the individual Fourier transforms + are calculated via a (multi-)tapered FFT, then the pairwise + cross-spectra are computed. + + Averaging over tapers is done implicitly + for multi-taper analysis with `taper="dpss"`. + + Output consists of all (nChannels x nChannels+1)/2 different estimates + aranged in a symmetric fashion (CS_ij == CS_ji). The elements on the + main diagonal (CS_ii) are the auto-spectra. + + This is NOT the same as what is commonly referred to as + "cross spectral density" as there is no (time) averaging!! + Multi-tapering alone usually is not sufficient to get enough + statitstical power for a robust csd estimate. + + Parameters + ---------- + trl_dat : (K, N) :class:`numpy.ndarray` + Uniformly sampled multi-channel time-series data + The 1st dimension is interpreted as the time axis, + columns represent individual channels. + Dimensions can be transposed to (N, K) with the `timeAxis` parameter. + samplerate : float + Samplerate in Hz + foi : 1D :class:`numpy.ndarray` + Frequencies of interest (Hz) for output. If desired frequencies + cannot be matched exactly the closest possible frequencies (respecting + data length and padding) are used. + padding_opt : dict + Parameters to be used for padding. See :func:`syncopy.padding` for + more details. + taper : str or None + Taper function to use, one of scipy.signal.windows + Set to `None` for no tapering. + taperopt : dict + Additional keyword arguments passed to the `taper` function. + For multi-tapering with `taper='dpss'` set the keys + `'Kmax'` and `'NW'`. + For further details, please refer to the + `SciPy docs `_ + polyremoval : int or None + **FIXME: Not implemented yet** + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` + subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic + etc.). If `polyremoval` is `None`, no de-trending is performed. + timeAxis : int + Index of running time axis in `trl_dat` (0 or 1) + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + + + Returns + ------- + CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` + Cross spectra for all channel combinations i,j. + `N` corresponds to number of input channels. + + freqs : (M,) :class:`numpy.ndarray` + The Fourier frequencies + + Notes + ----- + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + See also + -------- + mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` + (Multi-)tapered Fourier analysis + + """ + + # Re-arrange array if necessary and get dimensional information + if timeAxis != 0: + dat = trl_dat.T # does not copy but creates view of `trl_dat` + else: + dat = trl_dat + + # Symmetric Padding (updates no. of samples) + if padding_opt: + dat = padding(dat, **padding_opt) + + nChannels = dat.shape[1] + + # specs has shape (nTapers x nFreq x nChannels) + specs, freqs = mtmfft(trl_dat, samplerate, taper, taperopt) + if foi is not None: + _, freq_idx = best_match(freqs, foi, squash_duplicates=True) + nFreq = freq_idx.size + else: + freq_idx = slice(None) + nFreq = freqs.size + + # we always average over tapers here + # use dummy time-axis + outShape = (1, nFreq, nChannels, nChannels) + + # For initialization of computational routine, + # just return output shape and dtype + # cross spectra are complex! + if noCompute: + return outShape, spectralDTypes["fourier"] + + # outer product along channel axes + # has shape (nTapers x nFreq x nChannels x nChannels) + CS_ij = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() + + # average tapers and transpose: + # now has shape (nChannels x nChannels x nFreq) + CS_ij = CS_ij.mean(axis=0).T + + # where does freqs go/come from?! + return freqs[freq_idx], CS_ij[np.newaxis, ..., freq_idx] + + +def covariance_cF(trl_dat): + + """ + Single trial covariance estimates between all channels + of the input data. Output consists of all (nChannels x nChannels+1)/2 + different estimates aranged in a symmetric fashion + (COV_ij == COV_ji). The elements on the + main diagonal (CS_ii) are the channel variances. + + Parameters + ---------- + trl_dat : (K, N) :class:`numpy.ndarray` + Uniformly sampled multi-channel time-series data + The 1st dimension is interpreted as the time axis, + columns represent individual channels. + + Returns + ------- + COV_ij : (N, N, M) :class:`numpy.ndarray` + Covariances for all channel combinations i,j. + + See also + -------- + + + mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` + (Multi-)tapered Fourier analysis + + """ + + COV = np.cov(trl_dat, rowvar=False) + +# # main diagonal: the auto spectra +# # has shape (nChannels x nFreq) +# diag = CSD_ij.diagonal() +# # get the needed product pairs of the autospectra +# Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T +# CSD_ij = CSD_ij / Ciijj + + + diff --git a/syncopy/specest/csd.py b/syncopy/specest/csd.py index 34015156c..791008dca 100644 --- a/syncopy/specest/csd.py +++ b/syncopy/specest/csd.py @@ -10,16 +10,21 @@ from syncopy.specest.mtmfft import mtmfft -def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): +def cross_spectra(data_arr, samplerate, taper="hann", taperopt={}): """ - Cross spectral density (CSD) estimate between all channels + Single trial cross spectra estimates between all channels of the input data. First all the individual Fourier transforms are calculated via a (multi-)tapered FFT, then the pairwise cross-spectra are calculated. Averaging over tapers is done implicitly. - Output consists of all (nChannels x nChannels+1)/2 different CSD estimates - aranged in a symmetric fashion (CSD_ij == CSD_ji). The elements on the - main diagonal (CSD_ii) are the auto-spectra. + Output consists of all (nChannels x nChannels+1)/2 different estimates + aranged in a symmetric fashion (CS_ij == CS_ji). The elements on the + main diagonal (CS_ii) are the auto-spectra. + + This is NOT the same as what is commonly referred to as + "cross spectral density" as there is no (time) averaging!! + Multi-tapering alone usually is not sufficient to get enough + statitstical power for a robust csd estimate. Parameters ---------- @@ -27,13 +32,11 @@ def csd(data_arr, samplerate, taper="hann", taperopt={}, norm=False): Uniformly sampled multi-channel time-series data The 1st dimension is interpreted as the time axis, columns represent individual channels. - norm : bool - Set to `True` to normalize the cross spectra Returns ------- - CSD_ij : (N, N, M) :class:`numpy.ndarray` - Cross spectral densities or coherencies if `norm=True`. + CS_ij : (N, N, M) :class:`numpy.ndarray` + Cross spectra for all channel combinations i,j. `M = K // 2 + 1` is the number of Fourier frequency bins, `N` corresponds to number of input channels. From 815ac2ee0c7313ff5fb8bb959c242d020a6bbc63 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 11 Oct 2021 16:23:29 +0200 Subject: [PATCH 011/109] WIP: CSDs - single-trial multi-tapered coherence estimate - it's good for testing, and works reasonably well for simple noisy harmonic test signals Changes to be committed: modified: dev_csd.py modified: syncopy/connectivity/single_trial_compRoutines.py --- dev_csd.py | 16 ++-- .../connectivity/single_trial_compRoutines.py | 96 ++++++++++++++----- 2 files changed, 84 insertions(+), 28 deletions(-) diff --git a/dev_csd.py b/dev_csd.py index 447c3a45c..adcbcc2c3 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -5,7 +5,7 @@ # white noise ensemble -nSamples = 10000 +nSamples = 1000 fs = 1000 nChannels = 5 data1 = np.random.randn(nSamples, nChannels) @@ -32,8 +32,12 @@ def sci_est(x, y, nper, norm=False): freqs, CS = cross_spectra_cF(data1, fs, taper='bartlett') -# freqs, CS2, specs = cross_spectra(data1, fs, 'dpss', -# taperopt={'Kmax' : 60, 'NW' : 14}) +freqs, CS2 = cross_spectra_cF(data1, fs, taper='dpss', + taperopt={'Kmax' : 10, 'NW' : 3}, norm=True) + +freqs, CS3 = cross_spectra_cF(data1, fs, taper='dpss', + taperopt={'Kmax' : 20, 'NW' : 3}, norm=True) + # harmonics @@ -43,11 +47,11 @@ def sci_est(x, y, nper, norm=False): data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] data2 = np.array(data2).T -data2 = data2 + np.random.randn(nSamples, 3) * 1 +eps = 1 +data2 = 5 * (data2 + np.random.randn(nSamples, 3) * eps) x2 = data2[:, 0] y2 = data2[:, 1] freqs, CS = cross_spectra_cF(data2, fs, taper='bartlett') -freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : 12, 'NW' : 4}) - +freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : 20, 'NW' : 6}, norm=True) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 52a9a76c4..70c39e520 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# computeFunctions and -Routines to calculate +# computeFunctions and -Routines to parallel calculate # single trial measures needed for the averaged # measures like cross spectral densities # @@ -26,6 +26,7 @@ def cross_spectra_cF(trl_dat, taperopt={}, polyremoval=False, timeAxis=0, + norm=False, noCompute=False): """ @@ -43,8 +44,10 @@ def cross_spectra_cF(trl_dat, This is NOT the same as what is commonly referred to as "cross spectral density" as there is no (time) averaging!! - Multi-tapering alone usually is not sufficient to get enough - statitstical power for a robust csd estimate. + Multi-tapering alone is not necessarily sufficient to get enough + statitstical power for a robust csd estimate. Yet for completeness + and testing the option `norm=True` will output a single-trial + coherence estimate. Parameters ---------- @@ -55,7 +58,7 @@ def cross_spectra_cF(trl_dat, Dimensions can be transposed to (N, K) with the `timeAxis` parameter. samplerate : float Samplerate in Hz - foi : 1D :class:`numpy.ndarray` + foi : 1D :class:`numpy.ndarray` or None, optional Frequencies of interest (Hz) for output. If desired frequencies cannot be matched exactly the closest possible frequencies (respecting data length and padding) are used. @@ -65,7 +68,7 @@ def cross_spectra_cF(trl_dat, taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. - taperopt : dict + taperopt : dict, optional Additional keyword arguments passed to the `taper` function. For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. @@ -79,8 +82,11 @@ def cross_spectra_cF(trl_dat, least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic etc.). If `polyremoval` is `None`, no de-trending is performed. - timeAxis : int + timeAxis : int, optional Index of running time axis in `trl_dat` (0 or 1) + norm : bool, optional + Set to `True` to normalize for a single-trial coherence measure. + Only meaningful in a multi-taper (`taper="dpss"`) setup! noCompute : bool Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output @@ -151,11 +157,27 @@ def cross_spectra_cF(trl_dat, # now has shape (nChannels x nChannels x nFreq) CS_ij = CS_ij.mean(axis=0).T + if norm: + # only meaningful for multi-tapering + assert taper == 'dpss' + # main diagonal: the auto spectra + # has shape (nChannels x nFreq) + diag = CS_ij.diagonal() + # # get the needed product pairs of the autospectra + Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T + CS_ij = CS_ij / Ciijj + # where does freqs go/come from?! return freqs[freq_idx], CS_ij[np.newaxis, ..., freq_idx] -def covariance_cF(trl_dat): +def cross_covariance_cF(trl_dat, + samplerate=1, + padding_opt={}, + polyremoval=False, + timeAxis=0, + norm=False, + noCompute=False): """ Single trial covariance estimates between all channels @@ -170,29 +192,59 @@ def covariance_cF(trl_dat): Uniformly sampled multi-channel time-series data The 1st dimension is interpreted as the time axis, columns represent individual channels. + Dimensions can be transposed to (N, K) with the `timeAxis` parameter. + samplerate : float + Samplerate in Hz + padding_opt : dict + Parameters to be used for padding. See :func:`syncopy.padding` for + more details. + polyremoval : int or None + **FIXME: Not implemented yet** + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` + subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic + etc.). If `polyremoval` is `None`, no de-trending is performed. + timeAxis : int, optional + Index of running time axis in `trl_dat` (0 or 1) + norm : bool, optional + Set to `True` to normalize for single-trial cross-correlation. + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + Returns ------- - COV_ij : (N, N, M) :class:`numpy.ndarray` - Covariances for all channel combinations i,j. - - See also - -------- + CC_ij : (K, 1, N, N) :class:`numpy.ndarray` + Cross covariance for all channel combinations i,j. + `N` corresponds to number of input channels. - - mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` - (Multi-)tapered Fourier analysis + Notes + ----- + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. """ - COV = np.cov(trl_dat, rowvar=False) + # Re-arrange array if necessary and get dimensional information + if timeAxis != 0: + dat = trl_dat.T # does not copy but creates view of `trl_dat` + else: + dat = trl_dat + + # Symmetric Padding (updates no. of samples) + if padding_opt: + dat = padding(dat, **padding_opt) + + nChannels = dat.shape[1] -# # main diagonal: the auto spectra -# # has shape (nChannels x nFreq) -# diag = CSD_ij.diagonal() -# # get the needed product pairs of the autospectra -# Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T -# CSD_ij = CSD_ij / Ciijj From 23abefb0d978de70b81af38adcb778c6b3a66380 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 11 Oct 2021 16:44:27 +0200 Subject: [PATCH 012/109] WIP: MTM Coherence examples, preparation for backend tests/tutorial Changes to be committed: modified: dev_csd.py modified: syncopy/connectivity/single_trial_compRoutines.py --- dev_csd.py | 36 +++++++++++++++---- .../connectivity/single_trial_compRoutines.py | 3 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/dev_csd.py b/dev_csd.py index adcbcc2c3..e9abfbcc5 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -45,13 +45,35 @@ def sci_est(x, y, nper, norm=False): omegas = np.array([30, 80]) * 2 * np.pi phase_shifts = np.array([0, np.pi / 2, np.pi]) -data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] -data2 = np.array(data2).T +NN = 50 +res = np.zeros((len(freqs), NN)) eps = 1 -data2 = 5 * (data2 + np.random.randn(nSamples, 3) * eps) +Kmax = 5 +for i in range(NN): -x2 = data2[:, 0] -y2 = data2[:, 1] + data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] + data2 = np.array(data2).T + data2 = 5 * (data2 + np.random.randn(nSamples, 3) * eps) + + x2 = data2[:, 0] + y2 = data2[:, 1] + + freqs, CS = cross_spectra_cF(data2, fs, taper='bartlett') + freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + + res[:, i] = np.abs(CS2[0, 0, 1, :]) + +q1 = np.percentile(res, 25, axis=1) +q3 = np.percentile(res, 75, axis=1) +med = np.percentile(res, 50, axis=1) + +fig, ax = ppl.subplots(figsize=(6,4), num=None) +ax.set_xlabel('frequency (Hz)') +ax.set_ylabel('coherence') +ax.set_ylim((-.02,1.05)) +ax.set_title(f'MTM coherence, {Kmax} tapers, SNR={1/eps**2}') + +c = 'cornflowerblue' +ax.plot(freqs, med, lw=2, alpha=0.8, c=c) +ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) -freqs, CS = cross_spectra_cF(data2, fs, taper='bartlett') -freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : 20, 'NW' : 6}, norm=True) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 70c39e520..693f3fbb0 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -86,7 +86,8 @@ def cross_spectra_cF(trl_dat, Index of running time axis in `trl_dat` (0 or 1) norm : bool, optional Set to `True` to normalize for a single-trial coherence measure. - Only meaningful in a multi-taper (`taper="dpss"`) setup! + Only meaningful in a multi-taper (`taper="dpss"`) setup and if no + additional (trial-)averaging is perfomed afterwards. noCompute : bool Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output From e3264b6d2204f3d16b74659ecd751af94b381fa1 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 12 Oct 2021 13:14:05 +0200 Subject: [PATCH 013/109] WIP: Cross-Correlations --- dev_csd.py | 90 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/dev_csd.py b/dev_csd.py index e9abfbcc5..7bd21091b 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -1,5 +1,6 @@ import numpy as np from scipy.signal import csd as sci_csd +import scipy.signal as sci from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF import matplotlib.pyplot as ppl @@ -7,6 +8,8 @@ # white noise ensemble nSamples = 1000 fs = 1000 +tvec = np.arange(nSamples) / fs + nChannels = 5 data1 = np.random.randn(nSamples, nChannels) @@ -31,49 +34,70 @@ def sci_est(x, y, nper, norm=False): return (freqs1, np.abs(csd1)), (freqs2, np.abs(csd2)) -freqs, CS = cross_spectra_cF(data1, fs, taper='bartlett') -freqs, CS2 = cross_spectra_cF(data1, fs, taper='dpss', - taperopt={'Kmax' : 10, 'NW' : 3}, norm=True) +def mtm_csd_harmonics(): + + omegas = np.array([30, 80]) * 2 * np.pi + phase_shifts = np.array([0, np.pi / 2, np.pi]) -freqs, CS3 = cross_spectra_cF(data1, fs, taper='dpss', - taperopt={'Kmax' : 20, 'NW' : 3}, norm=True) + NN = 50 + res = np.zeros((nSamples // 2 + 1, NN)) + eps = 1 + Kmax = 5 + data = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] + data = np.array(data).T + data = 5 * (data + np.random.randn(nSamples, 3) * eps) + + for i in range(NN): + freqs, CS = cross_spectra_cF(data, fs, taper='bartlett') + freqs, CS2 = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + res[:, i] = np.abs(CS2[0, 0, 1, :]) -# harmonics -tvec = np.arange(nSamples) / fs -omegas = np.array([30, 80]) * 2 * np.pi -phase_shifts = np.array([0, np.pi / 2, np.pi]) + q1 = np.percentile(res, 25, axis=1) + q3 = np.percentile(res, 75, axis=1) + med = np.percentile(res, 50, axis=1) + + fig, ax = ppl.subplots(figsize=(6,4), num=None) + ax.set_xlabel('frequency (Hz)') + ax.set_ylabel('coherence') + ax.set_ylim((-.02,1.05)) + ax.set_title(f'MTM coherence, {Kmax} tapers, SNR={1/eps**2}') + + c = 'cornflowerblue' + ax.plot(freqs, med, lw=2, alpha=0.8, c=c) + ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) + + +sig1 = np.cos(2 * np.pi * 30 * tvec) +sig2 = np.sin(2 * np.pi * 30 * tvec) + +mode = 'same' +r = sci.correlate(sig1, sig2, mode=mode, method='fft') +r2 = sci.fftconvolve(sig1, sig2[::-1], mode=mode) +assert np.all(r == r2) + +lags = np.arange(-nSamples // 2, nSamples // 2) -NN = 50 -res = np.zeros((len(freqs), NN)) -eps = 1 -Kmax = 5 -for i in range(NN): +ppl.figure(1) +ppl.xlabel('lag (s)') +ppl.ylabel('convolution result') - data2 = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] - data2 = np.array(data2).T - data2 = 5 * (data2 + np.random.randn(nSamples, 3) * eps) +ppl.plot(lags, r2, lw = 1.5) +# ppl.xlim((490,600)) - x2 = data2[:, 0] - y2 = data2[:, 1] +norm = np.arange(nSamples // 2, nSamples) / 2 +norm = np.r_[norm, norm[::-1]] - freqs, CS = cross_spectra_cF(data2, fs, taper='bartlett') - freqs, CS2 = cross_spectra_cF(data2, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) +ppl.figure(2) +ppl.xlabel('lag (s)') +ppl.ylabel('correlation') +ppl.plot(lags, r2 / norm, lw = 1.5) - res[:, i] = np.abs(CS2[0, 0, 1, :]) -q1 = np.percentile(res, 25, axis=1) -q3 = np.percentile(res, 75, axis=1) -med = np.percentile(res, 50, axis=1) +sig3 = np.cos(2 * np.pi * 30 * tvec + np.pi) +data = np.c_[sig1, sig2, sig3, sig1] -fig, ax = ppl.subplots(figsize=(6,4), num=None) -ax.set_xlabel('frequency (Hz)') -ax.set_ylabel('coherence') -ax.set_ylim((-.02,1.05)) -ax.set_title(f'MTM coherence, {Kmax} tapers, SNR={1/eps**2}') +rr = sci.fftconvolve(data, data[::-1, ::-1], mode='same') -c = 'cornflowerblue' -ax.plot(freqs, med, lw=2, alpha=0.8, c=c) -ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) From 8d32eb5d196639655ab0ae309183ebd2629111a1 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 14 Oct 2021 12:55:37 +0200 Subject: [PATCH 014/109] WIP: Single trial cross-covariance - added `cross_covariance_cF` compute function Changes to be committed: modified: dev_csd.py modified: syncopy/connectivity/single_trial_compRoutines.py --- dev_csd.py | 87 +++++++++++-------- .../connectivity/single_trial_compRoutines.py | 57 +++++++++--- 2 files changed, 96 insertions(+), 48 deletions(-) diff --git a/dev_csd.py b/dev_csd.py index 7bd21091b..4680d6334 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -2,13 +2,17 @@ from scipy.signal import csd as sci_csd import scipy.signal as sci from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF +from syncopy.connectivity.single_trial_compRoutines import cross_covariance_cF import matplotlib.pyplot as ppl # white noise ensemble -nSamples = 1000 +nSamples = 1001 fs = 1000 tvec = np.arange(nSamples) / fs +omegas = np.array([21, 42, 59, 78]) * 2 * np.pi +omegas = np.arange(10, 40) * 2 * np.pi +data = np.c_[[1*np.cos(om * tvec) for om in omegas]].T nChannels = 5 data1 = np.random.randn(nSamples, nChannels) @@ -17,6 +21,49 @@ y1 = data1[:, 1] +def dev_cc(nSamples=1001): + + tvec = np.arange(nSamples) / fs + + sig1 = np.cos(2 * np.pi * 30 * tvec) + sig2 = np.sin(2 * np.pi * 30 * tvec) + + mode = 'same' + t_half = nSamples // 2 + + r = sci.correlate(sig2, sig2, mode=mode, method='fft') + r2 = sci.fftconvolve(sig2, sig2[::-1], mode=mode) + assert np.all(r == r2) + + + lags = np.arange(-nSamples // 2, nSamples // 2) + if nSamples % 2 != 0: + lags = lags + 1 + lags = lags * 1 / fs + + if nSamples % 2 == 0: + half_lags = np.arange(0, nSamples // 2) + else: + half_lags = np.arange(0, nSamples // 2 + 1) + half_lags = half_lags * 1 / fs + + ppl.figure(1) + ppl.xlabel('lag (s)') + ppl.ylabel('convolution result') + + ppl.plot(lags, r2, lw = 1.5) + # ppl.xlim((490,600)) + + + norm = np.arange(nSamples, t_half, step = -1) / 2 + # norm = np.r_[norm, norm[::-1]] + + ppl.figure(2) + ppl.xlabel('lag (s)') + ppl.ylabel('correlation') + ppl.plot(half_lags, r2[nSamples // 2:] / norm, lw = 1.5) + + def sci_est(x, y, nper, norm=False): freqs1, csd1 = sci_csd(x, y, fs, window='bartlett', nperseg=nper) freqs2, csd2 = sci_csd(x, y, fs, window='bartlett', nperseg=nSamples) @@ -49,8 +96,8 @@ def mtm_csd_harmonics(): for i in range(NN): - freqs, CS = cross_spectra_cF(data, fs, taper='bartlett') - freqs, CS2 = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + CS, freqs = cross_spectra_cF(data, fs, taper='bartlett') + CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) res[:, i] = np.abs(CS2[0, 0, 1, :]) @@ -67,37 +114,3 @@ def mtm_csd_harmonics(): c = 'cornflowerblue' ax.plot(freqs, med, lw=2, alpha=0.8, c=c) ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) - - -sig1 = np.cos(2 * np.pi * 30 * tvec) -sig2 = np.sin(2 * np.pi * 30 * tvec) - -mode = 'same' -r = sci.correlate(sig1, sig2, mode=mode, method='fft') -r2 = sci.fftconvolve(sig1, sig2[::-1], mode=mode) -assert np.all(r == r2) - -lags = np.arange(-nSamples // 2, nSamples // 2) - -ppl.figure(1) -ppl.xlabel('lag (s)') -ppl.ylabel('convolution result') - -ppl.plot(lags, r2, lw = 1.5) -# ppl.xlim((490,600)) - -norm = np.arange(nSamples // 2, nSamples) / 2 -norm = np.r_[norm, norm[::-1]] - -ppl.figure(2) -ppl.xlabel('lag (s)') -ppl.ylabel('correlation') -ppl.plot(lags, r2 / norm, lw = 1.5) - - -sig3 = np.cos(2 * np.pi * 30 * tvec + np.pi) -data = np.c_[sig1, sig2, sig3, sig1] - -rr = sci.fftconvolve(data, data[::-1, ::-1], mode='same') - - diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 693f3fbb0..810ed3bb8 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -7,6 +7,7 @@ # Builtin/3rd party package imports import numpy as np +from scipy.signal import fftconvolve # syncopy imports from syncopy.specest.mtmfft import mtmfft @@ -93,10 +94,9 @@ def cross_spectra_cF(trl_dat, instead return expected shape and :class:`numpy.dtype` of output array. - Returns ------- - CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` + CS_ij : (nFreq, N, N) :class:`numpy.ndarray` Cross spectra for all channel combinations i,j. `N` corresponds to number of input channels. @@ -141,8 +141,7 @@ def cross_spectra_cF(trl_dat, nFreq = freqs.size # we always average over tapers here - # use dummy time-axis - outShape = (1, nFreq, nChannels, nChannels) + outShape = (nFreq, nChannels, nChannels) # For initialization of computational routine, # just return output shape and dtype @@ -161,15 +160,14 @@ def cross_spectra_cF(trl_dat, if norm: # only meaningful for multi-tapering assert taper == 'dpss' - # main diagonal: the auto spectra - # has shape (nChannels x nFreq) + # main diagonal has shape (nChannels x nFreq): the auto spectra diag = CS_ij.diagonal() - # # get the needed product pairs of the autospectra + # get the needed product pairs of the autospectra Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj # where does freqs go/come from?! - return freqs[freq_idx], CS_ij[np.newaxis, ..., freq_idx] + return CS_ij[..., freq_idx].transpose(2, 0, 1), freqs[freq_idx] def cross_covariance_cF(trl_dat, @@ -216,10 +214,9 @@ def cross_covariance_cF(trl_dat, instead return expected shape and :class:`numpy.dtype` of output array. - Returns ------- - CC_ij : (K, 1, N, N) :class:`numpy.ndarray` + CC_ij : (K, N, N) :class:`numpy.ndarray` Cross covariance for all channel combinations i,j. `N` corresponds to number of input channels. @@ -243,9 +240,47 @@ def cross_covariance_cF(trl_dat, # Symmetric Padding (updates no. of samples) if padding_opt: dat = padding(dat, **padding_opt) - + + nSamples = dat.shape[0] nChannels = dat.shape[1] + # positive lags in time units + if nSamples % 2 == 0: + lags = np.arange(0, nSamples // 2) + else: + lags = np.arange(0, nSamples // 2 + 1) + lags = lags * 1 / samplerate + + outShape = (len(lags), nChannels, nChannels) + + # For initialization of computational routine, + # just return output shape and dtype + # cross spectra are complex! + if noCompute: + return outShape, spectralDTypes["abs"] + + + # re-normalize output for different effective overlaps + norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) / 4 + + CC = np.empty(outShape) + for i in range(nChannels): + for j in range(i + 1): + cc12 = fftconvolve(dat[:, i], dat[::-1, j], mode='same') + CC[:, i, j] = cc12[nSamples // 2:] / norm_overlap + if i != j: + # cross-correlation is NOT symmetric.. + cc21 = fftconvolve(dat[:, j], dat[::-1, i], mode='same') + CC[:, j, i] = cc21[nSamples // 2:] / norm_overlap + + # normalize with products of std + if norm: + STDs = np.std(dat, axis=0) + N = STDs[:, None] * STDs[None, :] + CC = CC / N + + return CC, lags + From b13c81c998b116c0556dcc80be6c455c2e96fcf0 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 20 Oct 2021 17:17:30 +0200 Subject: [PATCH 015/109] NEW: Backend tests for single trial cross-spectra and cross-cov - most likely has to be adjusted once the frontend integration is done Changes to be committed: modified: ../../../dev_csd.py modified: ../../connectivity/single_trial_compRoutines.py deleted: test_coherence.py new file: test_connectivity.py --- dev_csd.py | 50 ++++++++++++-- .../connectivity/single_trial_compRoutines.py | 2 +- syncopy/tests/backend/test_coherence.py | 45 ------------- syncopy/tests/backend/test_connectivity.py | 65 +++++++++++++++++++ 4 files changed, 111 insertions(+), 51 deletions(-) delete mode 100644 syncopy/tests/backend/test_coherence.py create mode 100644 syncopy/tests/backend/test_connectivity.py diff --git a/dev_csd.py b/dev_csd.py index 4680d6334..9be3130a4 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -22,7 +22,9 @@ def dev_cc(nSamples=1001): - + + nSamples = 1001 + fs = 1000 tvec = np.arange(nSamples) / fs sig1 = np.cos(2 * np.pi * 30 * tvec) @@ -89,17 +91,17 @@ def mtm_csd_harmonics(): NN = 50 res = np.zeros((nSamples // 2 + 1, NN)) eps = 1 - Kmax = 5 + Kmax = 15 data = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] data = np.array(data).T - data = 5 * (data + np.random.randn(nSamples, 3) * eps) for i in range(NN): + dataR = 5 * (data + np.random.randn(nSamples, 3) * eps) CS, freqs = cross_spectra_cF(data, fs, taper='bartlett') - CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + CS2, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) - res[:, i] = np.abs(CS2[0, 0, 1, :]) + res[:, i] = np.abs(CS2[:, 0, 1]) q1 = np.percentile(res, 25, axis=1) q3 = np.percentile(res, 75, axis=1) @@ -114,3 +116,41 @@ def mtm_csd_harmonics(): c = 'cornflowerblue' ax.plot(freqs, med, lw=2, alpha=0.8, c=c) ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) + + +# omegas = np.arange(30, 50, step=1) * 2 * np.pi +# data = np.array([np.cos(om * tvec) for om in omegas]).T +# dataR = 1 * (data + np.random.randn(*data.shape) * 1) +# CS, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=True) + +# CS2, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=False) + +# noisy phase evolution +def phase_evo(omega0, eps, fs=1000, N=1000): + wn = np.random.randn(N) * 1 / fs + delta_ts = np.ones(N) * 1 / fs + phase = np.cumsum(omega0 * delta_ts + eps * wn) + return phase + + +eps = 0.3 * fs +omega = 50 * 2 * np.pi +p1 = phase_evo(omega, eps, N=nSamples) +p2 = phase_evo(omega, eps, N=nSamples) +s1 = np.cos(p1) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) +s2 = np.cos(p2) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) +data = np.c_[s1, s2] + +CS, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=True) +CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=False) + +fig, ax = ppl.subplots(figsize=(6,4), num=None) +ax.set_xlabel('frequency (Hz)') +ax.set_ylabel('$|CSD(f)|$') +ax.set_ylim((-.02,1.25)) + +ax.plot(freqs, np.abs(CS2[:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) +ax.plot(freqs, np.abs(CS2[:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) +ax.plot(freqs, np.abs(CS2[:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) + +ax.legend() diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 810ed3bb8..e9fcb078e 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -261,7 +261,7 @@ def cross_covariance_cF(trl_dat, # re-normalize output for different effective overlaps - norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) / 4 + norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) CC = np.empty(outShape) for i in range(nChannels): diff --git a/syncopy/tests/backend/test_coherence.py b/syncopy/tests/backend/test_coherence.py deleted file mode 100644 index 01519213d..000000000 --- a/syncopy/tests/backend/test_coherence.py +++ /dev/null @@ -1,45 +0,0 @@ -import numpy as np -import matplotlib.pyplot as ppl - -from syncopy.connectivity import csd - - -def gen_testdata(eps=0.3): - - ''' - Superposition of harmonics with - distinct phase relationships between channel. - - Add some white noise to avoid almost 0 frequency bins. - - Every channel has a 30Hz and a pi/2 shifted 80Hz band, - plus an additional channel specific shift: - Channel1 : 0 - Channel2 : pi/2 - Channel3 : pi - - So the coherencies should be: - C_12 = 0, C_23 = 0, C_13 = -1 - ''' - - fs = 1000 # sampling frequency - nSamples = fs # for integer Fourier freq bins - nChannels = 3 - tvec = np.arange(nSamples) / fs - omegas = np.array([30, 80]) * 2 * np.pi - phase_shifts = np.array([0, np.pi / 2, np.pi]) - - data = np.zeros((nSamples, nChannels)) - - for i, pshift in enumerate(phase_shifts): - sig = 2 * np.cos(omegas[0] * tvec + pshift) - sig += np.cos(omegas[1] * tvec + pshift + np.pi / 2) - - data[:, i] = sig + eps * np.random.randn(nSamples) - - return data - - -data = gen_testdata() -data2 = np.random.randn(1000,3) - diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py new file mode 100644 index 000000000..0b7e7cce3 --- /dev/null +++ b/syncopy/tests/backend/test_connectivity.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import matplotlib.pyplot as ppl +from syncopy.connectivity import single_trial_compRoutines as stCR + + +def test_csd(): + + nSamples = 1001 + fs = 1000 + tvec = np.arange(nSamples) / fs + harm_freq = 40 + phase_shifts = np.array([0, np.pi / 2, np.pi]) + + # 1 phase phase shifted harmonics + white noise, SNR = 1 + data = [np.cos(harm_freq * 2 * np. pi * tvec + ps) + for ps in phase_shifts] + data = np.array(data).T + data = np.array(data) + np.random.randn(nSamples, len(phase_shifts)) + + Kmax = 8 # multiple tapers for single trial coherence + CSD, freqs = stCR.cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + + fig, ax = ppl.subplots(figsize=(6,4), num=None) + ax.set_xlabel('frequency (Hz)') + ax.set_ylabel('coherence') + ax.set_ylim((-.02,1.05)) + ax.set_title(f'MTM coherence, {Kmax} tapers, SNR=1') + + coh = np.abs(CSD[:, 0, 1]) + assert ax.plot(freqs, coh, lw=2, alpha=0.8, c='cornflowerblue') + + # we test for the highest peak sitting at + # the vicinity (± 5Hz) of one of the harmonics + peak_val = np.max(coh) + peak_idx = np.argmax(coh) + peak_freq = freqs[peak_idx] + print(peak_freq, peak_val) + assert harm_freq - 5 < peak_freq < harm_freq + 5 + + # we test that the peak value + # is at least 0.9 and max 1 + assert 0.8 < peak_val < 1 + + +def test_cross_cov(): + + nSamples = 1001 + fs = 1000 + tvec = np.arange(nSamples) / fs + + cosine = np.cos(2 * np.pi * 30 * tvec) + sine = np.sin(2 * np.pi * 30 * tvec) + data = np.c_[cosine, sine] + + # output shape is (nLags x nChannels x nChannels) + CC, lags = stCR.cross_covariance_cF(data, samplerate=fs, norm=True) + # test for result is returned in the [0, np.ceil(nSamples / 2)] lag interval + nLags = int(np.ceil(nSamples / 2)) + assert CC.shape[0] == nLags + + # cross-correlation (normalized cross-covariance) between + # cosine and sine analytically equals minus sine + assert np.all(CC[:, 0, 1] + sine[:nLags] < 1e-5) From ab2b0b60d16cbbe99f446e8a680ba88146b22f04 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 22 Oct 2021 14:39:12 +0200 Subject: [PATCH 016/109] WIP: Single trial connectivity measures dimensions - for measures like csd's and cross-covariances we define an universal return shape: nTime x nFreqs x nChannels x nChannels where often one of the time or frequency axes will be a singleton Changes to be committed: modified: connectivity/single_trial_compRoutines.py modified: tests/backend/test_connectivity.py --- .../connectivity/single_trial_compRoutines.py | 27 +++++++++---------- syncopy/tests/backend/test_connectivity.py | 19 ++++++++----- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index e9fcb078e..6b8d1f32e 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -39,9 +39,9 @@ def cross_spectra_cF(trl_dat, Averaging over tapers is done implicitly for multi-taper analysis with `taper="dpss"`. - Output consists of all (nChannels x nChannels+1)/2 different estimates - aranged in a symmetric fashion (CS_ij == CS_ji). The elements on the - main diagonal (CS_ii) are the auto-spectra. + Output consists of all (nChannels x nChannels+1)/2 different complex + estimates aranged in a symmetric fashion (CS_ij == CS_ji*). The + elements on the main diagonal (CS_ii) are the (real) auto-spectra. This is NOT the same as what is commonly referred to as "cross spectral density" as there is no (time) averaging!! @@ -96,7 +96,7 @@ def cross_spectra_cF(trl_dat, Returns ------- - CS_ij : (nFreq, N, N) :class:`numpy.ndarray` + CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` Cross spectra for all channel combinations i,j. `N` corresponds to number of input channels. @@ -131,7 +131,7 @@ def cross_spectra_cF(trl_dat, nChannels = dat.shape[1] - # specs has shape (nTapers x nFreq x nChannels) + # specs have shape (nTapers x nFreq x nChannels) specs, freqs = mtmfft(trl_dat, samplerate, taper, taperopt) if foi is not None: _, freq_idx = best_match(freqs, foi, squash_duplicates=True) @@ -141,7 +141,7 @@ def cross_spectra_cF(trl_dat, nFreq = freqs.size # we always average over tapers here - outShape = (nFreq, nChannels, nChannels) + outShape = (1, nFreq, nChannels, nChannels) # For initialization of computational routine, # just return output shape and dtype @@ -166,8 +166,8 @@ def cross_spectra_cF(trl_dat, Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj - # where does freqs go/come from?! - return CS_ij[..., freq_idx].transpose(2, 0, 1), freqs[freq_idx] + # where does freqs go/come from - we will allow tuples as return values yeah! + return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] def cross_covariance_cF(trl_dat, @@ -216,7 +216,7 @@ def cross_covariance_cF(trl_dat, Returns ------- - CC_ij : (K, N, N) :class:`numpy.ndarray` + CC_ij : (K, 1, N, N) :class:`numpy.ndarray` Cross covariance for all channel combinations i,j. `N` corresponds to number of input channels. @@ -251,14 +251,13 @@ def cross_covariance_cF(trl_dat, lags = np.arange(0, nSamples // 2 + 1) lags = lags * 1 / samplerate - outShape = (len(lags), nChannels, nChannels) + outShape = (len(lags), 1, nChannels, nChannels) # For initialization of computational routine, # just return output shape and dtype - # cross spectra are complex! + # cross covariances are real! if noCompute: return outShape, spectralDTypes["abs"] - # re-normalize output for different effective overlaps norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) @@ -267,11 +266,11 @@ def cross_covariance_cF(trl_dat, for i in range(nChannels): for j in range(i + 1): cc12 = fftconvolve(dat[:, i], dat[::-1, j], mode='same') - CC[:, i, j] = cc12[nSamples // 2:] / norm_overlap + CC[:,0, i, j] = cc12[nSamples // 2:] / norm_overlap if i != j: # cross-correlation is NOT symmetric.. cc21 = fftconvolve(dat[:, j], dat[::-1, i], mode='same') - CC[:, j, i] = cc21[nSamples // 2:] / norm_overlap + CC[:, 0, j, i] = cc21[nSamples // 2:] / norm_overlap # normalize with products of std if norm: diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 0b7e7cce3..5748b9651 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -22,17 +22,22 @@ def test_csd(): Kmax = 8 # multiple tapers for single trial coherence CSD, freqs = stCR.cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + # output has shape (1, nFreq, nChannels, nChannels) + assert CSD.shape == (1, len(freqs), data.shape[1], data.shape[1]) + + # single trial coherence between channel 0 and 1 + coh = np.abs(CSD[0, :, 0, 1]) + fig, ax = ppl.subplots(figsize=(6,4), num=None) ax.set_xlabel('frequency (Hz)') ax.set_ylabel('coherence') ax.set_ylim((-.02,1.05)) ax.set_title(f'MTM coherence, {Kmax} tapers, SNR=1') - coh = np.abs(CSD[:, 0, 1]) assert ax.plot(freqs, coh, lw=2, alpha=0.8, c='cornflowerblue') # we test for the highest peak sitting at - # the vicinity (± 5Hz) of one of the harmonics + # the vicinity (± 5Hz) of one the harmonic peak_val = np.max(coh) peak_idx = np.argmax(coh) peak_freq = freqs[peak_idx] @@ -41,7 +46,7 @@ def test_csd(): # we test that the peak value # is at least 0.9 and max 1 - assert 0.8 < peak_val < 1 + assert 0.9 < peak_val < 1 def test_cross_cov(): @@ -54,12 +59,14 @@ def test_cross_cov(): sine = np.sin(2 * np.pi * 30 * tvec) data = np.c_[cosine, sine] - # output shape is (nLags x nChannels x nChannels) + # output shape is (nLags x 1 x nChannels x nChannels) CC, lags = stCR.cross_covariance_cF(data, samplerate=fs, norm=True) # test for result is returned in the [0, np.ceil(nSamples / 2)] lag interval nLags = int(np.ceil(nSamples / 2)) - assert CC.shape[0] == nLags + + # output has shape (nLags, 1, nChannels, nChannels) + assert CC.shape == (nLags, 1, data.shape[1], data.shape[1]) # cross-correlation (normalized cross-covariance) between # cosine and sine analytically equals minus sine - assert np.all(CC[:, 0, 1] + sine[:nLags] < 1e-5) + assert np.all(CC[:, 0, 0, 1] + sine[:nLags] < 1e-5) From 00436fa26feb81019176a8c4a8c73c03326432b2 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 22 Oct 2021 15:34:29 +0200 Subject: [PATCH 017/109] NEW : Linear detrending for connectivity methods - just harnessing scipy.signal.detrend - TODO: add this to all specest methods Changes to be committed: modified: ../../connectivity/single_trial_compRoutines.py modified: test_connectivity.py --- .../connectivity/single_trial_compRoutines.py | 46 +++++++++++-------- syncopy/tests/backend/test_connectivity.py | 14 ++++-- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 6b8d1f32e..165e52c3f 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -7,7 +7,7 @@ # Builtin/3rd party package imports import numpy as np -from scipy.signal import fftconvolve +from scipy.signal import fftconvolve, detrend # syncopy imports from syncopy.specest.mtmfft import mtmfft @@ -76,13 +76,11 @@ def cross_spectra_cF(trl_dat, For further details, please refer to the `SciPy docs `_ polyremoval : int or None - **FIXME: Not implemented yet** - Order of polynomial used for de-trending data in the time domain prior - to spectral analysis. A value of 0 corresponds to subtracting the mean - ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the - least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` - subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic - etc.). If `polyremoval` is `None`, no de-trending is performed. + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. timeAxis : int, optional Index of running time axis in `trl_dat` (0 or 1) norm : bool, optional @@ -125,6 +123,13 @@ def cross_spectra_cF(trl_dat, else: dat = trl_dat + # detrend + if polyremoval == 0: + # SciPy's overwrite_data not working for type='constant' :/ + dat = detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + detrend(dat, type='linear', axis=0, overwrite_data=True) + # Symmetric Padding (updates no. of samples) if padding_opt: dat = padding(dat, **padding_opt) @@ -132,7 +137,7 @@ def cross_spectra_cF(trl_dat, nChannels = dat.shape[1] # specs have shape (nTapers x nFreq x nChannels) - specs, freqs = mtmfft(trl_dat, samplerate, taper, taperopt) + specs, freqs = mtmfft(dat, samplerate, taper, taperopt) if foi is not None: _, freq_idx = best_match(freqs, foi, squash_duplicates=True) nFreq = freq_idx.size @@ -198,13 +203,11 @@ def cross_covariance_cF(trl_dat, Parameters to be used for padding. See :func:`syncopy.padding` for more details. polyremoval : int or None - **FIXME: Not implemented yet** - Order of polynomial used for de-trending data in the time domain prior - to spectral analysis. A value of 0 corresponds to subtracting the mean - ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the - least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` - subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic - etc.). If `polyremoval` is `None`, no de-trending is performed. + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. timeAxis : int, optional Index of running time axis in `trl_dat` (0 or 1) norm : bool, optional @@ -237,6 +240,13 @@ def cross_covariance_cF(trl_dat, else: dat = trl_dat + # detrend + if polyremoval == 0: + # SciPy's overwrite_data not working for type='constant' :/ + dat = detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + detrend(dat, type='linear', axis=0, overwrite_data=True) + # Symmetric Padding (updates no. of samples) if padding_opt: dat = padding(dat, **padding_opt) @@ -279,7 +289,3 @@ def cross_covariance_cF(trl_dat, CC = CC / N return CC, lags - - - - diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 5748b9651..8f10b2a87 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -13,14 +13,18 @@ def test_csd(): harm_freq = 40 phase_shifts = np.array([0, np.pi / 2, np.pi]) - # 1 phase phase shifted harmonics + white noise, SNR = 1 - data = [np.cos(harm_freq * 2 * np. pi * tvec + ps) - for ps in phase_shifts] + # 1 phase phase shifted harmonics + white noise + constant, SNR = 1 + data = [10 + np.cos(harm_freq * 2 * np. pi * tvec + ps) + for ps in phase_shifts] data = np.array(data).T data = np.array(data) + np.random.randn(nSamples, len(phase_shifts)) - + Kmax = 8 # multiple tapers for single trial coherence - CSD, freqs = stCR.cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) + CSD, freqs = stCR.cross_spectra_cF(data, fs, + polyremoval=1, + taper='dpss', + taperopt={'Kmax' : Kmax, 'NW' : 6}, + norm=True) # output has shape (1, nFreq, nChannels, nChannels) assert CSD.shape == (1, len(freqs), data.shape[1], data.shape[1]) From 87c9f09d30c9b31c162710dd876dfd1bb277739a Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 25 Oct 2021 17:02:42 +0200 Subject: [PATCH 018/109] WIP : Connectivity frontend Changes to be committed: new file: dev_frontend.py new file: syncopy/connectivity/__init__.py new file: syncopy/connectivity/connectivity_analysis.py new file: syncopy/connectivity/const_def.py modified: syncopy/connectivity/single_trial_compRoutines.py --- dev_frontend.py | 18 +++ syncopy/connectivity/__init__.py | 12 ++ syncopy/connectivity/connectivity_analysis.py | 126 ++++++++++++++++++ syncopy/connectivity/const_def.py | 21 +++ .../connectivity/single_trial_compRoutines.py | 4 +- 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 dev_frontend.py create mode 100644 syncopy/connectivity/__init__.py create mode 100644 syncopy/connectivity/connectivity_analysis.py create mode 100644 syncopy/connectivity/const_def.py diff --git a/dev_frontend.py b/dev_frontend.py new file mode 100644 index 000000000..4ec72d183 --- /dev/null +++ b/dev_frontend.py @@ -0,0 +1,18 @@ +import numpy as np +import scipy.signal as sci +from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF +from syncopy.connectivity.single_trial_compRoutines import cross_covariance_cF +from syncopy.connectivity import connectivityanalysis +import matplotlib.pyplot as ppl + +from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.tools import get_defaults +from syncopy.datatype import SpectralData, padding + +from syncopy.tests.misc import generate_artificial_data +tdat = generate_artificial_data(inmemory=True) + +sdict = {"trials": [0], 'channels' : ['channel1']} +connectivityanalysis(data=tdat, select=sdict, pad_to_length=4200) +connectivityanalysis(data=tdat, select=sdict, pad_to_length=None) + diff --git a/syncopy/connectivity/__init__.py b/syncopy/connectivity/__init__.py new file mode 100644 index 000000000..d5521f3de --- /dev/null +++ b/syncopy/connectivity/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# +# Populate namespace with user exposed +# connectivity methods +# + +from .connectivity_analysis import connectivityanalysis +from .connectivity_analysis import __all__ as _all_ + +# Populate local __all__ namespace +__all__ = [] +__all__.extend(_all_) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py new file mode 100644 index 000000000..4cda6329f --- /dev/null +++ b/syncopy/connectivity/connectivity_analysis.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# +# Syncopy connectivity analysis methods +# + +# Builtin/3rd party package imports +import numpy as np + +# Syncopy imports +from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.tools import get_defaults +from syncopy.datatype import SpectralData, padding +from syncopy.datatype.methods.padding import _nextpow2 +from syncopy.shared.tools import best_match +from syncopy.shared.errors import ( + SPYValueError, + SPYTypeError, + SPYWarning, + SPYInfo) + +from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, + detect_parallel_client) + +# Local imports +from .const_def import ( + availableTapers, + availableMethods, + generalParameters +) + +# CRs still missing, CFs are already there +from .single_trial_compRoutines import ( + cross_spectra_cF, + cross_covariance_cF +) + +__all__ = ["connectivityanalysis"] + + +@unwrap_cfg +@unwrap_select +@detect_parallel_client +def connectivityanalysis(data, method='csd', + foi=None, foilim=None, pad_to_length=None, + polyremoval=None, taper="hann", tapsmofrq=None, + nTaper=None, toi="all", + **kwargs): + + """ + coming soon.. + """ + + # Make sure our one mandatory input object can be processed + try: + data_parser(data, varname="data", dataclass="AnalogData", + writable=None, empty=False) + except Exception as exc: + raise exc + timeAxis = data.dimord.index("time") + + + # Get everything of interest in local namespace + defaults = get_defaults(connectivityanalysis) + lcls = locals() + + # Ensure a valid computational method was selected + if method not in availableMethods: + lgl = "'" + "or '".join(opt + "' " for opt in availableMethods) + raise SPYValueError(legal=lgl, varname="method", actual=method) + + + # If only a subset of `data` is to be processed, + # make some necessary adjustments + # and compute minimal sample-count across (selected) trials + if data._selection is not None: + trialList = data._selection.trials + sinfo = np.zeros((len(trialList), 2)) + for tk, trlno in enumerate(trialList): + trl = data._preview_trial(trlno) + tsel = trl.idx[timeAxis] + if isinstance(tsel, list): + sinfo[tk, :] = [0, len(tsel)] + else: + sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] + else: + trialList = list(range(len(data.trials))) + sinfo = data.sampleinfo + + lenTrials = np.diff(sinfo).squeeze() + + # here we enforce equal lengths trials as is required for + # sensical trial averaging - user is responsible for trial + # specific padding and time axis alignments + # OR we do a brute force 'maxlen' padding if there is unequal lengths?! + if not lenTrials.min() == lenTrials.max(): + lgl = "trials of same lengths" + actual = "trials of different lengths - please pre-pad!" + raise SPYValueError(legal=lgl, varname="lenTrials", actual=actual) + + numTrials = len(trialList) + + print(lenTrials) + + # manual symmetric zero padding of ALL trials the same way + + if pad_to_length is not None: + + scalar_parser(pad_to_length, + varname='pad_to_length', + ntype='int_like', + lims=[lenTrials.max(), np.inf]) + + padding_opt = { + 'padtype' : 'zero', + 'pad' : 'absolute', + 'padlength' : pad_to_length + } + + else: + padding_opt = None + + if method == 'csd': + # for now manually select a trial + single_trial = data.trials[data._selection.trials] + res, freqs = cross_spectra_cF(single_trial, padding_opt=padding_opt) + print(res.shape) diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py new file mode 100644 index 000000000..557671570 --- /dev/null +++ b/syncopy/connectivity/const_def.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Constant definitions and helper functions for spectral estimations +# + +# Builtin/3rd party package imports +import numpy as np + +# Module-wide output specs +spectralDTypes = {"complex": np.complex64, + "real": np.float32} + +#: available tapers of :func:`~syncopy.connectivity_analysis` +availableTapers = ("hann", "dpss") + +#: available spectral estimation methods of :func:`~syncopy.connectivity_analysis` +availableMethods = ("csd", "corr") + +#: general, method agnostic, parameters of :func:`~syncopy.connectivity_analysis` +generalParameters = ("method", "output", "keeptrials", + "foi", "foilim", "polyremoval", "out") diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index 165e52c3f..e2bb45b73 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -19,6 +19,7 @@ from syncopy.shared.kwarg_decorators import unwrap_io +@unwrap_io def cross_spectra_cF(trl_dat, samplerate=1, foi=None, @@ -95,7 +96,7 @@ def cross_spectra_cF(trl_dat, Returns ------- CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` - Cross spectra for all channel combinations i,j. + Complex cross spectra for all channel combinations i,j. `N` corresponds to number of input channels. freqs : (M,) :class:`numpy.ndarray` @@ -175,6 +176,7 @@ def cross_spectra_cF(trl_dat, return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] +@unwrap_io def cross_covariance_cF(trl_dat, samplerate=1, padding_opt={}, From 65afce9a6043556830836a1f60fb8e1b92587d9e Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 26 Oct 2021 16:04:31 +0200 Subject: [PATCH 019/109] WIP: connectivity frontend - toi and foi sanitization, starting to run into the same pattern again: too much stuff is being done in the fronend. Probably it's better to delegate more stuff into the CRs, which are still missing here Changes to be committed: modified: dev_frontend.py modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/connectivity/const_def.py --- dev_frontend.py | 19 ++- syncopy/connectivity/connectivity_analysis.py | 115 +++++++++++++++--- syncopy/connectivity/const_def.py | 8 ++ 3 files changed, 121 insertions(+), 21 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 4ec72d183..f511f3916 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -12,7 +12,20 @@ from syncopy.tests.misc import generate_artificial_data tdat = generate_artificial_data(inmemory=True) -sdict = {"trials": [0], 'channels' : ['channel1']} -connectivityanalysis(data=tdat, select=sdict, pad_to_length=4200) -connectivityanalysis(data=tdat, select=sdict, pad_to_length=None) +# this still gives type(tsel) = slice :) +sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} +# this gives type(tsel) = list +# sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.array([0, 0.3, 1])} +sdict2 = {"trials": [0], 'channels' : ['channel1'], 'toilim' : [-1, 0]} + +print('sdict1') +connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) +#connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') +#connectivityanalysis(data=tdat, select=sdict1, pad_to_length=None) +print('sdict2') +#connectivityanalysis(data=tdat, select=sdict2) +print('no selection') +#connectivityanalysis(data=tdat, foi = np.arange(20, 80)) +#connectivityanalysis(data=tdat, foilim = [20, 80]) + diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 4cda6329f..e84f18a89 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -5,6 +5,7 @@ # Builtin/3rd party package imports import numpy as np +from numbers import Number # Syncopy imports from syncopy.shared.parsers import data_parser, scalar_parser, array_parser @@ -25,7 +26,8 @@ from .const_def import ( availableTapers, availableMethods, - generalParameters + generalParameters, + nextpow2 ) # CRs still missing, CFs are already there @@ -58,7 +60,6 @@ def connectivityanalysis(data, method='csd', raise exc timeAxis = data.dimord.index("time") - # Get everything of interest in local namespace defaults = get_defaults(connectivityanalysis) lcls = locals() @@ -68,7 +69,6 @@ def connectivityanalysis(data, method='csd', lgl = "'" + "or '".join(opt + "' " for opt in availableMethods) raise SPYValueError(legal=lgl, varname="method", actual=method) - # If only a subset of `data` is to be processed, # make some necessary adjustments # and compute minimal sample-count across (selected) trials @@ -78,14 +78,17 @@ def connectivityanalysis(data, method='csd', for tk, trlno in enumerate(trialList): trl = data._preview_trial(trlno) tsel = trl.idx[timeAxis] + + # user picked discrete set of time points if isinstance(tsel, list): - sinfo[tk, :] = [0, len(tsel)] - else: - sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] + lgl = "equidistant time points (toi) or time slice (toilim)" + actual = "non-equidistant set of time points" + raise SPYValueError(legal=lgl, varname="select", actual=actual) + + sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] else: trialList = list(range(len(data.trials))) - sinfo = data.sampleinfo - + sinfo = data.sampleinfo lenTrials = np.diff(sinfo).squeeze() # here we enforce equal lengths trials as is required for @@ -97,30 +100,106 @@ def connectivityanalysis(data, method='csd', actual = "trials of different lengths - please pre-pad!" raise SPYValueError(legal=lgl, varname="lenTrials", actual=actual) - numTrials = len(trialList) + numTrials = len(trialList) print(lenTrials) - - # manual symmetric zero padding of ALL trials the same way - if pad_to_length is not None: + # --- Padding --- + + # manual symmetric zero padding of ALL trials the same way + if isinstance(pad_to_length, Number): scalar_parser(pad_to_length, varname='pad_to_length', ntype='int_like', - lims=[lenTrials.max(), np.inf]) - + lims=[lenTrials.max(), np.inf]) padding_opt = { 'padtype' : 'zero', 'pad' : 'absolute', 'padlength' : pad_to_length } - - else: + # after padding! + nSamples = pad_to_length + # or pad to optimal FFT lengths + elif pad_to_length == 'nextpow2': + padding_opt = { + 'padtype' : 'zero', + 'pad' : 'nextpow2' + } + # after padding + nSamples = nextpow2(int(lenTrials.min())) + # no padding + else: padding_opt = None + nSamples = int(lenTrials.min()) + + # --- foi sanitization --- + + if foi is not None: + if isinstance(foi, str): + if foi == "all": + foi = None + else: + raise SPYValueError(legal="'all' or `None` or list/array", + varname="foi", actual=foi) + else: + try: + array_parser(foi, varname="foi", hasinf=False, hasnan=False, + lims=[0, data.samplerate/2], dims=(None,)) + except Exception as exc: + raise exc + foi = np.array(foi, dtype="float") + + if foilim is not None: + if isinstance(foilim, str): + if foilim == "all": + foilim = None + else: + raise SPYValueError(legal="'all' or `None` or `[fmin, fmax]`", + varname="foilim", actual=foilim) + else: + try: + array_parser(foilim, varname="foilim", hasinf=False, hasnan=False, + lims=[0, data.samplerate/2], dims=(2,)) + except Exception as exc: + raise exc + # foilim is of shape (2,) + if foilim[0] > foilim[1]: + msg = "Sorting foilim low to high.." + SPYInfo(msg) + foilim = np.sort(foilim) + + if foi is not None and foilim is not None: + lgl = "either `foi` or `foilim` specification" + act = "both" + raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) + + # only now set foi array for foilim in 1Hz steps + if foilim: + foi = np.arange(foilim[0], foilim[1] + 1) if method == 'csd': + + if foi is None and foilim is None: + # Construct array of maximally attainable frequencies + freqs = np.fft.rfftfreq(nSamples, 1 / data.samplerate) + msg = (f"Automatic FFT frequency selection from {freqs[0]:.1f}Hz to " + f"{freqs[-1]:.1f}Hz") + SPYInfo(msg) + foi = freqs + # for now manually select a trial - single_trial = data.trials[data._selection.trials] - res, freqs = cross_spectra_cF(single_trial, padding_opt=padding_opt) + if data._selection is not None: + single_trial = data.trials[data._selection.trials] + else: + single_trial = data.trials[0] + + res, freqs = cross_spectra_cF(single_trial, samplerate=data.samplerate, + padding_opt=padding_opt, foi=foi) + + print('A') print(res.shape) + print(freqs[-10:]) + print(foi[-10:]) + print('B') + diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py index 557671570..9f1e23ff7 100644 --- a/syncopy/connectivity/const_def.py +++ b/syncopy/connectivity/const_def.py @@ -19,3 +19,11 @@ #: general, method agnostic, parameters of :func:`~syncopy.connectivity_analysis` generalParameters = ("method", "output", "keeptrials", "foi", "foilim", "polyremoval", "out") + + +# auxiliary functions +def nextpow2(number): + n = 1 + while n < number: + n *= 2 + return n From e29782c63028ff1a2847d1a436a67368fcaf6ba8 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 28 Oct 2021 16:00:23 +0200 Subject: [PATCH 020/109] NEW: gitignore Mac OS .DS_Store files Changes to be committed: modified: .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a7d9512d8..4fe07edc2 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ requirements-test.txt # Editor-related stuff .vscode + +# Mac OS related stuff +.DS_Store \ No newline at end of file From d33fcc0ef010d34a0ef9d8476984c01f70d12560 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 29 Oct 2021 15:48:13 +0200 Subject: [PATCH 021/109] NEW: polyremoval for specest methods - using scipy.signal.detrend for all methods for either de-meaning or linear trend removal Changes to be committed: modified: dev_frontend.py modified: syncopy/connectivity/single_trial_compRoutines.py modified: syncopy/specest/compRoutines.py modified: syncopy/specest/freqanalysis.py modified: syncopy/specest/mtmconvol.py --- dev_frontend.py | 20 +++-- .../connectivity/single_trial_compRoutines.py | 2 +- syncopy/specest/compRoutines.py | 80 +++++++++++++------ syncopy/specest/freqanalysis.py | 17 ++-- syncopy/specest/mtmconvol.py | 6 +- 5 files changed, 81 insertions(+), 44 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index f511f3916..67dd5d62e 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -3,6 +3,7 @@ from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF from syncopy.connectivity.single_trial_compRoutines import cross_covariance_cF from syncopy.connectivity import connectivityanalysis +from syncopy.specest import freqanalysis import matplotlib.pyplot as ppl from syncopy.shared.parsers import data_parser, scalar_parser, array_parser @@ -12,6 +13,7 @@ from syncopy.tests.misc import generate_artificial_data tdat = generate_artificial_data(inmemory=True) +foilim = [30, 50] # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} # this gives type(tsel) = list @@ -20,12 +22,20 @@ print('sdict1') connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) -#connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') -#connectivityanalysis(data=tdat, select=sdict1, pad_to_length=None) +# connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') +# connectivityanalysis(data=tdat, select=sdict1, pad_to_length=None) print('sdict2') -#connectivityanalysis(data=tdat, select=sdict2) +# connectivityanalysis(data=tdat, select=sdict2) print('no selection') -#connectivityanalysis(data=tdat, foi = np.arange(20, 80)) -#connectivityanalysis(data=tdat, foilim = [20, 80]) +# connectivityanalysis(data=tdat, foi = np.arange(20, 80)) +# connectivityanalysis(data=tdat, foilim = [20, 80]) +res = freqanalysis(data=tdat, + method='mtmfft', + samplerate=tdat.samplerate, + order_max=20, + foi=np.arange(1, 150, 5), + output='abs', +# polyremoval=1, + t_ftimwin=0.5) diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/single_trial_compRoutines.py index e2bb45b73..6eff2c933 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/single_trial_compRoutines.py @@ -129,7 +129,7 @@ def cross_spectra_cF(trl_dat, # SciPy's overwrite_data not working for type='constant' :/ dat = detrend(dat, type='constant', axis=0, overwrite_data=True) elif polyremoval == 1: - detrend(dat, type='linear', axis=0, overwrite_data=True) + dat = detrend(dat, type='linear', axis=0, overwrite_data=True) # Symmetric Padding (updates no. of samples) if padding_opt: diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index 0e5ec9ba6..d48906e9d 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -91,13 +91,11 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, Number of samples to pad to data (if `pad` is 'absolute' or 'relative'). See :func:`syncopy.padding` for more information. polyremoval : int or None - **FIXME: Not implemented yet** Order of polynomial used for de-trending data in the time domain prior to spectral analysis. A value of 0 corresponds to subtracting the mean ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the - least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` - subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic - etc.). If `polyremoval` is `None`, no de-trending is performed. + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. output_fmt : str Output of spectral estimation; one of :data:`~syncopy.specest.const_def.availableOutputs` noCompute : bool @@ -168,6 +166,12 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, if noCompute: return outShape, spectralDTypes[output_fmt] + # detrend, does not work with 'FauxTrial' data.. + if polyremoval == 0: + dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) + # call actual specest method res, _ = mtmfft(dat, **method_kwargs) @@ -249,7 +253,7 @@ def mtmconvol_cF( toi=None, foi=None, nTaper=1, tapsmofrq=None, timeAxis=0, - keeptapers=True, polyremoval=None, output_fmt="pow", + keeptapers=True, polyremoval=0, output_fmt="pow", noCompute=False, chunkShape=None, method_kwargs=None): """ Perform time-frequency analysis on multi-channel time series data using a sliding window FFT @@ -300,13 +304,11 @@ def mtmconvol_cF( If `True`, results of Fourier transform are preserved for each taper, otherwise spectrum is averaged across tapers. polyremoval : int - **FIXME: Not implemented yet** - Order of polynomial used for de-trending. A value of 0 corresponds to - subtracting the mean ("de-meaning"), ``polyremoval = 1`` removes linear - trends (subtracting the least squares fit of a linear function), - ``polyremoval = N`` for `N > 1` subtracts a polynomial of order `N` (``N = 2`` - quadratic, ``N = 3`` cubic etc.). If `polyremoval` is `None`, no de-trending - is performed. + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial). Detrending is done on each segment! + If `polyremoval` is `None`, no de-trending is performed. output_fmt : str Output of spectral estimation; one of :data:`~syncopy.specest.const_def.availableOutputs` noCompute : bool @@ -348,7 +350,7 @@ def mtmconvol_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Pad input array if necessary if padbegin > 0 or padend > 0: dat = padding(dat, "zero", pad="relative", padlength=None, @@ -377,9 +379,18 @@ def mtmconvol_cF( if noCompute: return outShape, spectralDTypes[output_fmt] + # detrending options for each segment + if polyremoval == 0: + detrend = 'constant' + elif polyremoval == 1: + detrend = 'linear' + else: + detrend = False + # additional keyword args for `stft` in dictionary method_kwargs.update({"boundary": stftBdry, - "padded": stftPad}) + "padded": stftPad, + "detrend" : detrend}) if equidistant: ftr, freqs = mtmconvol(dat[soi, :], **method_kwargs) @@ -468,7 +479,7 @@ def wavelet_cF( postselect, toi=None, timeAxis=0, - polyremoval=None, + polyremoval=0, output_fmt="pow", noCompute=False, chunkShape=None, @@ -495,13 +506,11 @@ def wavelet_cF( timeAxis : int Index of running time axis in `trl_dat` (0 or 1) polyremoval : int - **FIXME: Not implemented yet** - Order of polynomial used for de-trending. A value of 0 corresponds to - subtracting the mean ("de-meaning"), ``polyremoval = 1`` removes linear - trends (subtracting the least squares fit of a linear function), - ``polyremoval = N`` for `N > 1` subtracts a polynomial of order `N` (``N = 2`` - quadratic, ``N = 3`` cubic etc.). If `polyremoval` is `None`, no de-trending - is performed. + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. output_fmt : str Output of spectral estimation; one of :data:`~syncopy.specest.const_def.availableOutputs` noCompute : bool @@ -550,7 +559,7 @@ def wavelet_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Get shape of output for dry-run phase nChannels = dat.shape[1] if isinstance(toi, np.ndarray): # `toi` is an array of time-points @@ -562,6 +571,12 @@ def wavelet_cF( if noCompute: return outShape, spectralDTypes[output_fmt] + # detrend, does not work with 'FauxTrial' data.. + if polyremoval == 0: + dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) + # ------------------ # actual method call # ------------------ @@ -633,10 +648,9 @@ def superlet_cF( trl_dat, preselect, postselect, - # padbegin, # were always 0! - # padend, toi=None, timeAxis=0, + polyremoval=0, output_fmt="pow", noCompute=False, chunkShape=None, @@ -661,6 +675,14 @@ def superlet_cF( Either array of equidistant time-points or `"all"` to perform analysis on all samples in `trl_dat`. Please refer to :func:`~syncopy.freqanalysis` for further details. + timeAxis : int + Index of running time axis in `trl_dat` (0 or 1) + polyremoval : int or None + Order of polynomial used for de-trending data in the time domain prior + to spectral analysis. A value of 0 corresponds to subtracting the mean + ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. output_fmt : str Output of spectral estimation; one of :data:`~syncopy.specest.const_def.availableOutputs` @@ -704,7 +726,7 @@ def superlet_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Get shape of output for dry-run phase nChannels = trl_dat.shape[1] if isinstance(toi, np.ndarray): # `toi` is an array of time-points @@ -716,6 +738,12 @@ def superlet_cF( if noCompute: return outShape, spectralDTypes[output_fmt] + # detrend, does not work with 'FauxTrial' data.. + if polyremoval == 0: + dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) + # ------------------ # actual method call # ------------------ diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 8441d4182..f6807ad68 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -46,7 +46,7 @@ @detect_parallel_client def freqanalysis(data, method='mtmfft', output='fourier', keeptrials=True, foi=None, foilim=None, pad=None, padtype='zero', - padlength=None, polyremoval=None, + padlength=None, polyremoval=0, taper="hann", tapsmofrq=None, nTaper=None, keeptapers=False, toi="all", t_ftimwin=None, wavelet="Morlet", width=6, order=None, order_max=None, order_min=1, c_1=3, adaptive=False, @@ -63,8 +63,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', * **foi**/**foilim** : frequencies of interest; either array of frequencies or frequency window (not both) * **keeptrials** : return individual trials or grand average - * **polyremoval** : de-trending method to use (0 = mean, 1 = linear, 2 = quadratic, - 3 = cubic, etc.) + * **polyremoval** : de-trending method to use (0 = mean, 1 = linear) List of available analysis methods and respective distinct options: @@ -192,13 +191,13 @@ def freqanalysis(data, method='mtmfft', output='fourier', samples to append to each trial. See :func:`syncopy.padding` for more information. polyremoval : int or None - **FIXME: Not implemented yet** Order of polynomial used for de-trending data in the time domain prior to spectral analysis. A value of 0 corresponds to subtracting the mean ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the - least squares fit of a linear polynomial), ``polyremoval = N`` for `N > 1` - subtracts a polynomial of order `N` (``N = 2`` quadratic, ``N = 3`` cubic - etc.). If `polyremoval` is `None`, no de-trending is performed. + least squares fit of a linear polynomial). + If `polyremoval` is `None`, no de-trending is performed. Note that + for spectral estimation de-meaning is very advisable and hence also the + default. taper : str Only valid if `method` is `'mtmfft'` or `'mtmconvol'`. Windowing function, one of :data:`~syncopy.specest.const_def.availableTapers` (see below). @@ -459,9 +458,8 @@ def freqanalysis(data, method='mtmfft', output='fourier', # FIXME: implement detrending # see also https://docs.obspy.org/_modules/obspy/signal/detrend.html#polynomial if polyremoval is not None: - raise NotImplementedError("Detrending has not been implemented yet.") try: - scalar_parser(polyremoval, varname="polyremoval", lims=[0, 8], ntype="int_like") + scalar_parser(polyremoval, varname="polyremoval", lims=[0, 1], ntype="int_like") except Exception as exc: raise exc @@ -989,6 +987,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', postSelect, toi=toi, timeAxis=timeAxis, + polyremoval=polyremoval, output_fmt=output, method_kwargs=method_kwargs) diff --git a/syncopy/specest/mtmconvol.py b/syncopy/specest/mtmconvol.py index 880ff1bda..951ece961 100644 --- a/syncopy/specest/mtmconvol.py +++ b/syncopy/specest/mtmconvol.py @@ -9,7 +9,7 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", - taperopt={}, boundary='zeros', padded=True): + taperopt={}, boundary='zeros', padded=True, detrend=False): ''' (Multi-)tapered short time fast Fourier transform. Returns @@ -108,8 +108,8 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", for taperIdx, win in enumerate(windows): # pxx has shape (nFreq, nChannels, nTime) _, _, pxx = signal.stft(data_arr, samplerate, win, - nperseg, noverlap, boundary=boundary, - padded=padded, axis=0) + nperseg, noverlap, boundary=boundary, + padded=padded, axis=0, detrend=detrend) if taper == 'dpss': # reverse scipy window normalization From d5198e33539580fc8a883c0abfb5cd74862cf4c4 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 2 Nov 2021 17:59:59 +0100 Subject: [PATCH 022/109] WIP: First draft of CrossSpectralData class - leverage functionality of `SpectralData` wherever possible - convert 2d channel x channel index tuple to linear channel list for actual indexing of underlying `SpectralData` object On branch coherence Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/tests/spy_setup.py --- syncopy/datatype/base_data.py | 2 +- syncopy/datatype/continuous_data.py | 297 ++++++++++++++++++---------- syncopy/tests/spy_setup.py | 5 +- 3 files changed, 203 insertions(+), 101 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 12769e876..822616b4b 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -27,7 +27,7 @@ from syncopy.shared.tools import StructDict from syncopy.shared.parsers import (scalar_parser, array_parser, io_parser, filename_parser, data_parser) -from syncopy.shared.errors import SPYTypeError, SPYValueError, SPYError, SPYWarning +from syncopy.shared.errors import SPYTypeError, SPYValueError, SPYError from syncopy.datatype.methods.definetrial import definetrial as _definetrial from syncopy import __version__, __storage__, __acme__, __sessionid__, __storagelimit__ if __acme__: diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 96cd07ff8..e1aee0585 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Syncopy's abstract base class for continuous data + regular children -# +# """Uniformly sampled (continuous data). @@ -22,7 +22,7 @@ from .methods.definetrial import definetrial from .methods.selectdata import selectdata from syncopy.shared.parsers import scalar_parser, array_parser -from syncopy.shared.errors import SPYValueError, SPYIOError +from syncopy.shared.errors import SPYValueError, SPYWarning from syncopy.shared.tools import best_match from syncopy.plotting import _plot_analog from syncopy.plotting import _plot_spectral @@ -38,15 +38,15 @@ class ContinuousData(BaseData, ABC): This class cannot be instantiated. Use one of the children instead. """ - + _infoFileProperties = BaseData._infoFileProperties + ("samplerate", "channel",) _hdfFileAttributeProperties = BaseData._hdfFileAttributeProperties + ("samplerate", "channel",) _hdfFileDatasetProperties = BaseData._hdfFileDatasetProperties + ("data",) - + @property def data(self): """array-like object representing data without trials - + Trials are concatenated along the time axis. """ @@ -57,7 +57,7 @@ def data(self): raise SPYValueError(legal=lgl, actual=act.format(self.filename), varname="data") return self._data - + @data.setter def data(self, inData): @@ -73,7 +73,7 @@ def __str__(self): ppattrs = [attr for attr in ppattrs if not (inspect.ismethod(getattr(self, attr)) or isinstance(getattr(self, attr), Iterator))] - + ppattrs.sort() # Construct string for pretty-printing class attributes @@ -122,7 +122,7 @@ def __str__(self): ppstr += printString.format(attr, valueString) ppstr += "\nUse `.log` to see object history" return ppstr - + @property def _shapes(self): if self.sampleinfo is not None: @@ -137,28 +137,28 @@ def channel(self): """ :class:`numpy.ndarray` : list of recording channel names """ # if data exists but no user-defined channel labels, create them on the fly if self._channel is None and self._data is not None: - nChannel = self.data.shape[self.dimord.index("channel")] + nChannel = self.data.shape[self.dimord.index("channel")] return np.array(["channel" + str(i + 1).zfill(len(str(nChannel))) - for i in range(nChannel)]) + for i in range(nChannel)]) return self._channel @channel.setter - def channel(self, channel): - + def channel(self, channel): + if channel is None: self._channel = None return - + if self.data is None: raise SPYValueError("Syncopy: Cannot assign `channels` without data. " + - "Please assign data first") - + "Please assign data first") + try: - array_parser(channel, varname="channel", ntype="str", + array_parser(channel, varname="channel", ntype="str", dims=(self.data.shape[self.dimord.index("channel")],)) except Exception as exc: raise exc - + self._channel = np.array(channel) @property @@ -171,7 +171,7 @@ def samplerate(self, sr): if sr is None: self._samplerate = None return - + try: scalar_parser(sr, varname="samplerate", lims=[np.finfo('float').eps, np.inf]) except Exception as exc: @@ -226,28 +226,28 @@ def _get_trial(self, trialno): sid = self.dimord.index("time") idx[sid] = slice(int(self.sampleinfo[trialno, 0]), int(self.sampleinfo[trialno, 1])) return self._data[tuple(idx)] - + def _is_empty(self): return super()._is_empty() or self.samplerate is None - - # Helper function that spawns a `FauxTrial` object given actual trial information + + # Helper function that spawns a `FauxTrial` object given actual trial information def _preview_trial(self, trialno): """ Generate a `FauxTrial` instance of a trial - + Parameters ---------- trialno : int Number of trial the `FauxTrial` object is intended to mimic - + Returns ------- faux_trl : :class:`syncopy.datatype.base_data.FauxTrial` An instance of :class:`syncopy.datatype.base_data.FauxTrial` mainly - intended to be used in `noCompute` runs of + intended to be used in `noCompute` runs of :meth:`syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` - to avoid loading actual trial-data into memory. - + to avoid loading actual trial-data into memory. + See also -------- syncopy.datatype.base_data.FauxTrial : class definition and further details @@ -260,15 +260,15 @@ def _preview_trial(self, trialno): start = int(self.sampleinfo[trialno, 0]) shp[tidx] = stop - start idx[tidx] = slice(start, stop) - + # process existing data selections if self._selection is not None: - + # time-selection is most delicate due to trial-offset tsel = self._selection.time[self._selection.trials.index(trialno)] if isinstance(tsel, slice): if tsel.start is not None: - tstart = tsel.start + tstart = tsel.start else: tstart = 0 if tsel.stop is not None: @@ -281,12 +281,12 @@ def _preview_trial(self, trialno): stop = start + (tstop - tstart) idx[tidx] = slice(start, stop) shp[tidx] = stop - start - + else: idx[tidx] = [tp + start for tp in tsel] shp[tidx] = len(tsel) - # process the rest + # process the rest for dim in ["channel", "freq", "taper"]: sel = getattr(self._selection, dim) if sel: @@ -308,38 +308,38 @@ def _preview_trial(self, trialno): idx[dimIdx] = slice(begin, end, delta) else: shp[dimIdx] = len(sel) - + return FauxTrial(shp, tuple(idx), self.data.dtype, self.dimord) - + # Helper function that extracts timing-related indices def _get_time(self, trials, toi=None, toilim=None): """ Get relative by-trial indices of time-selections - + Parameters ---------- trials : list List of trial-indices to perform selection on toi : None or list - Time-points to be selected (in seconds) on a by-trial scale. + Time-points to be selected (in seconds) on a by-trial scale. toilim : None or list Time-window to be selected (in seconds) on a by-trial scale - + Returns ------- timing : list of lists - List of by-trial sample-indices corresponding to provided + List of by-trial sample-indices corresponding to provided time-selection. If both `toi` and `toilim` are `None`, `timing` - is a list of universal (i.e., ``slice(None)``) selectors. - + is a list of universal (i.e., ``slice(None)``) selectors. + Notes ----- - This class method is intended to be solely used by - :class:`syncopy.datatype.base_data.Selector` objects and thus has purely + This class method is intended to be solely used by + :class:`syncopy.datatype.base_data.Selector` objects and thus has purely auxiliary character. Therefore, all input sanitization and error checking - is left to :class:`syncopy.datatype.base_data.Selector` and not - performed here. - + is left to :class:`syncopy.datatype.base_data.Selector` and not + performed here. + See also -------- syncopy.datatype.base_data.Selector : Syncopy data selectors @@ -353,7 +353,7 @@ def _get_time(self, trials, toi=None, toilim=None): timing.append(slice(selTime[0], selTime[-1] + 1, 1)) else: timing.append(selTime) - + elif toi is not None: for trlno in trials: _, selTime = best_match(self.time[trlno], toi) @@ -363,32 +363,32 @@ def _get_time(self, trials, toi=None, toilim=None): if timeSteps.min() == timeSteps.max() == 1: selTime = slice(selTime[0], selTime[-1] + 1, 1) timing.append(selTime) - + else: timing = [slice(None)] * len(trials) - + return timing # Make instantiation persistent in all subclasses - def __init__(self, data=None, channel=None, samplerate=None, **kwargs): - + def __init__(self, data=None, channel=None, samplerate=None, **kwargs): + self._channel = None self._samplerate = None self._data = None - + # Call initializer super().__init__(data=data, **kwargs) - + self.channel = channel - self.samplerate = samplerate # use setter for error-checking + self.samplerate = samplerate # use setter for error-checking self.data = data - + if self.data is not None: # In case of manual data allocation (reading routine would leave a # mark in `cfg`), fill in missing info if len(self.cfg) == 0: - + # First, fill in dimensional info definetrial(self, kwargs.get("trialdefinition")) @@ -401,23 +401,23 @@ class AnalogData(ContinuousData): position etc. The data is always stored as a two-dimensional array on disk. On disk, Trials are - concatenated along the time axis. + concatenated along the time axis. Data is only read from disk on demand, similar to memory maps and HDF5 files. """ - + _infoFileProperties = ContinuousData._infoFileProperties + ("_hdr",) _defaultDimord = ["time", "channel"] - + # Attach plotting routines to not clutter the core module code singlepanelplot = _plot_analog.singlepanelplot multipanelplot = _plot_analog.multipanelplot - + @property def hdr(self): """dict with information about raw data - + This property is empty for data created by Syncopy. """ return self._hdr @@ -426,19 +426,19 @@ def hdr(self): def selectdata(self, trials=None, channels=None, toi=None, toilim=None): """ Create new `AnalogData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - + + Please refer to :func:`syncopy.selectdata` for detailed usage information. + Examples -------- >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) - + See also -------- syncopy.selectdata : create new objects via deep-copy selections """ return selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim) - + # "Constructor" def __init__(self, data=None, @@ -448,14 +448,14 @@ def __init__(self, channel=None, dimord=None): """Initialize an :class:`AnalogData` object. - + Parameters ---------- - data : 2D :class:numpy.ndarray or HDF5 dataset - multi-channel time series data with uniform sampling + data : 2D :class:numpy.ndarray or HDF5 dataset + multi-channel time series data with uniform sampling filename : str path to target filename that should be used for writing - trialdefinition : :class:`EventData` object or Mx3 array + trialdefinition : :class:`EventData` object or Mx3 array [start, stop, trigger_offset] sample indices for `M` trials samplerate : float sampling rate in Hz @@ -465,17 +465,17 @@ def __init__(self, 1. `filename` + `data` : create hdf dataset incl. sampleinfo @filename 2. just `data` : try to attach data (error checking done by :meth:`AnalogData.data.setter`) - + See also -------- :func:`syncopy.definetrial` - + """ - # FIXME: I think escalating `dimord` to `BaseData` should be sufficient so that + # FIXME: I think escalating `dimord` to `BaseData` should be sufficient so that # the `if any(key...) loop in `BaseData.__init__()` takes care of assigning a default dimord if data is not None and dimord is None: - dimord = self._defaultDimord + dimord = self._defaultDimord # Assign default (blank) values self._hdr = None @@ -497,7 +497,7 @@ def __init__(self, # deep : bool # If `True`, a copy of the underlying data file is created in the temporary Syncopy folder - + # Returns # ------- # AnalogData @@ -510,7 +510,7 @@ def __init__(self, # """ # cpy = copy(self) - + # if deep: # if isinstance(self.data, VirtualData): # print("SyNCoPy core - copy: Deep copy not possible for " + @@ -531,14 +531,14 @@ class SpectralData(ContinuousData): and optionally a time axis. The datatype can be complex or float. """ - + _infoFileProperties = ContinuousData._infoFileProperties + ("taper", "freq",) _defaultDimord = ["time", "taper", "freq", "channel"] # Attach plotting routines to not clutter the core module code singlepanelplot = _plot_spectral.singlepanelplot multipanelplot = _plot_spectral.multipanelplot - + @property def taper(self): """ :class:`numpy.ndarray` : list of window functions used """ @@ -550,22 +550,21 @@ def taper(self): @taper.setter def taper(self, tpr): - + if tpr is None: self._taper = None return - + if self.data is None: print("Syncopy core - taper: Cannot assign `taper` without data. "+\ "Please assing data first") - return - + try: array_parser(tpr, dims=(self.data.shape[self.dimord.index("taper")],), varname="taper", ntype="str", ) except Exception as exc: raise exc - + self._taper = np.array(tpr) @property @@ -578,22 +577,22 @@ def freq(self): @freq.setter def freq(self, freq): - + if freq is None: self._freq = None return - + if self.data is None: print("Syncopy core - freq: Cannot assign `freq` without data. "+\ "Please assing data first") return try: - + array_parser(freq, varname="freq", hasnan=False, hasinf=False, dims=(self.data.shape[self.dimord.index("freq")],)) except Exception as exc: raise exc - + self._freq = np.array(freq) # Selector method @@ -601,24 +600,24 @@ def selectdata(self, trials=None, channels=None, toi=None, toilim=None, foi=None, foilim=None, tapers=None): """ Create new `SpectralData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - + + Please refer to :func:`syncopy.selectdata` for detailed usage information. + Examples -------- >>> spcBand = spc.selectdata(foilim=[10, 40]) - + See also -------- syncopy.selectdata : create new objects via deep-copy selections """ - return selectdata(self, trials=trials, channels=channels, toi=toi, + return selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim, foi=foi, foilim=foilim, tapers=tapers) - + # Helper function that extracts frequency-related indices def _get_freq(self, foi=None, foilim=None): """ - Coming soon... + Coming soon... Error checking is performed by `Selector` class """ if foilim is not None: @@ -626,7 +625,7 @@ def _get_freq(self, foi=None, foilim=None): selFreq = selFreq.tolist() if len(selFreq) > 1: selFreq = slice(selFreq[0], selFreq[-1] + 1, 1) - + elif foi is not None: _, selFreq = best_match(self.freq, foi) selFreq = selFreq.tolist() @@ -634,12 +633,12 @@ def _get_freq(self, foi=None, foilim=None): freqSteps = np.diff(selFreq) if freqSteps.min() == freqSteps.max() == 1: selFreq = slice(selFreq[0], selFreq[-1] + 1, 1) - + else: selFreq = slice(None) - + return selFreq - + # "Constructor" def __init__(self, data=None, @@ -653,11 +652,11 @@ def __init__(self, self._taper = None self._freq = None - + # FIXME: See similar comment above in `AnalogData.__init__()` if data is not None and dimord is None: dimord = self._defaultDimord - + # Call parent initializer super().__init__(data=data, filename=filename, @@ -684,3 +683,103 @@ def __init__(self, self.freq = [1] if taper is not None: self.taper = ['taper'] + +class CrossSpectralData(ContinuousData): + + _backingObject = None + _channel1 = None + _channel2 = None + + @property + def channel(self): + """ Linearized list of channel-channel combinations """ + if self._channel1 is None: + return None + return np.array([c1 + '-' + c2 for c1 in self._channel1 for c2 in self._channel2]) + + @channel.setter + def channel(self, channelTuple): + """ Set channel1 and channel2 """ + channel1, channel2 = channelTuple + if channel1 is channel2 is None: + SPYWarning("No channels provided for assignment", caller="CrossSpectralData") + return + if channel1 is None: + channel1 = channel2 + if channel2 is None: + channel2 = channel1 + self._channel1 = np.array(channel1) + self._channel2 = np.array(channel2) + + @property + def dimord(self): + """list(str): ordered list of data dimension labels""" + return self._dimord + + @dimord.setter + def dimord(self, dims): + """Override `dimord` setter from `BaseData`""" + if dims is not None: + try: + array_parser(dims, varname="dims", ntype="str", dims=1) + except Exception as exc: + raise exc + else: + return + + if self._dimord is not None: + lgl = "empty `dimord` attribute" + raise SPYValueError(legal=lgl, varname="dimord", actual=self._dimord) + + self._dimord = list(dims) + + # Override selector method + def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, + foi=None, foilim=None): + """ + Create new `CrossSpectralData` object from selection + + Please refer to :func:`syncopy.selectdata` for detailed usage information. + + Examples + -------- + >>> Coming soon + + See also + -------- + syncopy.selectdata : create new objects via deep-copy selections + """ + + return selectdata(self._backingObject, trials=trials, channels=channels, toi=toi, + toilim=toilim, foi=foi, foilim=foilim) + + def __init__(self, + data=None, + channel1=None, + channel2=None, + samplerate=None, + freq=None, + dimord=None): + + # If provided, build linear index so that backing object can be instantiated correctly + self.channel = (channel1, channel2) + + # Set dimensional labels + self.dimord = dimord + + # Allocate backing object + self._backingObject = SpectralData(data=data, + samplerate=samplerate, + channel=self.channel, + taper=None, + freq=freq) + + # Override class attributes: short-cut to backing object + self.data = self._backingObject.data + self.samplerate = self._backingObject.samplerate + self.freq = self._backingObject.freq + + # Override class helpers: short-cut to backing object + self._get_trial = self._backingObject._get_trial + + diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index b75334c24..cc2a275ca 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -23,4 +23,7 @@ if __name__ == "__main__": # Test stuff within here... - pass + artdata = generate_artificial_data(nTrials=5, nChannels=16, + equidistant=False, inmemory=False) + spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") + From 018c1f692c8f83eb4a2b329d0992e410a69ca244 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 3 Nov 2021 17:55:10 +0100 Subject: [PATCH 023/109] WIP: Integrating new CrossSpectralData and single trial compRoutine - added a few attributes missing due to composite class not calling the parent constructor, CrossSpectralData is now initializable - frontend calls for the single trial CR for the csd's - stuck at .compute(...) Changes to be committed: modified: ../dev_frontend.py renamed: connectivity/single_trial_compRoutines.py -> connectivity/ST_compRoutines.py modified: connectivity/connectivity_analysis.py modified: datatype/continuous_data.py --- dev_frontend.py | 18 ++-- ...ial_compRoutines.py => ST_compRoutines.py} | 85 ++++++++++++++++--- syncopy/connectivity/connectivity_analysis.py | 57 ++++++++----- syncopy/datatype/continuous_data.py | 17 +++- 4 files changed, 137 insertions(+), 40 deletions(-) rename syncopy/connectivity/{single_trial_compRoutines.py => ST_compRoutines.py} (83%) diff --git a/dev_frontend.py b/dev_frontend.py index 67dd5d62e..68de00de9 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -1,7 +1,9 @@ import numpy as np import scipy.signal as sci -from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF -from syncopy.connectivity.single_trial_compRoutines import cross_covariance_cF + +from syncopy.datatype import CrossSpectralData, padding, SpectralData +from syncopy.connectivity.ST_compRoutines import cross_spectra_cF, ST_CrossSpectra +from syncopy.connectivity.ST_compRoutines import cross_covariance_cF from syncopy.connectivity import connectivityanalysis from syncopy.specest import freqanalysis import matplotlib.pyplot as ppl @@ -23,13 +25,17 @@ print('sdict1') connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') -# connectivityanalysis(data=tdat, select=sdict1, pad_to_length=None) -print('sdict2') -# connectivityanalysis(data=tdat, select=sdict2) -print('no selection') + +# print('no selection') # connectivityanalysis(data=tdat, foi = np.arange(20, 80)) # connectivityanalysis(data=tdat, foilim = [20, 80]) +# the hard wired dimord of the cF +dimord = ['None', 'freq', 'channel1', 'channel2'] +# CrossSpectralData() +# CrossSpectralData(dimord=ST_CrossSpectra.dimord) +# SpectralData() + res = freqanalysis(data=tdat, method='mtmfft', diff --git a/syncopy/connectivity/single_trial_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py similarity index 83% rename from syncopy/connectivity/single_trial_compRoutines.py rename to syncopy/connectivity/ST_compRoutines.py index 6eff2c933..b15131ca3 100644 --- a/syncopy/connectivity/single_trial_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -8,6 +8,7 @@ # Builtin/3rd party package imports import numpy as np from scipy.signal import fftconvolve, detrend +from inspect import signature # syncopy imports from syncopy.specest.mtmfft import mtmfft @@ -29,6 +30,7 @@ def cross_spectra_cF(trl_dat, polyremoval=False, timeAxis=0, norm=False, + chunkShape=None, noCompute=False): """ @@ -117,35 +119,30 @@ def cross_spectra_cF(trl_dat, (Multi-)tapered Fourier analysis """ - + # Re-arrange array if necessary and get dimensional information if timeAxis != 0: dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - # detrend - if polyremoval == 0: - # SciPy's overwrite_data not working for type='constant' :/ - dat = detrend(dat, type='constant', axis=0, overwrite_data=True) - elif polyremoval == 1: - dat = detrend(dat, type='linear', axis=0, overwrite_data=True) - # Symmetric Padding (updates no. of samples) if padding_opt: dat = padding(dat, **padding_opt) - + nChannels = dat.shape[1] - # specs have shape (nTapers x nFreq x nChannels) - specs, freqs = mtmfft(dat, samplerate, taper, taperopt) + freqs = np.fft.rfftfreq(dat.shape[0], 1 / samplerate) + _, freq_idx = best_match(freqs, foi, squash_duplicates=True) + nFreq = freq_idx.size + if foi is not None: _, freq_idx = best_match(freqs, foi, squash_duplicates=True) nFreq = freq_idx.size else: freq_idx = slice(None) nFreq = freqs.size - + # we always average over tapers here outShape = (1, nFreq, nChannels, nChannels) @@ -155,6 +152,17 @@ def cross_spectra_cF(trl_dat, if noCompute: return outShape, spectralDTypes["fourier"] + # detrend + if polyremoval == 0: + # SciPy's overwrite_data not working for type='constant' :/ + dat = detrend(dat, type='constant', axis=0, overwrite_data=True) + elif polyremoval == 1: + dat = detrend(dat, type='linear', axis=0, overwrite_data=True) + + # compute the individual spectra + # specs have shape (nTapers x nFreq x nChannels) + specs, freqs = mtmfft(dat, samplerate, taper, taperopt) + # outer product along channel axes # has shape (nTapers x nFreq x nChannels x nChannels) CS_ij = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() @@ -176,6 +184,58 @@ def cross_spectra_cF(trl_dat, return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] +class ST_CrossSpectra(ComputationalRoutine): + + """ + Compute class that calculates single-trial (multi-)tapered cross spectra + of :class:`~syncopy.AnalogData` objects + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.connectivityanalysis : parent metafunction + """ + + # the hard wired dimord of the cF + dimord = ['None', 'freq', 'channel1', 'channel2'] + + computeFunction = staticmethod(cross_spectra_cF) + + method = "cross_spectra" + # 1st argument,the data, gets omitted + method_keys = list(signature(cross_spectra_cF).parameters.keys())[1:] + cF_keys = list(signature(cross_spectra_cF).parameters.keys())[1:] + + def process_metadata(self, data, out): + + # Some index gymnastics to get trial begin/end "samples" + if data._selection is not None: + chanSec = data._selection.channel + trl = data._selection.trialdefinition + for row in range(trl.shape[0]): + trl[row, :2] = [row, row + 1] + else: + chanSec = slice(None) + time = np.arange(len(data.trials)) + time = time.reshape((time.size, 1)) + trl = np.hstack((time, time + 1, + np.zeros((len(data.trials), 1)), + np.array(data.trialinfo))) + + # Attach constructed trialdef-array (if even necessary) + if self.keeptrials: + out.trialdefinition = trl + else: + out.trialdefinition = np.array([[0, 1, 0]]) + + # Attach remaining meta-data + out.samplerate = data.samplerate + out.channel = np.array(data.channel[chanSec]) + + @unwrap_io def cross_covariance_cF(trl_dat, samplerate=1, @@ -183,6 +243,7 @@ def cross_covariance_cF(trl_dat, polyremoval=False, timeAxis=0, norm=False, + chunkShape=None, noCompute=False): """ diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index e84f18a89..5a015fa3a 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -10,7 +10,7 @@ # Syncopy imports from syncopy.shared.parsers import data_parser, scalar_parser, array_parser from syncopy.shared.tools import get_defaults -from syncopy.datatype import SpectralData, padding +from syncopy.datatype import CrossSpectralData, padding from syncopy.datatype.methods.padding import _nextpow2 from syncopy.shared.tools import best_match from syncopy.shared.errors import ( @@ -31,9 +31,8 @@ ) # CRs still missing, CFs are already there -from .single_trial_compRoutines import ( - cross_spectra_cF, - cross_covariance_cF +from .ST_compRoutines import ( + ST_CrossSpectra ) __all__ = ["connectivityanalysis"] @@ -45,7 +44,7 @@ def connectivityanalysis(data, method='csd', foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", + nTaper=None, toi="all", out=None, **kwargs): """ @@ -177,7 +176,7 @@ def connectivityanalysis(data, method='csd', # only now set foi array for foilim in 1Hz steps if foilim: foi = np.arange(foilim[0], foilim[1] + 1) - + if method == 'csd': if foi is None and foilim is None: @@ -188,18 +187,36 @@ def connectivityanalysis(data, method='csd', SPYInfo(msg) foi = freqs - # for now manually select a trial - if data._selection is not None: - single_trial = data.trials[data._selection.trials] - else: - single_trial = data.trials[0] + st_CompRoutine = ST_CrossSpectra(samplerate=data.samplerate, + padding_opt=padding_opt, + foi=foi) + + # hard coded as class attribute + st_dimord = ST_CrossSpectra.dimord + + # -------------------------------------------------------- + # Sanitize output and call the chosen ComputationalRoutine + # -------------------------------------------------------- + + # If provided, make sure output object is appropriate + if out is not None: + try: + data_parser(out, varname="out", writable=True, empty=True, + dataclass="CrossSpectralData", + dimord=st_dimord) + except Exception as exc: + raise exc + new_out = False + else: + out = CrossSpectralData(dimord=st_dimord) + new_out = True + + # Perform actual computation + st_CompRoutine.initialize(data, + chan_per_worker=kwargs.get("chan_per_worker"), + keeptrials=True) + st_CompRoutine.compute(data, out, parallel=kwargs.get("parallel"), log_dict={}) + + # Either return newly created output object or simply quit + return out if new_out else None - res, freqs = cross_spectra_cF(single_trial, samplerate=data.samplerate, - padding_opt=padding_opt, foi=foi) - - print('A') - print(res.shape) - print(freqs[-10:]) - print(foi[-10:]) - print('B') - diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index e1aee0585..54f559c39 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -27,7 +27,7 @@ from syncopy.plotting import _plot_analog from syncopy.plotting import _plot_spectral -__all__ = ["AnalogData", "SpectralData"] +__all__ = ["AnalogData", "SpectralData", "CrossSpectralData"] class ContinuousData(BaseData, ABC): @@ -684,8 +684,11 @@ def __init__(self, if taper is not None: self.taper = ['taper'] + class CrossSpectralData(ContinuousData): + _defaultDimord = ["time", "freq", "channel1", "channel2"] + _backingObject = None _channel1 = None _channel2 = None @@ -700,7 +703,7 @@ def channel(self): @channel.setter def channel(self, channelTuple): """ Set channel1 and channel2 """ - channel1, channel2 = channelTuple + channel1, channel2 = channelTuple if channel1 is channel2 is None: SPYWarning("No channels provided for assignment", caller="CrossSpectralData") return @@ -719,6 +722,7 @@ def dimord(self): @dimord.setter def dimord(self, dims): """Override `dimord` setter from `BaseData`""" + if dims is not None: try: array_parser(dims, varname="dims", ntype="str", dims=1) @@ -755,6 +759,7 @@ def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toil def __init__(self, data=None, + filename=None, channel1=None, channel2=None, samplerate=None, @@ -764,6 +769,14 @@ def __init__(self, # If provided, build linear index so that backing object can be instantiated correctly self.channel = (channel1, channel2) + # as we don't call the parent contructor + # we have to initialize some things by hand + self._dimord = None + if filename is not None: + self.filename = filename + else: + self.filename = self._gen_filename() + # Set dimensional labels self.dimord = dimord From 501064c4232f0207ee5a00ffb5051d6521574052 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 5 Nov 2021 13:58:27 +0100 Subject: [PATCH 024/109] NEW: Updated CrossSpectralData class definition - included tuple-to-linear channel converter - modified class attributes wrt to backing object - fixed `dimord` field in pretty-print output of class objects (closes # 89) - wrapped up first draft of class definition (closes #143) On branch coherence Changes to be committed: modified: syncopy/connectivity/ST_compRoutines.py modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/methods/definetrial.py --- syncopy/connectivity/ST_compRoutines.py | 88 ++++++++++---------- syncopy/datatype/base_data.py | 12 +-- syncopy/datatype/continuous_data.py | 106 ++++++++++++++++++------ syncopy/datatype/methods/definetrial.py | 54 ++++++------ 4 files changed, 159 insertions(+), 101 deletions(-) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index b15131ca3..c9c2c0ead 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -30,23 +30,23 @@ def cross_spectra_cF(trl_dat, polyremoval=False, timeAxis=0, norm=False, - chunkShape=None, + chunkShape=None, noCompute=False): """ Single trial Fourier cross spectra estimates between all channels of the input data. First all the individual Fourier transforms are calculated via a (multi-)tapered FFT, then the pairwise - cross-spectra are computed. + cross-spectra are computed. Averaging over tapers is done implicitly for multi-taper analysis with `taper="dpss"`. - Output consists of all (nChannels x nChannels+1)/2 different complex - estimates aranged in a symmetric fashion (CS_ij == CS_ji*). The + Output consists of all (nChannels x nChannels+1)/2 different complex + estimates aranged in a symmetric fashion (CS_ij == CS_ji*). The elements on the main diagonal (CS_ii) are the (real) auto-spectra. - This is NOT the same as what is commonly referred to as + This is NOT the same as what is commonly referred to as "cross spectral density" as there is no (time) averaging!! Multi-tapering alone is not necessarily sufficient to get enough statitstical power for a robust csd estimate. Yet for completeness @@ -64,19 +64,19 @@ def cross_spectra_cF(trl_dat, Samplerate in Hz foi : 1D :class:`numpy.ndarray` or None, optional Frequencies of interest (Hz) for output. If desired frequencies - cannot be matched exactly the closest possible frequencies (respecting + cannot be matched exactly the closest possible frequencies (respecting data length and padding) are used. padding_opt : dict - Parameters to be used for padding. See :func:`syncopy.padding` for + Parameters to be used for padding. See :func:`syncopy.padding` for more details. taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. taperopt : dict, optional - Additional keyword arguments passed to the `taper` function. - For multi-tapering with `taper='dpss'` set the keys + Additional keyword arguments passed to the `taper` function. + For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. - For further details, please refer to the + For further details, please refer to the `SciPy docs `_ polyremoval : int or None Order of polynomial used for de-trending data in the time domain prior @@ -96,7 +96,7 @@ def cross_spectra_cF(trl_dat, array. Returns - ------- + ------- CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` Complex cross spectra for all channel combinations i,j. `N` corresponds to number of input channels. @@ -106,12 +106,12 @@ def cross_spectra_cF(trl_dat, Notes ----- - This method is intended to be used as + This method is intended to be used as :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` - inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. - Thus, input parameters are presumed to be forwarded from a parent metafunction. - Consequently, this function does **not** perform any error checking and operates - under the assumption that all inputs have been externally validated and cross-checked. + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. See also -------- @@ -135,17 +135,17 @@ def cross_spectra_cF(trl_dat, freqs = np.fft.rfftfreq(dat.shape[0], 1 / samplerate) _, freq_idx = best_match(freqs, foi, squash_duplicates=True) nFreq = freq_idx.size - + if foi is not None: _, freq_idx = best_match(freqs, foi, squash_duplicates=True) - nFreq = freq_idx.size + nFreq = freq_idx.size else: freq_idx = slice(None) nFreq = freqs.size # we always average over tapers here outShape = (1, nFreq, nChannels, nChannels) - + # For initialization of computational routine, # just return output shape and dtype # cross spectra are complex! @@ -160,15 +160,15 @@ def cross_spectra_cF(trl_dat, dat = detrend(dat, type='linear', axis=0, overwrite_data=True) # compute the individual spectra - # specs have shape (nTapers x nFreq x nChannels) + # specs have shape (nTapers x nFreq x nChannels) specs, freqs = mtmfft(dat, samplerate, taper, taperopt) - + # outer product along channel axes - # has shape (nTapers x nFreq x nChannels x nChannels) + # has shape (nTapers x nFreq x nChannels x nChannels) CS_ij = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() - + # average tapers and transpose: - # now has shape (nChannels x nChannels x nFreq) + # now has shape (nChannels x nChannels x nFreq) CS_ij = CS_ij.mean(axis=0).T if norm: @@ -179,7 +179,7 @@ def cross_spectra_cF(trl_dat, # get the needed product pairs of the autospectra Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj - + # where does freqs go/come from - we will allow tuples as return values yeah! return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] @@ -187,7 +187,7 @@ def cross_spectra_cF(trl_dat, class ST_CrossSpectra(ComputationalRoutine): """ - Compute class that calculates single-trial (multi-)tapered cross spectra + Compute class that calculates single-trial (multi-)tapered cross spectra of :class:`~syncopy.AnalogData` objects Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, @@ -200,8 +200,8 @@ class ST_CrossSpectra(ComputationalRoutine): """ # the hard wired dimord of the cF - dimord = ['None', 'freq', 'channel1', 'channel2'] - + dimord = ['freq', 'channel1', 'channel2'] + computeFunction = staticmethod(cross_spectra_cF) method = "cross_spectra" @@ -230,7 +230,7 @@ def process_metadata(self, data, out): out.trialdefinition = trl else: out.trialdefinition = np.array([[0, 1, 0]]) - + # Attach remaining meta-data out.samplerate = data.samplerate out.channel = np.array(data.channel[chanSec]) @@ -243,13 +243,13 @@ def cross_covariance_cF(trl_dat, polyremoval=False, timeAxis=0, norm=False, - chunkShape=None, + chunkShape=None, noCompute=False): """ Single trial covariance estimates between all channels - of the input data. Output consists of all (nChannels x nChannels+1)/2 - different estimates aranged in a symmetric fashion + of the input data. Output consists of all (nChannels x nChannels+1)/2 + different estimates aranged in a symmetric fashion (COV_ij == COV_ji). The elements on the main diagonal (CS_ii) are the channel variances. @@ -263,7 +263,7 @@ def cross_covariance_cF(trl_dat, samplerate : float Samplerate in Hz padding_opt : dict - Parameters to be used for padding. See :func:`syncopy.padding` for + Parameters to be used for padding. See :func:`syncopy.padding` for more details. polyremoval : int or None Order of polynomial used for de-trending data in the time domain prior @@ -281,19 +281,19 @@ def cross_covariance_cF(trl_dat, array. Returns - ------- + ------- CC_ij : (K, 1, N, N) :class:`numpy.ndarray` Cross covariance for all channel combinations i,j. `N` corresponds to number of input channels. Notes ----- - This method is intended to be used as + This method is intended to be used as :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` - inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. - Thus, input parameters are presumed to be forwarded from a parent metafunction. - Consequently, this function does **not** perform any error checking and operates - under the assumption that all inputs have been externally validated and cross-checked. + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. """ @@ -309,7 +309,7 @@ def cross_covariance_cF(trl_dat, dat = detrend(dat, type='constant', axis=0, overwrite_data=True) elif polyremoval == 1: detrend(dat, type='linear', axis=0, overwrite_data=True) - + # Symmetric Padding (updates no. of samples) if padding_opt: dat = padding(dat, **padding_opt) @@ -325,15 +325,15 @@ def cross_covariance_cF(trl_dat, lags = lags * 1 / samplerate outShape = (len(lags), 1, nChannels, nChannels) - + # For initialization of computational routine, # just return output shape and dtype # cross covariances are real! if noCompute: return outShape, spectralDTypes["abs"] - + # re-normalize output for different effective overlaps - norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) + norm_overlap = np.arange(nSamples, nSamples // 2, step = -1) CC = np.empty(outShape) for i in range(nChannels): @@ -350,5 +350,5 @@ def cross_covariance_cF(trl_dat, STDs = np.std(dat, axis=0) N = STDs[:, None] * STDs[None, :] CC = CC / N - + return CC, lags diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 822616b4b..7bbe3d8b2 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -71,6 +71,13 @@ class BaseData(ABC): # Set caller for `SPYWarning` to not have it show up as '' _spwCaller = "BaseData.{}" + # Initialize hidden attributes used by all children + _cfg = {} + _filename = None + _trialdefinition = None + _dimord = None + _mode = None + @property @classmethod @abstractmethod @@ -703,11 +710,6 @@ def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): """ # Initialize hidden attributes - self._cfg = {} - self._filename = None - self._trialdefinition = None - self._dimord = None - self._mode = None for propertyName in self._hdfFileDatasetProperties: setattr(self, "_" + propertyName, None) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 54f559c39..306a12486 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -77,16 +77,15 @@ def __str__(self): ppattrs.sort() # Construct string for pretty-printing class attributes - dsep = "' x '" - dinfo = "" + dsep = " by " hdstr = "Syncopy {clname:s} object with fields\n\n" - ppstr = hdstr.format(diminfo=dinfo + "'" + \ - dsep.join(dim for dim in self.dimord) + "' " if self.dimord is not None else "Empty ", - clname=self.__class__.__name__) + ppstr = hdstr.format(clname=self.__class__.__name__) maxKeyLength = max([len(k) for k in ppattrs]) printString = "{0:>" + str(maxKeyLength + 5) + "} : {1:}\n" for attr in ppattrs: value = getattr(self, attr) + if attr == "dimord" and value is not None: + valueString = dsep.join(dim for dim in self.dimord) if hasattr(value, 'shape') and attr == "data" and self.sampleinfo is not None: tlen = np.unique([sinfo[1] - sinfo[0] for sinfo in self.sampleinfo]) if tlen.size == 1: @@ -110,7 +109,10 @@ def __str__(self): valueString = "[" + " x ".join([str(numel) for numel in value.shape]) \ + "] element " + str(type(value)) elif isinstance(value, list): - valueString = "{0} element list".format(len(value)) + if attr == "dimord" and value is not None: + valueString = dsep.join(dim for dim in self.dimord) + else: + valueString = "{0} element list".format(len(value)) elif isinstance(value, dict): msg = "dictionary with {nk:s}keys{ks:s}" keylist = value.keys() @@ -525,11 +527,11 @@ def __init__(self, class SpectralData(ContinuousData): - """Multi-channel, real or complex spectral data + """ + Multi-channel, real or complex spectral data This class can be used for representing any data with a frequency, channel, and optionally a time axis. The datatype can be complex or float. - """ _infoFileProperties = ContinuousData._infoFileProperties + ("taper", "freq",) @@ -684,15 +686,22 @@ def __init__(self, if taper is not None: self.taper = ['taper'] - + class CrossSpectralData(ContinuousData): + """ + Multi-channel real or complex spectral connectivity data + + This class can be used for representing channel-channel interactions involving + frequency and optionally time or lag. The datatype can be complex or float. + """ _defaultDimord = ["time", "freq", "channel1", "channel2"] - + _backingObject = None _channel1 = None _channel2 = None + # Override channel: `CrossSpectralData` uses channel combinations @property def channel(self): """ Linearized list of channel-channel combinations """ @@ -700,10 +709,11 @@ def channel(self): return None return np.array([c1 + '-' + c2 for c1 in self._channel1 for c2 in self._channel2]) + # Override channel-setter as well @channel.setter def channel(self, channelTuple): """ Set channel1 and channel2 """ - channel1, channel2 = channelTuple + channel1, channel2 = channelTuple if channel1 is channel2 is None: SPYWarning("No channels provided for assignment", caller="CrossSpectralData") return @@ -714,11 +724,14 @@ def channel(self, channelTuple): self._channel1 = np.array(channel1) self._channel2 = np.array(channel2) + # Override dimord: since `CrossSpectralData` uses a "virtual" dimord we need some + # customizations @property def dimord(self): """list(str): ordered list of data dimension labels""" return self._dimord + # Override dimord setter as well @dimord.setter def dimord(self, dims): """Override `dimord` setter from `BaseData`""" @@ -737,6 +750,16 @@ def dimord(self, dims): self._dimord = list(dims) + # Override property so that setter points to backing object + @property + def trialdefinition(self): + """nTrials x >=3 :class:`numpy.ndarray` of [start, end, offset, trialinfo[:]]""" + return np.array(self._trialdefinition) + + @trialdefinition.setter + def trialdefinition(self, trl): + self.backingObject._definetrial(self._backingObject, trialdefinition=trl) + # Override selector method def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, foi=None, foilim=None): @@ -753,10 +776,37 @@ def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toil -------- syncopy.selectdata : create new objects via deep-copy selections """ - + channels = self._ind2sub(channels1, channels2) return selectdata(self._backingObject, trials=trials, channels=channels, toi=toi, toilim=toilim, foi=foi, foilim=foilim) + # Local 2d -> 1d channel index converter + def _ind2sub(self, channel1, channel2): + """Convert 2d channel tuple to linear 1d index""" + + chanIdx = [] + for channel in (channel1, channel2): + if isinstance(channel, (slice, range)): + channel = list(channel) + if all(isinstance(c, str) for c in channel): + target = self.channel + else: + target = np.arange(self.channel.size) + + # Use set comparison to ensure (a) no mixed-type selections (['a', 2, 'c']) + # and (b) no invalid selections ([-99, 0.01]) + if not set(channel).issubset(target): + lgl = "list/array of existing channel names or indices" + raise SPYValueError(legal=lgl, varname="channel") + + # Preserve order and duplicates of selection - don't use `np.isin` here! + chanIdx.append([np.where(target == c)[0] for c in channel]) + + # Almost: `ravel_multi_index` expects a tuple of arrays, so perform some zipping + linearIndex = [(c1, c2) for c1 in chanIdx[0] for c2 in chanIdx[1]] + return np.ravel_multi_index(tuple(zip(*linearIndex)), + dims=(self._channel1.size, self._channel2.size)) + def __init__(self, data=None, filename=None, @@ -769,30 +819,36 @@ def __init__(self, # If provided, build linear index so that backing object can be instantiated correctly self.channel = (channel1, channel2) - # as we don't call the parent contructor - # we have to initialize some things by hand - self._dimord = None - if filename is not None: - self.filename = filename - else: - self.filename = self._gen_filename() - # Set dimensional labels self.dimord = dimord + # We're not calling our parent constructor, so do this by hand + self.filename = self._gen_filename() + # Allocate backing object self._backingObject = SpectralData(data=data, + dimord=SpectralData._defaultDimord, + filename=self.filename, samplerate=samplerate, channel=self.channel, taper=None, freq=freq) - # Override class attributes: short-cut to backing object - self.data = self._backingObject.data - self.samplerate = self._backingObject.samplerate - self.freq = self._backingObject.freq + # Override class attributes: short-cut to backing object; Note: `filename` + # takes care of correctly pointing to `container` and `tag` + self._data = self._backingObject.data + self._freq = self._backingObject.freq + self._samplerate = self._backingObject.samplerate - # Override class helpers: short-cut to backing object + # Override class helpers: short-cut to backing object; Note: by pointing + # `_trialdefinition` to backing object, `sampleinfo`, `trialinfo` etc. + # are automatically processed correctly (all wrangle `_trialdefinition`) + self.definetrial = self._backingObject.definetrial self._get_trial = self._backingObject._get_trial + self._trialdefinition = self._backingObject._trialdefinition + + # The other way round: ensure on-disk backing device modes align + self.mode = "r+" + self._backingObject._mode = self._mode diff --git a/syncopy/datatype/methods/definetrial.py b/syncopy/datatype/methods/definetrial.py index c196d8064..6e0108263 100644 --- a/syncopy/datatype/methods/definetrial.py +++ b/syncopy/datatype/methods/definetrial.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Set/update trial settings of Syncopy data objects -# +# # Builtin/3rd party package imports import numbers @@ -18,7 +18,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, trigger=None, stop=None, clip_edges=False): """(Re-)define trials of a Syncopy data object - + Data can be structured into trials based on timestamps of a start, trigger and end events:: @@ -29,46 +29,46 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, Parameters ---------- obj : Syncopy data object (:class:`BaseData`-like) - trialdefinition : :class:`EventData` object or Mx3 array + trialdefinition : :class:`EventData` object or Mx3 array [start, stop, trigger_offset] sample indices for `M` trials pre : float offset time (s) before start event - post : float + post : float offset time (s) after end event start : int event code (id) to be used for start of trial stop : int event code (id) to be used for end of trial - trigger : - event code (id) to be used center (t=0) of trial + trigger : + event code (id) to be used center (t=0) of trial clip_edges : bool - trim trials to actual data-boundaries. + trim trials to actual data-boundaries. Returns ------- Syncopy data object (:class:`BaseData`-like)) - - + + Notes ----- :func:`definetrial` supports the following argument combinations: - + >>> # define M trials based on [start, end, offset] indices - >>> definetrial(obj, trialdefinition=[M x 3] array) + >>> definetrial(obj, trialdefinition=[M x 3] array) >>> # define trials based on event codes stored in <:class:`EventData` object> - >>> definetrial(obj, trialdefinition=, - pre=0, post=0, start=startCode, stop=stopCode, + >>> definetrial(obj, trialdefinition=, + pre=0, post=0, start=startCode, stop=stopCode, trigger=triggerCode) >>> # apply same trial definition as defined in <:class:`EventData` object> - >>> definetrial(, + >>> definetrial(, trialdefinition=) - >>> # define whole recording as single trial + >>> # define whole recording as single trial >>> definetrial(obj, trialdefinition=None) - + """ # Start by vetting input object @@ -95,17 +95,17 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, array_parser(trialdefinition, varname="trialdefinition", dims=2) except Exception as exc: raise exc - + if any(["ContinuousData" in str(base) for base in obj.__class__.__mro__]): scount = obj.data.shape[obj.dimord.index("time")] else: scount = np.inf try: - array_parser(trialdefinition[:, :2], varname="sampleinfo", dims=(None, 2), hasnan=False, + array_parser(trialdefinition[:, :2], varname="sampleinfo", dims=(None, 2), hasnan=False, hasinf=False, ntype="int_like", lims=[0, scount]) except Exception as exc: - raise exc - + raise exc + trl = np.array(trialdefinition, dtype="float") ref = obj tgt = obj @@ -139,7 +139,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, if any([kw is not None for kw in [pre, post, start, trigger, stop]]): # Make sure we actually have valid data objects to work with - if obj.__class__.__name__ == "EventData" and evt is False: + if obj.__class__.__name__ == "EventData" and evt is False: ref = obj tgt = obj elif obj.__class__.__name__ == "AnalogData" and evt is True: @@ -191,7 +191,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, kwrds = {} vdict = {"pre": {"var": pre, "hasnan": False, "ntype": None, "fillvalue": 0}, "post": {"var": post, "hasnan": False, "ntype": None, "fillvalue": 0}, - "start": {"var": start, "hasnan": None, "ntype": "int_like", "fillvalue": np.nan}, + "start": {"var": start, "hasnan": None, "ntype": "int_like", "fillvalue": np.nan}, "trigger": {"var": trigger, "hasnan": None, "ntype": "int_like", "fillvalue": np.nan}, "stop": {"var": stop, "hasnan": None, "ntype": "int_like", "fillvalue": np.nan}} for vname, opts in vdict.items(): @@ -244,7 +244,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, begin = evtsp[sidx]/ref.samplerate evtid[sidx] = -np.pi idxl.append(sidx) - + if not np.isnan(kwrds["trigger"][trialno]): try: idx = evtid.index(kwrds["trigger"][trialno]) @@ -285,7 +285,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, lgl = "non-overlapping trial begin-/end-samples" act = "trial-begin at {}, trial-end at {}".format(str(begin), str(end)) raise SPYValueError(legal=lgl, actual=act) - + # Finally, write line of `trl` trl.append([begin, end, t0]) @@ -317,7 +317,7 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, lgl = "non-overlapping trials" act = "some trials are overlapping after clipping to AnalogData object range" raise SPYValueError(legal=lgl, actual=act) - + # The triplet `sampleinfo`, `t0` and `trialinfo` works identically for # all data genres if trl.shape[1] < 3: @@ -355,5 +355,5 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, tgt.cfg = {"method" : sys._getframe().f_code.co_name, "EventData object": ref.cfg} ref.log = "updated trial-defnition of {} object".format(tgt.__class__.__name__) - + return From 3c5fc2a3bc2d388ae5eaea3d1cf180c739690513 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 5 Nov 2021 17:06:57 +0100 Subject: [PATCH 025/109] FIX: CrossSpectralData selections: all-by-all channel selections - modified `CrossSpectralData` to follow `Selector` logic: if no channels are provided for selection, do not exclude any and all channels but build linear indexing array to *include* everything (inclusion by default) - overload `_selection` and `_preview_trial` to deliberately fool `ComputationalRoutine` to work w/underlying "regular" `SpectralData` object On branch coherence Changes to be committed: modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/continuous_data.py | 45 ++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 306a12486..3ce1120cb 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -15,10 +15,11 @@ import numpy as np from abc import ABC from collections.abc import Iterator +from numpy.lib.arraysetops import isin from numpy.lib.format import open_memmap # Local imports -from .base_data import BaseData, FauxTrial +from .base_data import BaseData, FauxTrial, Selector from .methods.definetrial import definetrial from .methods.selectdata import selectdata from syncopy.shared.parsers import scalar_parser, array_parser @@ -750,6 +751,24 @@ def dimord(self, dims): self._dimord = list(dims) + # Override property so that setter points to backing object + @property + def _selection(self): + """Data selection specified by :class:`Selector`""" + return self._selector + + @_selection.setter + def _selection(self, select): + if select is None: + self._selector = None + else: + if "channels1" or "channels2" in select.keys(): + actualSelect = dict(select) + channels1 = actualSelect.pop("channels1", None) + channels2 = actualSelect.pop("channels2", None) + actualSelect["channels"] = self._ind2sub(channels1, channels2) + self._selector = Selector(self._backingObject, actualSelect) + # Override property so that setter points to backing object @property def trialdefinition(self): @@ -758,7 +777,7 @@ def trialdefinition(self): @trialdefinition.setter def trialdefinition(self, trl): - self.backingObject._definetrial(self._backingObject, trialdefinition=trl) + self._backingObject._definetrial(self._backingObject, trialdefinition=trl) # Override selector method def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, @@ -785,19 +804,28 @@ def _ind2sub(self, channel1, channel2): """Convert 2d channel tuple to linear 1d index""" chanIdx = [] - for channel in (channel1, channel2): - if isinstance(channel, (slice, range)): + for ck, channel in enumerate((channel1, channel2)): + target = getattr(self, "_channel{}".format(ck + 1)) + if isinstance(channel, str): + if channel == "all": + channel = None + else: + raise SPYValueError(legal="'all' or `None` or list/array", + varname="channels", actual=channel) + if channel is None: + channel = target + if isinstance(channel, range): channel = list(channel) - if all(isinstance(c, str) for c in channel): - target = self.channel - else: - target = np.arange(self.channel.size) + elif isinstance(channel, slice): + channel = target[channel] # Use set comparison to ensure (a) no mixed-type selections (['a', 2, 'c']) # and (b) no invalid selections ([-99, 0.01]) if not set(channel).issubset(target): lgl = "list/array of existing channel names or indices" raise SPYValueError(legal=lgl, varname="channel") + if not all(isinstance(c, str) for c in channel): + target = np.arange(target.size) # Preserve order and duplicates of selection - don't use `np.isin` here! chanIdx.append([np.where(target == c)[0] for c in channel]) @@ -846,6 +874,7 @@ def __init__(self, self.definetrial = self._backingObject.definetrial self._get_trial = self._backingObject._get_trial self._trialdefinition = self._backingObject._trialdefinition + self._preview_trial = self._backingObject._preview_trial # The other way round: ensure on-disk backing device modes align self.mode = "r+" From 925747a4144630096001b0801b7e3291583ceade Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 5 Nov 2021 17:43:46 +0100 Subject: [PATCH 026/109] WIP: CrossSpectralData --- dev_frontend.py | 4 ++-- syncopy/connectivity/ST_compRoutines.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 68de00de9..b9f66f48e 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -23,11 +23,11 @@ sdict2 = {"trials": [0], 'channels' : ['channel1'], 'toilim' : [-1, 0]} print('sdict1') -connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) +# connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') # print('no selection') -# connectivityanalysis(data=tdat, foi = np.arange(20, 80)) +connectivityanalysis(data=tdat)#, foi = np.arange(20, 80)) # connectivityanalysis(data=tdat, foilim = [20, 80]) # the hard wired dimord of the cF diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index c9c2c0ead..379bb79de 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -180,8 +180,9 @@ def cross_spectra_cF(trl_dat, Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj - # where does freqs go/come from - we will allow tuples as return values yeah! - return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] + # where does freqs go/come from - + # we will eventually allow tuples as return values yeah! + return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2)#, freqs[freq_idx] class ST_CrossSpectra(ComputationalRoutine): From d176cd814ada47ae2e0cde23c7a785e27a67b085 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 8 Nov 2021 14:27:24 +0100 Subject: [PATCH 027/109] FIX: Included specific overrides for class properties - simply "symlinking" the class attributes underlying properties (e.g., the attribute `_freq` holding the frequencies returned by the property `freq`) does not keep attributes in sync b/w the `CrossSpectralData` composite class and its `_backingObject`. This has been repaired by explicitly overloading the corresponding properties - moved attaching step of `_log` and `_log_header` out of `BaseData`'s constructor: assign attributes in class head so that they are available to all children (even without explicitly initializing `BaseData`) - fixed channel allocation of `CrossSpectralData` object in connectivity_analysis.py On branch coherence Changes to be committed: modified: dev_frontend.py modified: syncopy/connectivity/ST_compRoutines.py modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py --- dev_frontend.py | 6 +- syncopy/connectivity/ST_compRoutines.py | 2 +- syncopy/connectivity/connectivity_analysis.py | 40 ++++---- syncopy/datatype/base_data.py | 36 +++---- syncopy/datatype/continuous_data.py | 93 ++++++++++++++++--- 5 files changed, 120 insertions(+), 57 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index b9f66f48e..1a4cbdfa4 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -8,8 +8,8 @@ from syncopy.specest import freqanalysis import matplotlib.pyplot as ppl -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser -from syncopy.shared.tools import get_defaults +from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.tools import get_defaults from syncopy.datatype import SpectralData, padding from syncopy.tests.misc import generate_artificial_data @@ -27,7 +27,7 @@ # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') # print('no selection') -connectivityanalysis(data=tdat)#, foi = np.arange(20, 80)) +csd = connectivityanalysis(data=tdat)#, foi = np.arange(20, 80)) # connectivityanalysis(data=tdat, foilim = [20, 80]) # the hard wired dimord of the cF diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 379bb79de..ddae90a27 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -234,7 +234,7 @@ def process_metadata(self, data, out): # Attach remaining meta-data out.samplerate = data.samplerate - out.channel = np.array(data.channel[chanSec]) + out.channel = (np.array(data.channel[chanSec]), np.array(data.channel[chanSec])) @unwrap_io diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 5a015fa3a..b572d2586 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -8,7 +8,7 @@ from numbers import Number # Syncopy imports -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.parsers import data_parser, scalar_parser, array_parser from syncopy.shared.tools import get_defaults from syncopy.datatype import CrossSpectralData, padding from syncopy.datatype.methods.padding import _nextpow2 @@ -41,7 +41,7 @@ @unwrap_cfg @unwrap_select @detect_parallel_client -def connectivityanalysis(data, method='csd', +def connectivityanalysis(data, method='csd', foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, nTaper=None, toi="all", out=None, @@ -60,7 +60,7 @@ def connectivityanalysis(data, method='csd', timeAxis = data.dimord.index("time") # Get everything of interest in local namespace - defaults = get_defaults(connectivityanalysis) + defaults = get_defaults(connectivityanalysis) lcls = locals() # Ensure a valid computational method was selected @@ -82,12 +82,12 @@ def connectivityanalysis(data, method='csd', if isinstance(tsel, list): lgl = "equidistant time points (toi) or time slice (toilim)" actual = "non-equidistant set of time points" - raise SPYValueError(legal=lgl, varname="select", actual=actual) + raise SPYValueError(legal=lgl, varname="select", actual=actual) sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] else: trialList = list(range(len(data.trials))) - sinfo = data.sampleinfo + sinfo = data.sampleinfo lenTrials = np.diff(sinfo).squeeze() # here we enforce equal lengths trials as is required for @@ -99,19 +99,19 @@ def connectivityanalysis(data, method='csd', actual = "trials of different lengths - please pre-pad!" raise SPYValueError(legal=lgl, varname="lenTrials", actual=actual) - numTrials = len(trialList) - + numTrials = len(trialList) + print(lenTrials) - + # --- Padding --- - # manual symmetric zero padding of ALL trials the same way + # manual symmetric zero padding of ALL trials the same way if isinstance(pad_to_length, Number): scalar_parser(pad_to_length, varname='pad_to_length', ntype='int_like', - lims=[lenTrials.max(), np.inf]) + lims=[lenTrials.max(), np.inf]) padding_opt = { 'padtype' : 'zero', 'pad' : 'absolute', @@ -128,12 +128,12 @@ def connectivityanalysis(data, method='csd', # after padding nSamples = nextpow2(int(lenTrials.min())) # no padding - else: + else: padding_opt = None nSamples = int(lenTrials.min()) # --- foi sanitization --- - + if foi is not None: if isinstance(foi, str): if foi == "all": @@ -148,7 +148,7 @@ def connectivityanalysis(data, method='csd', except Exception as exc: raise exc foi = np.array(foi, dtype="float") - + if foilim is not None: if isinstance(foilim, str): if foilim == "all": @@ -172,21 +172,21 @@ def connectivityanalysis(data, method='csd', lgl = "either `foi` or `foilim` specification" act = "both" raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) - + # only now set foi array for foilim in 1Hz steps if foilim: foi = np.arange(foilim[0], foilim[1] + 1) - + if method == 'csd': if foi is None and foilim is None: # Construct array of maximally attainable frequencies freqs = np.fft.rfftfreq(nSamples, 1 / data.samplerate) - msg = (f"Automatic FFT frequency selection from {freqs[0]:.1f}Hz to " + msg = (f"Automatic FFT frequency selection from {freqs[0]:.1f}Hz to " f"{freqs[-1]:.1f}Hz") SPYInfo(msg) foi = freqs - + st_CompRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, foi=foi) @@ -197,7 +197,7 @@ def connectivityanalysis(data, method='csd', # -------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine # -------------------------------------------------------- - + # If provided, make sure output object is appropriate if out is not None: try: @@ -210,7 +210,7 @@ def connectivityanalysis(data, method='csd', else: out = CrossSpectralData(dimord=st_dimord) new_out = True - + # Perform actual computation st_CompRoutine.initialize(data, chan_per_worker=kwargs.get("chan_per_worker"), @@ -219,4 +219,4 @@ def connectivityanalysis(data, method='csd', # Either return newly created output object or simply quit return out if new_out else None - + diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 7bbe3d8b2..5f8f216cc 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -77,6 +77,23 @@ class BaseData(ABC): _trialdefinition = None _dimord = None _mode = None + _lhd = "\n\t\t>>> SyNCopy v. {ver:s} <<< \n\n" +\ + "Created: {timestamp:s} \n\n" +\ + "System Profile: \n" +\ + "{sysver:s} \n" +\ + "ACME: {acver:s}\n" +\ + "Dask: {daver:s}\n" +\ + "NumPy: {npver:s}\n" +\ + "SciPy: {spver:s}\n\n" +\ + "--- LOG ---" + _log_header = _lhd.format(ver=__version__, + timestamp=time.asctime(), + sysver=sys.version, + acver=acme.__version__ if __acme__ else "--", + daver=dask.__version__ if __acme__ else "--", + npver=np.__version__, + spver=sp.__version__) + _log = "" @property @classmethod @@ -740,24 +757,7 @@ def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): for propertyName in self._hdfFileDatasetProperties: setattr(self, propertyName, kwargs[propertyName]) - # Prepare log + header and write first entry - lhd = "\n\t\t>>> SyNCopy v. {ver:s} <<< \n\n" +\ - "Created: {timestamp:s} \n\n" +\ - "System Profile: \n" +\ - "{sysver:s} \n" +\ - "ACME: {acver:s}\n" +\ - "Dask: {daver:s}\n" +\ - "NumPy: {npver:s}\n" +\ - "SciPy: {spver:s}\n\n" +\ - "--- LOG ---" - self._log_header = lhd.format(ver=__version__, - timestamp=time.asctime(), - sysver=sys.version, - acver=acme.__version__ if __acme__ else "--", - daver=dask.__version__ if __acme__ else "--", - npver=np.__version__, - spver=sp.__version__) - self._log = "" + # Write initial log entry self.log = "created {clname:s} object".format(clname=self.__class__.__name__) # Write version diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 3ce1120cb..4962bca0c 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -697,7 +697,7 @@ class CrossSpectralData(ContinuousData): """ _defaultDimord = ["time", "freq", "channel1", "channel2"] - + _data = None _backingObject = None _channel1 = None _channel2 = None @@ -751,6 +751,54 @@ def dimord(self, dims): self._dimord = list(dims) + # Override data property: keep in sync w/`_backingObject` + @property + def data(self): + """ + Point to data property of `self._backingObject` + """ + return self._backingObject.data + + @data.setter + def data(self, inData): + self._backingObject.data = inData + + # Override freq property: keep in sync w/`_backingObject` + @property + def freq(self): + """ + Point to data property of `self._backingObject` + """ + return self._backingObject.freq + + @freq.setter + def freq(self, freq): + self._backingObject.freq = freq + + # Override mode property: keep in sync w/`_backingObject` + @property + def mode(self): + """ + Point to mode property of `self._backingObject` + """ + return self._backingObject.mode + + @mode.setter + def mode(self, md): + self._backingObject.mode = md + + # Override samplerate property: keep in sync w/`_backingObject` + @property + def samplerate(self): + """ + Point to samplerate property of `self._backingObject` + """ + return self._backingObject.samplerate + + @samplerate.setter + def samplerate(self, sr): + self._backingObject.samplerate = sr + # Override property so that setter points to backing object @property def _selection(self): @@ -769,15 +817,39 @@ def _selection(self, select): actualSelect["channels"] = self._ind2sub(channels1, channels2) self._selector = Selector(self._backingObject, actualSelect) + # Override property to point to backing object + @property + def sampleinfo(self): + """Point to sampleinfo property of `self._backingObject`""" + return self._backingObject.sampleinfo + + # Override property so that setter points to backing object + @property + def time(self): + """Point to time property of `self._backingObject`""" + return self._backingObject.time + # Override property so that setter points to backing object @property def trialdefinition(self): - """nTrials x >=3 :class:`numpy.ndarray` of [start, end, offset, trialinfo[:]]""" - return np.array(self._trialdefinition) + """Point to trialdefinition property of `self._backingObject`""" + return self._backingObject.trialdefinition @trialdefinition.setter def trialdefinition(self, trl): - self._backingObject._definetrial(self._backingObject, trialdefinition=trl) + self._backingObject.trialdefinition = trl + + # Override property to point to backing object + @property + def trialinfo(self): + """Point to trialinfo property of `self._backingObject`""" + return self._backingObject.trialinfo + + # Override property so that setter points to backing object + @property + def trials(self): + """Point to trials property of `self._backingObject`""" + return self._backingObject.trials # Override selector method def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, @@ -862,12 +934,6 @@ def __init__(self, taper=None, freq=freq) - # Override class attributes: short-cut to backing object; Note: `filename` - # takes care of correctly pointing to `container` and `tag` - self._data = self._backingObject.data - self._freq = self._backingObject.freq - self._samplerate = self._backingObject.samplerate - # Override class helpers: short-cut to backing object; Note: by pointing # `_trialdefinition` to backing object, `sampleinfo`, `trialinfo` etc. # are automatically processed correctly (all wrangle `_trialdefinition`) @@ -876,8 +942,5 @@ def __init__(self, self._trialdefinition = self._backingObject._trialdefinition self._preview_trial = self._backingObject._preview_trial - # The other way round: ensure on-disk backing device modes align - self.mode = "r+" - self._backingObject._mode = self._mode - - + # Manually create object log (due to uncalled parent constructor) + self.log = "created {clname:s} object".format(clname=self.__class__.__name__) From 331aac7f309d81445e810f26d8c658130143ec6a Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 8 Nov 2021 19:21:50 +0100 Subject: [PATCH 028/109] FIX: Repaired "src and dest dataspaces have different number of elements selected" bug - this was quite subtle: some (pathological) selections might leave a trial empty (e.g., selecting timings not present in the trial); as in the sequential case, take care of these empty source selections by not including `(0,*)`-shaped datasets into the virtual layout of the resulting dataset On branch coherence Changes to be committed: modified: syncopy/datatype/methods/selectdata.py modified: syncopy/shared/computational_routine.py --- syncopy/datatype/methods/selectdata.py | 258 ++++++++++++------------ syncopy/shared/computational_routine.py | 4 +- 2 files changed, 132 insertions(+), 130 deletions(-) diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 6e90dd6af..bcc78da54 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Syncopy data selection methods -# +# # Local imports from syncopy.shared.parsers import data_parser @@ -15,181 +15,181 @@ @unwrap_cfg @detect_parallel_client def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None, - foilim=None, tapers=None, units=None, eventids=None, + foilim=None, tapers=None, units=None, eventids=None, out=None, **kwargs): """ Create a new Syncopy object from a selection - **Usage Notice** - - Syncopy offers two modes for selecting data: - - * **in-place** selections mark subsets of a Syncopy data object for processing + **Usage Notice** + + Syncopy offers two modes for selecting data: + + * **in-place** selections mark subsets of a Syncopy data object for processing via a ``select`` dictionary *without* creating a new object - * **deep-copy** selections copy subsets of a Syncopy data object to keep and + * **deep-copy** selections copy subsets of a Syncopy data object to keep and preserve in a new object created by :func:`~syncopy.selectdata` - - All Syncopy metafunctions, such as :func:`~syncopy.freqanalysis`, support + + All Syncopy metafunctions, such as :func:`~syncopy.freqanalysis`, support **in-place** data selection via a ``select`` keyword, effectively avoiding - potentially slow copy operations and saving disk space. The keys accepted - by the `select` dictionary are identical to the keyword arguments discussed - below. In addition, ``select = "all"`` can be used to select entire object + potentially slow copy operations and saving disk space. The keys accepted + by the `select` dictionary are identical to the keyword arguments discussed + below. In addition, ``select = "all"`` can be used to select entire object contents. Examples - + >>> select = {"toilim" : [-0.25, 0]} >>> spy.freqanalysis(data, select=select) - >>> # or equivalently + >>> # or equivalently >>> cfg = spy.get_defaults(spy.freqanalysis) >>> cfg.select = select >>> spy.freqanalysis(cfg, data) - + **Usage Summary** - + List of Syncopy data objects and respective valid data selectors: - + :class:`~syncopy.AnalogData` : trials, channels, toi/toilim Examples - + >>> spy.selectdata(data, trials=[0, 3, 5], channels=["channel01", "channel02"]) - >>> cfg = spy.StructDict() + >>> cfg = spy.StructDict() >>> cfg.trials = [5, 3, 0]; cfg.toilim = [0.25, 0.5] >>> spy.selectdata(cfg, data) - + :class:`~syncopy.SpectralData` : trials, channels, toi/toilim, foi/foilim, tapers Examples - + >>> spy.selectdata(data, trials=[0, 3, 5], channels=["channel01", "channel02"]) >>> cfg = spy.StructDict() >>> cfg.foi = [30, 40, 50]; cfg.tapers = slice(2, 4) >>> spy.selectdata(cfg, data) - + :class:`~syncopy.EventData` : trials, toi/toilim, eventids Examples - + >>> spy.selectdata(data, toilim=[-1, 2.5], eventids=[0, 1]) >>> cfg = spy.StructDict() >>> cfg.trials = [0, 0, 1, 0]; cfg.eventids = slice(2, None) >>> spy.selectdata(cfg, data) - + :class:`~syncopy.SpikeData` : trials, toi/toilim, units, channels Examples - + >>> spy.selectdata(data, toilim=[-1, 2.5], units=range(0, 10)) >>> cfg = spy.StructDict() >>> cfg.toi = [1.25, 3.2]; cfg.trials = [0, 1, 2, 3] >>> spy.selectdata(cfg, data) - + **Note** Any property that is not specifically accessed via one of the provided selectors is taken as is, e.g., ``spy.selectdata(data, trials=[1, 2])`` - selects the entire contents of trials no. 2 and 3, while + selects the entire contents of trials no. 2 and 3, while ``spy.selectdata(data, channels=range(0, 50))`` selects the first 50 channels of `data` across all defined trials. Consequently, if no keywords are specified, - the entire contents of `data` is selected. - - **Full documentation below** - + the entire contents of `data` is selected. + + **Full documentation below** + Parameters ---------- data : Syncopy data object A non-empty Syncopy data object. **Note** the type of `data` determines - which keywords can be used. Some keywords are only valid for certain - types of Syncopy objects, e.g., "freqs" is not a valid selector for an - :class:`~syncopy.AnalogData` object. + which keywords can be used. Some keywords are only valid for certain + types of Syncopy objects, e.g., "freqs" is not a valid selector for an + :class:`~syncopy.AnalogData` object. trials : list (integers) or None or "all" - List of integers representing trial numbers to be selected; can include - repetitions and need not be sorted (e.g., ``trials = [0, 1, 0, 0, 2]`` - is valid) but must be finite and not NaN. If `trials` is `None`, or - ``trials = "all"`` all trials are selected. + List of integers representing trial numbers to be selected; can include + repetitions and need not be sorted (e.g., ``trials = [0, 1, 0, 0, 2]`` + is valid) but must be finite and not NaN. If `trials` is `None`, or + ``trials = "all"`` all trials are selected. channels : list (integers or strings), slice, range or None or "all" - Channel-selection; can be a list of channel names (``['channel3', 'channel1']``), - a list of channel indices (``[3, 5]``), a slice (``slice(3, 10)``) or - range (``range(3, 10)``). Note that following Python conventions, channels - are counted starting at zero, and range and slice selections are half-open - intervals of the form `[low, high)`, i.e., low is included , high is - excluded. Thus, ``channels = [0, 1, 2]`` or ``channels = slice(0, 3)`` - selects the first up to (and including) the third channel. Selections can - be unsorted and may include repetitions but must match exactly, be finite - and not NaN. If `channels` is `None`, or ``channels = "all"`` all channels - are selected. + Channel-selection; can be a list of channel names (``['channel3', 'channel1']``), + a list of channel indices (``[3, 5]``), a slice (``slice(3, 10)``) or + range (``range(3, 10)``). Note that following Python conventions, channels + are counted starting at zero, and range and slice selections are half-open + intervals of the form `[low, high)`, i.e., low is included , high is + excluded. Thus, ``channels = [0, 1, 2]`` or ``channels = slice(0, 3)`` + selects the first up to (and including) the third channel. Selections can + be unsorted and may include repetitions but must match exactly, be finite + and not NaN. If `channels` is `None`, or ``channels = "all"`` all channels + are selected. toi : list (floats) or None or "all" - Time-points to be selected (in seconds) in each trial. Timing is expected - to be on a by-trial basis (e.g., relative to trigger onsets). Selections - can be approximate, unsorted and may include repetitions but must be - finite and not NaN. Fuzzy matching is performed for approximate selections - (i.e., selected time-points are close but not identical to timing information - found in `data`) using a nearest-neighbor search for elements of `toi`. - If `toi` is `None` or ``toi = "all"``, the entire time-span in each trial - is selected. + Time-points to be selected (in seconds) in each trial. Timing is expected + to be on a by-trial basis (e.g., relative to trigger onsets). Selections + can be approximate, unsorted and may include repetitions but must be + finite and not NaN. Fuzzy matching is performed for approximate selections + (i.e., selected time-points are close but not identical to timing information + found in `data`) using a nearest-neighbor search for elements of `toi`. + If `toi` is `None` or ``toi = "all"``, the entire time-span in each trial + is selected. toilim : list (floats [tmin, tmax]) or None or "all" - Time-window ``[tmin, tmax]`` (in seconds) to be extracted from each trial. - Window specifications must be sorted (e.g., ``[2.2, 1.1]`` is invalid) - and not NaN but may be unbounded (e.g., ``[1.1, np.inf]`` is valid). Edges - `tmin` and `tmax` are included in the selection. - If `toilim` is `None` or ``toilim = "all"``, the entire time-span in each - trial is selected. + Time-window ``[tmin, tmax]`` (in seconds) to be extracted from each trial. + Window specifications must be sorted (e.g., ``[2.2, 1.1]`` is invalid) + and not NaN but may be unbounded (e.g., ``[1.1, np.inf]`` is valid). Edges + `tmin` and `tmax` are included in the selection. + If `toilim` is `None` or ``toilim = "all"``, the entire time-span in each + trial is selected. foi : list (floats) or None or "all" - Frequencies to be selected (in Hz). Selections can be approximate, unsorted - and may include repetitions but must be finite and not NaN. Fuzzy matching - is performed for approximate selections (i.e., selected frequencies are + Frequencies to be selected (in Hz). Selections can be approximate, unsorted + and may include repetitions but must be finite and not NaN. Fuzzy matching + is performed for approximate selections (i.e., selected frequencies are close but not identical to frequencies found in `data`) using a nearest- neighbor search for elements of `foi` in `data.freq`. If `foi` is `None` - or ``foi = "all"``, all frequencies are selected. + or ``foi = "all"``, all frequencies are selected. foilim : list (floats [fmin, fmax]) or None or "all" - Frequency-window ``[fmin, fmax]`` (in Hz) to be extracted. Window - specifications must be sorted (e.g., ``[90, 70]`` is invalid) and not NaN - but may be unbounded (e.g., ``[-np.inf, 60.5]`` is valid). Edges `fmin` - and `fmax` are included in the selection. If `foilim` is `None` or - ``foilim = "all"``, all frequencies are selected. + Frequency-window ``[fmin, fmax]`` (in Hz) to be extracted. Window + specifications must be sorted (e.g., ``[90, 70]`` is invalid) and not NaN + but may be unbounded (e.g., ``[-np.inf, 60.5]`` is valid). Edges `fmin` + and `fmax` are included in the selection. If `foilim` is `None` or + ``foilim = "all"``, all frequencies are selected. tapers : list (integers or strings), slice, range or None or "all" - Taper-selection; can be a list of taper names (``['dpss-win-1', 'dpss-win-3']``), - a list of taper indices (``[3, 5]``), a slice (``slice(3, 10)``) or range - (``range(3, 10)``). Note that following Python conventions, tapers are - counted starting at zero, and range and slice selections are half-open - intervals of the form `[low, high)`, i.e., low is included , high is - excluded. Thus, ``tapers = [0, 1, 2]`` or ``tapers = slice(0, 3)`` selects - the first up to (and including) the third taper. Selections can be unsorted - and may include repetitions but must match exactly, be finite and not NaN. - If `tapers` is `None` or ``tapers = "all"``, all tapers are selected. + Taper-selection; can be a list of taper names (``['dpss-win-1', 'dpss-win-3']``), + a list of taper indices (``[3, 5]``), a slice (``slice(3, 10)``) or range + (``range(3, 10)``). Note that following Python conventions, tapers are + counted starting at zero, and range and slice selections are half-open + intervals of the form `[low, high)`, i.e., low is included , high is + excluded. Thus, ``tapers = [0, 1, 2]`` or ``tapers = slice(0, 3)`` selects + the first up to (and including) the third taper. Selections can be unsorted + and may include repetitions but must match exactly, be finite and not NaN. + If `tapers` is `None` or ``tapers = "all"``, all tapers are selected. units : list (integers or strings), slice, range or None or "all" - Unit-selection; can be a list of unit names (``['unit10', 'unit3']``), a - list of unit indices (``[3, 5]``), a slice (``slice(3, 10)``) or range - (``range(3, 10)``). Note that following Python conventions, units are - counted starting at zero, and range and slice selections are half-open - intervals of the form `[low, high)`, i.e., low is included , high is - excluded. Thus, ``units = [0, 1, 2]`` or ``units = slice(0, 3)`` selects - the first up to (and including) the third unit. Selections can be unsorted + Unit-selection; can be a list of unit names (``['unit10', 'unit3']``), a + list of unit indices (``[3, 5]``), a slice (``slice(3, 10)``) or range + (``range(3, 10)``). Note that following Python conventions, units are + counted starting at zero, and range and slice selections are half-open + intervals of the form `[low, high)`, i.e., low is included , high is + excluded. Thus, ``units = [0, 1, 2]`` or ``units = slice(0, 3)`` selects + the first up to (and including) the third unit. Selections can be unsorted and may include repetitions but must match exactly, be finite and not NaN. - If `units` is `None` or ``units = "all"``, all units are selected. + If `units` is `None` or ``units = "all"``, all units are selected. eventids : list (integers), slice, range or None or "all" - Event-ID-selection; can be a list of event-id codes (``[2, 0, 1]``), slice - (``slice(0, 2)``) or range (``range(0, 2)``). Note that following Python - conventions, range and slice selections are half-open intervals of the - form `[low, high)`, i.e., low is included , high is excluded. Selections + Event-ID-selection; can be a list of event-id codes (``[2, 0, 1]``), slice + (``slice(0, 2)``) or range (``range(0, 2)``). Note that following Python + conventions, range and slice selections are half-open intervals of the + form `[low, high)`, i.e., low is included , high is excluded. Selections can be unsorted and may include repetitions but must match exactly, be - finite and not NaN. If `eventids` is `None` or ``eventids = "all"``, all - events are selected. - + finite and not NaN. If `eventids` is `None` or ``eventids = "all"``, all + events are selected. + Returns ------- dataselection : Syncopy data object - Syncopy data object of the same type as `data` but containing only the - subset specified by provided selectors. - + Syncopy data object of the same type as `data` but containing only the + subset specified by provided selectors. + Notes ----- This routine represents a convenience function for creating new Syncopy objects - based on existing data entities. However, in many situations, the creation - of a new object (and thus the allocation of additional disk-space) might not + based on existing data entities. However, in many situations, the creation + of a new object (and thus the allocation of additional disk-space) might not be necessary: all Syncopy metafunctions, such as :func:`~syncopy.freqanalysis`, - support **in-place** data selection. - - Consider the following example: assume `data` is an :class:`~syncopy.AnalogData` - object representing 220 trials of LFP recordings containing baseline (between - second -0.25 and 0) and stimulus-on data (on the interval [0.25, 0.5]). + support **in-place** data selection. + + Consider the following example: assume `data` is an :class:`~syncopy.AnalogData` + object representing 220 trials of LFP recordings containing baseline (between + second -0.25 and 0) and stimulus-on data (on the interval [0.25, 0.5]). To compute the baseline spectrum, data-selection does **not** have to be performed before calling :func:`~syncopy.freqanalysis` but instead can be done in-place: - + >>> import syncopy as spy >>> cfg = spy.get_defaults(spy.freqanalysis) >>> cfg.method = 'mtmfft' @@ -205,29 +205,29 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None >>> # in-place selection of stimulus-on time-frame performed by `freqanalysis` >>> cfg.select = stimSelect >>> stimonSpectrum = spy.freqanalysis(cfg, data) - + Especially for large data-sets, in-place data selection performed by Syncopy's - metafunctions does not only save disk-space but can significantly increase - performance. - + metafunctions does not only save disk-space but can significantly increase + performance. + Examples -------- - Use :func:`~syncopy.tests.misc.generate_artificial_data` to create a synthetic - :class:`syncopy.AnalogData` object. - + Use :func:`~syncopy.tests.misc.generate_artificial_data` to create a synthetic + :class:`syncopy.AnalogData` object. + >>> from syncopy.tests.misc import generate_artificial_data - >>> adata = generate_artificial_data(nTrials=10, nChannels=32) - + >>> adata = generate_artificial_data(nTrials=10, nChannels=32) + Assume a hypothetical trial onset at second 2.0 with the first second of each trial representing baseline recordings. To extract only the stimulus-on period from `adata`, one could use - + >>> stimon = spy.selectdata(adata, toilim=[2.0, np.inf]) - + Note that this is equivalent to - + >>> stimon = adata.selectdata(toilim=[2.0, np.inf]) - + See also -------- :meth:`syncopy.AnalogData.selectdata` : corresponding class method @@ -265,21 +265,21 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None "tapers": tapers, "units": units, "eventids": eventids} - - # Create inventory of all available selectors and actually provided values + + # Create inventory of all available selectors and actually provided values # to create a bookkeeping dict for logging provided = locals() available = get_defaults(data.selectdata) actualSelection = {} for key in available: actualSelection[key] = provided[key] - + # Fire up `ComputationalRoutine`-subclass to do the actual selecting/copying selectMethod = DataSelection() selectMethod.initialize(data, chan_per_worker=kwargs.get("chan_per_worker")) - selectMethod.compute(data, out, parallel=kwargs.get("parallel"), + selectMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=actualSelection) - + # Wipe data-selection slot to not alter input object data._selection = None @@ -299,14 +299,14 @@ class DataSelection(ComputationalRoutine): computeFunction = staticmethod(_selectdata) def process_metadata(self, data, out): - + # Get/set timing-related selection modifiers out.trialdefinition = data._selection.trialdefinition # if data._selection._timeShuffle: # FIXME: should be implemented done the road - # out.time = data._selection.timepoints + # out.time = data._selection.timepoints if data._selection._samplerate: out.samplerate = data.samplerate - + # Get/set dimensional attributes changed by selection for prop in data._selection._dimProps: selection = getattr(data._selection, prop) diff --git a/syncopy/shared/computational_routine.py b/syncopy/shared/computational_routine.py index 63f2e2b52..936f0bf1c 100644 --- a/syncopy/shared/computational_routine.py +++ b/syncopy/shared/computational_routine.py @@ -751,7 +751,9 @@ def preallocate_output(self, out, parallel_store=False): layout = h5py.VirtualLayout(shape=self.outputShape, dtype=self.dtype) for k, idx in enumerate(self.targetLayout): fname = os.path.join(self.virtualDatasetDir, "{0:d}.h5".format(k)) - layout[idx] = h5py.VirtualSource(fname, self.virtualDatasetNames, shape=self.targetShapes[k]) + # Catch empty selections: don't map empty sources into the layout of the VDS + if all([sel for sel in self.sourceLayout[k]]): + layout[idx] = h5py.VirtualSource(fname, self.virtualDatasetNames, shape=self.targetShapes[k]) self.VirtualDatasetLayout = layout self.outFileName = os.path.join(self.virtualDatasetDir, "{0:d}.h5") self.tmpDsetName = self.virtualDatasetNames From e3fd0935825f387b1ed64e598fb566b89bd8e86f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 9 Nov 2021 17:41:17 +0100 Subject: [PATCH 029/109] WIP: CrossSpectralData integration, computation - rewrote CrossSpectralData as derived class of ContrinuousData with dimord ['time', 'freq', 'channel_i', 'channel_j'] Works so far, there is just an empty channel attribute which might be better to move away from that base class - single trial CSD CR combined with keeptrials=False now returns the complex trial averages with correct shape and dimord. We 'just' need the normalization with the auto-spectra to finally arrive at the coherence :) - it is now possible to override the _defaultDimord (changed in the BaseClass ABC), yet the 1st entry always needs to be 'time' for the trial stacking of the CRs to work - added a fullOutput parameter to the connectivity cF's to already allow for multiple returns (for e.g. backend tests) Changes to be committed: modified: syncopy/connectivity/ST_compRoutines.py modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/specest/const_def.py modified: syncopy/tests/backend/test_connectivity.py --- dev_frontend.py | 16 +- syncopy/connectivity/ST_compRoutines.py | 38 +- syncopy/connectivity/connectivity_analysis.py | 6 +- syncopy/datatype/base_data.py | 15 +- syncopy/datatype/continuous_data.py | 340 ++++++++---------- syncopy/specest/const_def.py | 2 +- syncopy/tests/backend/test_connectivity.py | 8 +- 7 files changed, 201 insertions(+), 224 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 1a4cbdfa4..4fed89858 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -13,27 +13,28 @@ from syncopy.datatype import SpectralData, padding from syncopy.tests.misc import generate_artificial_data -tdat = generate_artificial_data(inmemory=True) +tdat = generate_artificial_data(inmemory=True, seed=1230) foilim = [30, 50] # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} # this gives type(tsel) = list # sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.array([0, 0.3, 1])} -sdict2 = {"trials": [0], 'channels' : ['channel1'], 'toilim' : [-1, 0]} +sdict2 = {"trials": [0], 'toilim' : [-1, 0]} print('sdict1') # connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') # print('no selection') -csd = connectivityanalysis(data=tdat)#, foi = np.arange(20, 80)) +csd = connectivityanalysis(data=tdat, keeptrials=False, foi = np.arange(20, 80)) +# csd = connectivityanalysis(data=tdat, keeptrials=False)#, select=sdict2) # connectivityanalysis(data=tdat, foilim = [20, 80]) # the hard wired dimord of the cF -dimord = ['None', 'freq', 'channel1', 'channel2'] +dimord = ['None', 'freq', 'channel_i', 'channel_j'] # CrossSpectralData() -# CrossSpectralData(dimord=ST_CrossSpectra.dimord) +# CrossSpectralData(dimord=dimord) # SpectralData() @@ -44,4 +45,7 @@ foi=np.arange(1, 150, 5), output='abs', # polyremoval=1, - t_ftimwin=0.5) + t_ftimwin=0.5, + keeptrials=False, + parallel=False, # try this!!!!!! + select={"trials" : [0,1]}) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index ddae90a27..ca5a40275 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -31,7 +31,8 @@ def cross_spectra_cF(trl_dat, timeAxis=0, norm=False, chunkShape=None, - noCompute=False): + noCompute=False, + fullOutput=False): """ Single trial Fourier cross spectra estimates between all channels @@ -94,6 +95,9 @@ def cross_spectra_cF(trl_dat, Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output array. + fullOutput : bool + For backend testing or stand-alone applications, set to `True` + to return also the `freqs` array. Returns ------- @@ -102,7 +106,7 @@ def cross_spectra_cF(trl_dat, `N` corresponds to number of input channels. freqs : (M,) :class:`numpy.ndarray` - The Fourier frequencies + The Fourier frequencies if `fullOutput=True` Notes ----- @@ -133,8 +137,6 @@ def cross_spectra_cF(trl_dat, nChannels = dat.shape[1] freqs = np.fft.rfftfreq(dat.shape[0], 1 / samplerate) - _, freq_idx = best_match(freqs, foi, squash_duplicates=True) - nFreq = freq_idx.size if foi is not None: _, freq_idx = best_match(freqs, foi, squash_duplicates=True) @@ -182,9 +184,12 @@ def cross_spectra_cF(trl_dat, # where does freqs go/come from - # we will eventually allow tuples as return values yeah! - return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2)#, freqs[freq_idx] - + if not fullOutput: + return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2) + else: + return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] + class ST_CrossSpectra(ComputationalRoutine): """ @@ -201,7 +206,7 @@ class ST_CrossSpectra(ComputationalRoutine): """ # the hard wired dimord of the cF - dimord = ['freq', 'channel1', 'channel2'] + dimord = ['time', 'freq', 'channel_i', 'channel_j'] computeFunction = staticmethod(cross_spectra_cF) @@ -231,10 +236,11 @@ def process_metadata(self, data, out): out.trialdefinition = trl else: out.trialdefinition = np.array([[0, 1, 0]]) - + # Attach remaining meta-data out.samplerate = data.samplerate - out.channel = (np.array(data.channel[chanSec]), np.array(data.channel[chanSec])) + out.channel_i = np.array(data.channel[chanSec]) + out.channel_j = np.array(data.channel[chanSec]) @unwrap_io @@ -245,7 +251,8 @@ def cross_covariance_cF(trl_dat, timeAxis=0, norm=False, chunkShape=None, - noCompute=False): + noCompute=False, + fullOutput=False): """ Single trial covariance estimates between all channels @@ -280,6 +287,9 @@ def cross_covariance_cF(trl_dat, Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output array. + fullOutput : bool + For backend testing or stand-alone applications, set to `True` + to return also the `lags` array. Returns ------- @@ -287,6 +297,9 @@ def cross_covariance_cF(trl_dat, Cross covariance for all channel combinations i,j. `N` corresponds to number of input channels. + lags : (M,) :class:`numpy.ndarray` + The lag times if `fullOutput=True` + Notes ----- This method is intended to be used as @@ -352,4 +365,7 @@ def cross_covariance_cF(trl_dat, N = STDs[:, None] * STDs[None, :] CC = CC / N - return CC, lags + if not fullOutput: + return CC + else: + return CC, lags diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index b572d2586..b9013ccc8 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -41,10 +41,10 @@ @unwrap_cfg @unwrap_select @detect_parallel_client -def connectivityanalysis(data, method='csd', +def connectivityanalysis(data, method='csd', keeptrials=False, foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", out=None, + nTaper=None, toi="all", out=None, **kwargs): """ @@ -214,7 +214,7 @@ def connectivityanalysis(data, method='csd', # Perform actual computation st_CompRoutine.initialize(data, chan_per_worker=kwargs.get("chan_per_worker"), - keeptrials=True) + keeptrials=keeptrials) st_CompRoutine.compute(data, out, parallel=kwargs.get("parallel"), log_dict={}) # Either return newly created output object or simply quit diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 5f8f216cc..d129c403c 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -345,10 +345,17 @@ def dimord(self, dims): self._dimord = None return - if set(dims) != set(self._defaultDimord): - base = "dimensional labels {}" - lgl = base.format("'" + "' x '".join(str(dim) for dim in self._defaultDimord) + "'") - act = base.format("'" + "' x '".join(str(dim) for dim in dims) + "'") + # this enforces the _defaultDimord + # if set(dims) != set(self._defaultDimord): + # base = "dimensional labels {}" + # lgl = base.format("'" + "' x '".join(str(dim) for dim in self._defaultDimord) + "'") + # act = base.format("'" + "' x '".join(str(dim) for dim in dims) + "'") + # raise SPYValueError(legal=lgl, varname="dimord", actual=act) + + # this enforces that custom dimords are set for every axis + if len(dims) != len(self._defaultDimord): + lgl = f"Custom dimord has length {len(self._defaultDimord)}" + act = f"Custom dimord has length {len(dims)}" raise SPYValueError(legal=lgl, varname="dimord", actual=act) # Canonical way to perform initial allocation of dimensional properties diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 4962bca0c..bdee99fcc 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -290,7 +290,9 @@ def _preview_trial(self, trialno): shp[tidx] = len(tsel) # process the rest - for dim in ["channel", "freq", "taper"]: + dims = list(self.dimord) + dims.pop(dims.index("time")) + for dim in dims: sel = getattr(self._selection, dim) if sel: dimIdx = self.dimord.index(dim) @@ -696,160 +698,119 @@ class CrossSpectralData(ContinuousData): frequency and optionally time or lag. The datatype can be complex or float. """ - _defaultDimord = ["time", "freq", "channel1", "channel2"] + _infoFileProperties = ContinuousData._infoFileProperties + ("freq",) + _defaultDimord = ["time", "freq", "channel_i", "channel_j"] + _channel_i = None + _channel_j = None + _samplerate = None _data = None - _backingObject = None - _channel1 = None - _channel2 = None - # Override channel: `CrossSpectralData` uses channel combinations + # override channel property to avoid accidental access @property def channel(self): - """ Linearized list of channel-channel combinations """ - if self._channel1 is None: - return None - return np.array([c1 + '-' + c2 for c1 in self._channel1 for c2 in self._channel2]) + pass + # msg = f"CrossSpectralData has no 'channel' but dimord: {self._dimord}" + # SPYWarning(msg) + # raise NotImplementedError(msg) - # Override channel-setter as well @channel.setter - def channel(self, channelTuple): - """ Set channel1 and channel2 """ - channel1, channel2 = channelTuple - if channel1 is channel2 is None: - SPYWarning("No channels provided for assignment", caller="CrossSpectralData") - return - if channel1 is None: - channel1 = channel2 - if channel2 is None: - channel2 = channel1 - self._channel1 = np.array(channel1) - self._channel2 = np.array(channel2) - - # Override dimord: since `CrossSpectralData` uses a "virtual" dimord we need some - # customizations - @property - def dimord(self): - """list(str): ordered list of data dimension labels""" - return self._dimord - - # Override dimord setter as well - @dimord.setter - def dimord(self, dims): - """Override `dimord` setter from `BaseData`""" - - if dims is not None: - try: - array_parser(dims, varname="dims", ntype="str", dims=1) - except Exception as exc: - raise exc + def channel(self, channel): + if channel is None: + # print('channel None setter called') + pass else: + msg = f"CrossSpectralData has no 'channel' to set but dimord: {self._dimord}" + raise NotImplementedError(msg) + + @property + def channel_i(self): + """ :class:`numpy.ndarray` : list of recording channel names """ + # if data exists but no user-defined channel labels, create them on the fly + if self._channel_i is None and self._data is not None: + nChannel = self.data.shape[self.dimord.index("channel_i")] + return np.array(["channel_i-" + str(i + 1).zfill(len(str(nChannel))) + for i in range(nChannel)]) + + return self._channel_i + + @channel_i.setter + def channel_i(self, channel_i): + """ :class:`numpy.ndarray` : list of channel labels """ + if channel_i is None: + self._channel_i = None return - if self._dimord is not None: - lgl = "empty `dimord` attribute" - raise SPYValueError(legal=lgl, varname="dimord", actual=self._dimord) + if self.data is None: + raise SPYValueError("Syncopy: Cannot assign `channels` without data. " + + "Please assign data first") - self._dimord = list(dims) + try: + array_parser(channel_i, varname="channel_i", ntype="str", + dims=(self.data.shape[self.dimord.index("channel_i")],)) + except Exception as exc: + raise exc + + self._channel_i = np.array(channel_i) - # Override data property: keep in sync w/`_backingObject` @property - def data(self): - """ - Point to data property of `self._backingObject` - """ - return self._backingObject.data + def channel_j(self): + """ :class:`numpy.ndarray` : list of recording channel names """ + # if data exists but no user-defined channel labels, create them on the fly + if self._channel_j is None and self._data is not None: + nChannel = self.data.shape[self.dimord.index("channel_j")] + return np.array(["channel_j-" + str(i + 1).zfill(len(str(nChannel))) + for i in range(nChannel)]) + + return self._channel_j + + @channel_j.setter + def channel_j(self, channel_j): + """ :class:`numpy.ndarray` : list of channel labels """ + if channel_j is None: + self._channel_j = None + return - @data.setter - def data(self, inData): - self._backingObject.data = inData + if self.data is None: + raise SPYValueError("Syncopy: Cannot assign `channels` without data. " + + "Please assign data first") + + try: + array_parser(channel_j, varname="channel_j", ntype="str", + dims=(self.data.shape[self.dimord.index("channel_j")],)) + except Exception as exc: + raise exc - # Override freq property: keep in sync w/`_backingObject` + self._channel_j = np.array(channel_j) + @property def freq(self): - """ - Point to data property of `self._backingObject` - """ - return self._backingObject.freq + """:class:`numpy.ndarray`: frequency axis in Hz """ + # if data exists but no user-defined frequency axis, + # create a dummy one on the fly + + if self._freq is None and self._data is not None: + return np.arange(self.data.shape[self.dimord.index("freq")]) + return self._freq @freq.setter def freq(self, freq): - self._backingObject.freq = freq - - # Override mode property: keep in sync w/`_backingObject` - @property - def mode(self): - """ - Point to mode property of `self._backingObject` - """ - return self._backingObject.mode - - @mode.setter - def mode(self, md): - self._backingObject.mode = md - - # Override samplerate property: keep in sync w/`_backingObject` - @property - def samplerate(self): - """ - Point to samplerate property of `self._backingObject` - """ - return self._backingObject.samplerate - - @samplerate.setter - def samplerate(self, sr): - self._backingObject.samplerate = sr - - # Override property so that setter points to backing object - @property - def _selection(self): - """Data selection specified by :class:`Selector`""" - return self._selector - - @_selection.setter - def _selection(self, select): - if select is None: - self._selector = None - else: - if "channels1" or "channels2" in select.keys(): - actualSelect = dict(select) - channels1 = actualSelect.pop("channels1", None) - channels2 = actualSelect.pop("channels2", None) - actualSelect["channels"] = self._ind2sub(channels1, channels2) - self._selector = Selector(self._backingObject, actualSelect) - - # Override property to point to backing object - @property - def sampleinfo(self): - """Point to sampleinfo property of `self._backingObject`""" - return self._backingObject.sampleinfo - # Override property so that setter points to backing object - @property - def time(self): - """Point to time property of `self._backingObject`""" - return self._backingObject.time - - # Override property so that setter points to backing object - @property - def trialdefinition(self): - """Point to trialdefinition property of `self._backingObject`""" - return self._backingObject.trialdefinition + if freq is None: + self._freq = None + return - @trialdefinition.setter - def trialdefinition(self, trl): - self._backingObject.trialdefinition = trl + if self.data is None: + print("Syncopy core - freq: Cannot assign `freq` without data. "+\ + "Please assing data first") + return + try: - # Override property to point to backing object - @property - def trialinfo(self): - """Point to trialinfo property of `self._backingObject`""" - return self._backingObject.trialinfo + array_parser(freq, varname="freq", hasnan=False, hasinf=False, + dims=(self.data.shape[self.dimord.index("freq")],)) + except Exception as exc: + raise exc - # Override property so that setter points to backing object - @property - def trials(self): - """Point to trials property of `self._backingObject`""" - return self._backingObject.trials + self._freq = np.array(freq) # Override selector method def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, @@ -867,80 +828,67 @@ def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toil -------- syncopy.selectdata : create new objects via deep-copy selections """ - channels = self._ind2sub(channels1, channels2) - return selectdata(self._backingObject, trials=trials, channels=channels, toi=toi, - toilim=toilim, foi=foi, foilim=foilim) - # Local 2d -> 1d channel index converter - def _ind2sub(self, channel1, channel2): - """Convert 2d channel tuple to linear 1d index""" + if channels1 is not None or channels2 is not None: + raise NotImplementedError("Channel selection not yet supported for CrossSpectralData") - chanIdx = [] - for ck, channel in enumerate((channel1, channel2)): - target = getattr(self, "_channel{}".format(ck + 1)) - if isinstance(channel, str): - if channel == "all": - channel = None - else: - raise SPYValueError(legal="'all' or `None` or list/array", - varname="channels", actual=channel) - if channel is None: - channel = target - if isinstance(channel, range): - channel = list(channel) - elif isinstance(channel, slice): - channel = target[channel] - - # Use set comparison to ensure (a) no mixed-type selections (['a', 2, 'c']) - # and (b) no invalid selections ([-99, 0.01]) - if not set(channel).issubset(target): - lgl = "list/array of existing channel names or indices" - raise SPYValueError(legal=lgl, varname="channel") - if not all(isinstance(c, str) for c in channel): - target = np.arange(target.size) - - # Preserve order and duplicates of selection - don't use `np.isin` here! - chanIdx.append([np.where(target == c)[0] for c in channel]) - - # Almost: `ravel_multi_index` expects a tuple of arrays, so perform some zipping - linearIndex = [(c1, c2) for c1 in chanIdx[0] for c2 in chanIdx[1]] - return np.ravel_multi_index(tuple(zip(*linearIndex)), - dims=(self._channel1.size, self._channel2.size)) + return selectdata(self, trials=trials, toi=toi, + toilim=toilim, foi=foi, foilim=foilim) + + # # Local 2d -> 1d channel index converter + # def _ind2sub(self, channel1, channel2): + # """Convert 2d channel tuple to linear 1d index""" + + # chanIdx = [] + # for ck, channel in enumerate((channel1, channel2)): + # target = getattr(self, "_channel{}".format(ck + 1)) + # if isinstance(channel, str): + # if channel == "all": + # channel = None + # else: + # raise SPYValueError(legal="'all' or `None` or list/array", + # varname="channels", actual=channel) + # if channel is None: + # channel = target + # if isinstance(channel, range): + # channel = list(channel) + # elif isinstance(channel, slice): + # channel = target[channel] + + # # Use set comparison to ensure (a) no mixed-type selections (['a', 2, 'c']) + # # and (b) no invalid selections ([-99, 0.01]) + # if not set(channel).issubset(target): + # lgl = "list/array of existing channel names or indices" + # raise SPYValueError(legal=lgl, varname="channel") + # if not all(isinstance(c, str) for c in channel): + # target = np.arange(target.size) + + # # Preserve order and duplicates of selection - don't use `np.isin` here! + # chanIdx.append([np.where(target == c)[0] for c in channel]) + + # # Almost: `ravel_multi_index` expects a tuple of arrays, so perform some zipping + # linearIndex = [(c1, c2) for c1 in chanIdx[0] for c2 in chanIdx[1]] + # return np.ravel_multi_index(tuple(zip(*linearIndex)), + # dims=(self._channel1.size, self._channel2.size)) def __init__(self, data=None, filename=None, - channel1=None, - channel2=None, + channel_i=None, + channel_j=None, samplerate=None, freq=None, dimord=None): - # If provided, build linear index so that backing object can be instantiated correctly - self.channel = (channel1, channel2) - # Set dimensional labels self.dimord = dimord + # set frequencies + self.freq = freq - # We're not calling our parent constructor, so do this by hand - self.filename = self._gen_filename() - - # Allocate backing object - self._backingObject = SpectralData(data=data, - dimord=SpectralData._defaultDimord, - filename=self.filename, - samplerate=samplerate, - channel=self.channel, - taper=None, - freq=freq) - - # Override class helpers: short-cut to backing object; Note: by pointing - # `_trialdefinition` to backing object, `sampleinfo`, `trialinfo` etc. - # are automatically processed correctly (all wrangle `_trialdefinition`) - self.definetrial = self._backingObject.definetrial - self._get_trial = self._backingObject._get_trial - self._trialdefinition = self._backingObject._trialdefinition - self._preview_trial = self._backingObject._preview_trial - - # Manually create object log (due to uncalled parent constructor) - self.log = "created {clname:s} object".format(clname=self.__class__.__name__) + # Call parent initializer + super().__init__(data=data, + filename=filename, + samplerate=samplerate, + freq=freq, + dimord=dimord) + diff --git a/syncopy/specest/const_def.py b/syncopy/specest/const_def.py index 2a20069a6..2be4b1e41 100644 --- a/syncopy/specest/const_def.py +++ b/syncopy/specest/const_def.py @@ -8,7 +8,7 @@ # Module-wide output specs spectralDTypes = {"pow": np.float32, - "fourier": np.complex128, + "fourier": np.complex64, "abs": np.float32} #: output conversion of complex fourier coefficients diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 8f10b2a87..3f8a6af70 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -2,7 +2,7 @@ import numpy as np import matplotlib.pyplot as ppl -from syncopy.connectivity import single_trial_compRoutines as stCR +from syncopy.connectivity import ST_compRoutines as stCR def test_csd(): @@ -24,7 +24,8 @@ def test_csd(): polyremoval=1, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, - norm=True) + norm=True, + fullOutput=True) # output has shape (1, nFreq, nChannels, nChannels) assert CSD.shape == (1, len(freqs), data.shape[1], data.shape[1]) @@ -64,7 +65,8 @@ def test_cross_cov(): data = np.c_[cosine, sine] # output shape is (nLags x 1 x nChannels x nChannels) - CC, lags = stCR.cross_covariance_cF(data, samplerate=fs, norm=True) + CC, lags = stCR.cross_covariance_cF(data, samplerate=fs, norm=True, fullOutput=True) + # test for result is returned in the [0, np.ceil(nSamples / 2)] lag interval nLags = int(np.ceil(nSamples / 2)) From faf759a9dfb390cbd09d42f123830ed7ba4894cb Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 9 Nov 2021 18:05:40 +0100 Subject: [PATCH 030/109] FIX: Allow for single trial selection in freqanalysis Changes to be committed: modified: syncopy/specest/freqanalysis.py --- syncopy/specest/freqanalysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index f6807ad68..70df65463 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -353,6 +353,8 @@ def freqanalysis(data, method='mtmfft', output='fourier', trialList = list(range(len(data.trials))) sinfo = data.sampleinfo lenTrials = np.diff(sinfo).squeeze() + if not lenTrials.shape: + lenTrials = lenTrials[None] numTrials = len(trialList) # Sliding window FFT does not support "fancy" padding From e02970028499df9e84ec15fbe3c8b29545da1848 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 10 Nov 2021 16:54:21 +0100 Subject: [PATCH 031/109] NEW: Coherence estimate from single trial MTM cross-spectra - new CR's and CF's for post-processing the parallelised single trial routines in 'AV_compRoutines'. They are intended to get called directly within the frontend on the ST_compRoutines results to perform computations after trial averaging. Right now only the `Norm_CrossMeasures` is implemented which computes coherences/correlations from the respective matrix quantities. Their input is intended to be strictly 1-Trial (after averaging) and don't trivially support parallelisation over channels, hence parallel=False is hardcoded for this post-processing methods. - Added backend test for the new trial averaged coherence - converted all remaining `np.complex128` to `np.complex64` in specest for consistency Changes to be committed: modified: ../../dev_frontend.py new file: AV_compRoutines.py modified: ST_compRoutines.py modified: connectivity_analysis.py modified: const_def.py modified: ../specest/const_def.py modified: ../specest/freqanalysis.py modified: ../specest/mtmfft.py modified: ../tests/backend/test_connectivity.py --- dev_frontend.py | 6 +- syncopy/connectivity/AV_compRoutines.py | 183 ++++++++++++++++++ syncopy/connectivity/ST_compRoutines.py | 3 +- syncopy/connectivity/connectivity_analysis.py | 45 +++-- syncopy/connectivity/const_def.py | 10 +- syncopy/specest/const_def.py | 2 +- syncopy/specest/freqanalysis.py | 2 +- syncopy/specest/mtmfft.py | 2 +- syncopy/tests/backend/test_connectivity.py | 86 +++++++- 9 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 syncopy/connectivity/AV_compRoutines.py diff --git a/dev_frontend.py b/dev_frontend.py index 4fed89858..8532787af 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -13,7 +13,7 @@ from syncopy.datatype import SpectralData, padding from syncopy.tests.misc import generate_artificial_data -tdat = generate_artificial_data(inmemory=True, seed=1230) +tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=25) foilim = [30, 50] # this still gives type(tsel) = slice :) @@ -27,7 +27,7 @@ # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') # print('no selection') -csd = connectivityanalysis(data=tdat, keeptrials=False, foi = np.arange(20, 80)) +coherence = connectivityanalysis(data=tdat, keeptrials=False, foi = np.arange(1, 70)) # csd = connectivityanalysis(data=tdat, keeptrials=False)#, select=sdict2) # connectivityanalysis(data=tdat, foilim = [20, 80]) @@ -36,7 +36,7 @@ # CrossSpectralData() # CrossSpectralData(dimord=dimord) # SpectralData() - +print('s') res = freqanalysis(data=tdat, method='mtmfft', diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py new file mode 100644 index 000000000..5d2e3bbb0 --- /dev/null +++ b/syncopy/connectivity/AV_compRoutines.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# +# computeFunctions and -Routines to post-process +# the parallel single trial computations to be found in ST_compRoutines.py +# The standard use case involves computations on the +# trial average, meaning that the SyNCoPy input to these routines +# consists of only '1 trial` and parallelising over channels +# is non trivial and atm also not supported. Pre-processing +# like padding or detrending already happened in the single trial +# compute functions. +# + +# Builtin/3rd party package imports +import numpy as np +from inspect import signature + +# syncopy imports +from syncopy.specest.const_def import spectralDTypes, spectralConversions +from syncopy.shared.errors import SPYWarning +from syncopy.shared.computational_routine import ComputationalRoutine +from syncopy.shared.kwarg_decorators import unwrap_io +from syncopy.shared.errors import ( + SPYValueError, + SPYTypeError, + SPYWarning, + SPYInfo) + + +@unwrap_io +def normalize_csd_cF(trl_dat, + output='abs', + chunkShape=None, + noCompute=False): + + """ + Given the trial averaged cross spectral densities, + calculates the normalizations to arrive at the + channel x channel coherencies. If S_ij(f) is the + averaged cross-spectrum between channel i and j, the + coherency [1]_ is defined as: + + C_ij = S_ij(f) / (|S_ii| |S_jj|) + + The coherence is now defined as either |C_ij| + or |C_ij|^2, this can be controlled with the `output` + parameter. + + Parameters + ---------- + trl_dat : (1, nFreq, N, N) :class:`numpy.ndarray` + Cross-spectral densities for `N` x `N` channels + and `nFreq` frequencies. + output : {'abs', 'pow', 'fourier'}, default: 'abs' + Also after normalization the coherency is still complex (`'fourier'`), + to get the real valued coherence 0 < C_ij(f) < 1 one can either take the + absolute (`'abs'`) or the absolute squared (`'pow'`) values of the + coherencies. The definitions are not uniform in the literature, + hence multiple output types are supported. + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + + Returns + ------- + CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` + Coherence for all channel combinations i,j. + `N` corresponds to number of input channels. + + Notes + ----- + + This function also normalizes cross-covariances to cross-correlations. + + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + .. [1] Nolte, Guido, et al. "Identifying true brain interaction from EEG + data using the imaginary part of coherency." + Clinical neurophysiology 115.10 (2004): 2292-2307. + + + See also + -------- + cross_spectra_cF : :func:`~syncopy.connectivity.ST_compRoutines.cross_spectra_cF` + Single trial (Multi-)tapered cross spectral densities. + + """ + + # it's the same as the input shape! + outShape = trl_dat.shape + + # For initialization of computational routine, + # just return output shape and dtype + # cross spectra are complex! + if noCompute: + return outShape, spectralDTypes[output] + + # re-shape to (nChannels x nChannels x nFreq) + CS_ij = trl_dat.transpose(0, 2, 3, 1)[0, ...] + + # main diagonal has shape (nChannels x nFreq): the auto spectra + diag = CS_ij.diagonal() + # get the needed product pairs of the autospectra + Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T + CS_ij = CS_ij / Ciijj + + CS_ij = spectralConversions[output](CS_ij) + + # re-shape to original form and re-attach dummy time axis + return CS_ij[None, ...].transpose(0, 3, 1, 2) + + +class Normalize_CrossMeasure(ComputationalRoutine): + + """ + Compute class that normalizes trial averaged quantities + of :class:`~syncopy.CrossSpectralData` objects + like cross-spectra or cross-covariances to arrive at + coherencies or cross-correlations respectively. + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.connectivityanalysis : parent metafunction + """ + + # the hard wired dimord of the cF + dimord = ['time', 'freq', 'channel_i', 'channel_j'] + + computeFunction = staticmethod(normalize_csd_cF) + + method = "" # there is no backend + # 1st argument,the data, gets omitted + method_keys = {} + cF_keys = list(signature(normalize_csd_cF).parameters.keys())[1:] + + def pre_check(self): + ''' + Make sure we have a trial average, + so the input data only consists of `1 trial`. + Can only be performed after initialization! + ''' + + if self.numTrials != 1: + lgl = "1 trial: normalizations can only be done on averaged quantities!" + act = f"DataSet contains {self.numTrials} trials" + raise SPYValueError(legal=lgl, varname="data", actual=act) + + def process_metadata(self, data, out): + + # Some index gymnastics to get trial begin/end "samples" + if data._selection is not None: + chanSec = data._selection.channel + trl = data._selection.trialdefinition + for row in range(trl.shape[0]): + trl[row, :2] = [row, row + 1] + else: + chanSec = slice(None) + time = np.arange(len(data.trials)) + time = time.reshape((time.size, 1)) + trl = np.hstack((time, time + 1, + np.zeros((len(data.trials), 1)), + np.array(data.trialinfo))) + + # Attach constructed trialdef-array (if even necessary) + if self.keeptrials: + out.trialdefinition = trl + else: + out.trialdefinition = np.array([[0, 1, 0]]) + + # Attach remaining meta-data + out.samplerate = data.samplerate + out.channel_i = np.array(data.channel_i[chanSec]) + out.channel_j = np.array(data.channel_j[chanSec]) + out.freq = data.freq diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index ca5a40275..fc550fcce 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -105,7 +105,7 @@ def cross_spectra_cF(trl_dat, Complex cross spectra for all channel combinations i,j. `N` corresponds to number of input channels. - freqs : (M,) :class:`numpy.ndarray` + freqs : (nFreq,) :class:`numpy.ndarray` The Fourier frequencies if `fullOutput=True` Notes @@ -241,6 +241,7 @@ def process_metadata(self, data, out): out.samplerate = data.samplerate out.channel_i = np.array(data.channel[chanSec]) out.channel_j = np.array(data.channel[chanSec]) + out.freq = self.cfg['foi'] @unwrap_io diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index b9013ccc8..b8fa86fbf 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -34,14 +34,16 @@ from .ST_compRoutines import ( ST_CrossSpectra ) - +from .AV_compRoutines import ( + Normalize_CrossMeasure +) __all__ = ["connectivityanalysis"] @unwrap_cfg @unwrap_select @detect_parallel_client -def connectivityanalysis(data, method='csd', keeptrials=False, +def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, nTaper=None, toi="all", out=None, @@ -101,8 +103,6 @@ def connectivityanalysis(data, method='csd', keeptrials=False, numTrials = len(trialList) - print(lenTrials) - # --- Padding --- # manual symmetric zero padding of ALL trials the same way @@ -187,18 +187,36 @@ def connectivityanalysis(data, method='csd', keeptrials=False, SPYInfo(msg) foi = freqs - st_CompRoutine = ST_CrossSpectra(samplerate=data.samplerate, + # parallel computation over trials + st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, foi=foi) - + # hard coded as class attribute st_dimord = ST_CrossSpectra.dimord + + # final normalization after trial averaging + av_compRoutine = Normalize_CrossMeasure(output=output) + + # ------------------------------------------------- + # Call the chosen single trial ComputationalRoutine + # ------------------------------------------------- + + # the single trial results need a new DataSet + st_out = CrossSpectralData(dimord=st_dimord) + + + # Perform the trial-parallelized computation of the matrix quantity + st_compRoutine.initialize(data, + chan_per_worker=None, # no parallelisation over channel possible + keeptrials=False) # we need trial averaging! + st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) # -------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine # -------------------------------------------------------- - # If provided, make sure output object is appropriate + # If provided, make sure output object is appropriate if out is not None: try: data_parser(out, varname="out", writable=True, empty=True, @@ -210,13 +228,12 @@ def connectivityanalysis(data, method='csd', keeptrials=False, else: out = CrossSpectralData(dimord=st_dimord) new_out = True - - # Perform actual computation - st_CompRoutine.initialize(data, - chan_per_worker=kwargs.get("chan_per_worker"), - keeptrials=keeptrials) - st_CompRoutine.compute(data, out, parallel=kwargs.get("parallel"), log_dict={}) - + + # now take the trial average from the single trial CR as input + av_compRoutine.initialize(st_out, chan_per_worker=None) + av_compRoutine.pre_check() # make sure we got a trial_average + av_compRoutine.compute(st_out, out, parallel=False) + # Either return newly created output object or simply quit return out if new_out else None diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py index 9f1e23ff7..6b9400fa4 100644 --- a/syncopy/connectivity/const_def.py +++ b/syncopy/connectivity/const_def.py @@ -7,8 +7,14 @@ import numpy as np # Module-wide output specs -spectralDTypes = {"complex": np.complex64, - "real": np.float32} +spectralDTypes = {"pow": np.float32, + "fourier": np.complex64, + "abs": np.float32} + +#: output conversion of complex fourier coefficients +spectralConversions = {"pow": lambda x: (x * np.conj(x)).real.astype(np.float32), + "fourier": lambda x: x.astype(np.complex64), + "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} #: available tapers of :func:`~syncopy.connectivity_analysis` availableTapers = ("hann", "dpss") diff --git a/syncopy/specest/const_def.py b/syncopy/specest/const_def.py index 2be4b1e41..886f2dd22 100644 --- a/syncopy/specest/const_def.py +++ b/syncopy/specest/const_def.py @@ -13,7 +13,7 @@ #: output conversion of complex fourier coefficients spectralConversions = {"pow": lambda x: (x * np.conj(x)).real.astype(np.float32), - "fourier": lambda x: x.astype(np.complex128), + "fourier": lambda x: x.astype(np.complex64), "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} #: available outputs of :func:`~syncopy.freqanalysis` diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 70df65463..586a58d04 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -142,7 +142,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', output : str Output of spectral estimation. One of :data:`~syncopy.specest.const_def.availableOutputs` (see below); use `'pow'` for power spectrum (:obj:`numpy.float32`), `'fourier'` for complex - Fourier coefficients (:obj:`numpy.complex128`) or `'abs'` for absolute + Fourier coefficients (:obj:`numpy.complex64`) or `'abs'` for absolute values (:obj:`numpy.float32`). keeptrials : bool If `True` spectral estimates of individual trials are returned, otherwise diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index faecc9cff..94f4f2a67 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -80,7 +80,7 @@ def mtmfft(data_arr, samplerate, taper="hann", taperopt={}): windows = windows * np.sqrt(nSamples) / np.sum(windows) # Fourier transforms (nTapers x nFreq x nChannels) - ftr = np.zeros((windows.shape[0], nFreq, nChannels), dtype='complex128') + ftr = np.zeros((windows.shape[0], nFreq, nChannels), dtype='complex64') for taperIdx, win in enumerate(windows): win = np.tile(win, (nChannels, 1)).T diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 3f8a6af70..b22a896fb 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -3,10 +3,94 @@ import numpy as np import matplotlib.pyplot as ppl from syncopy.connectivity import ST_compRoutines as stCR +from syncopy.connectivity import AV_compRoutines as avCR +def test_coherence(): + + ''' + Tests the normalization cF to + arrive at the coherence given + a trial averaged csd + ''' + + nSamples = 1001 + fs = 1000 + tvec = np.arange(nSamples) / fs + harm_freq = 40 + phase_shifts = np.array([0, np.pi / 2, np.pi]) + + nTrials = 50 + + # shape is (1, nFreq, nChannel, nChannel) + nFreq = nSamples // 2 + 1 + nChannel = len(phase_shifts) + avCSD = np.zeros((1, nFreq, nChannel, nChannel), dtype=np.complex64) + + for i in range(nTrials): + + # 1 phase phase shifted harmonics + white noise + constant, SNR = 1 + trl_dat = [10 + np.cos(harm_freq * 2 * np. pi * tvec + ps) + for ps in phase_shifts] + trl_dat = np.array(trl_dat).T + trl_dat = np.array(trl_dat) + np.random.randn(nSamples, len(phase_shifts)) + + # process every trial individually + CSD, freqs = stCR.cross_spectra_cF(trl_dat, fs, + polyremoval=1, + taper='hann', + norm=False, # this is important! + fullOutput=True) + + assert avCSD.shape == CSD.shape + avCSD += CSD + + # this is the result of the + avCSD /= nTrials + + # perform the normalisation on the trial averaged csd's + Cij = avCR.normalize_csd_cF(avCSD) + + # output has shape (1, nFreq, nChannels, nChannels) + assert Cij.shape == avCSD.shape + + # coherence between channel 0 and 1 + coh = Cij[0, :, 0, 1] + + fig, ax = ppl.subplots(figsize=(6,4), num=None) + ax.set_xlabel('frequency (Hz)') + ax.set_ylabel('coherence') + ax.set_ylim((-.02,1.05)) + ax.set_title('Trial average coherence, SNR=1') + + assert ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') + + # we test for the highest peak sitting at + # the vicinity (± 5Hz) of one the harmonic + peak_val = np.max(coh) + peak_idx = np.argmax(coh) + peak_freq = freqs[peak_idx] + print(peak_freq, peak_val) + assert harm_freq - 5 < peak_freq < harm_freq + 5 + + # we test that the peak value + # is at least 0.9 and max 1 + assert 0.9 < peak_val < 1 + + # trial averaging should suppress the noise + # we test that away from the harmonic the coherence is low + level = 0.4 + assert np.all(coh[:peak_idx - 2] < level) + assert np.all(coh[peak_idx + 2:] < level) + + def test_csd(): + ''' + Tests multi-tapered single trial cross spectral + densities + ''' + nSamples = 1001 fs = 1000 tvec = np.arange(nSamples) / fs @@ -39,7 +123,7 @@ def test_csd(): ax.set_ylim((-.02,1.05)) ax.set_title(f'MTM coherence, {Kmax} tapers, SNR=1') - assert ax.plot(freqs, coh, lw=2, alpha=0.8, c='cornflowerblue') + assert ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') # we test for the highest peak sitting at # the vicinity (± 5Hz) of one the harmonic From 155473efd6538fbce41d4c86c53022581415f919 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 11 Nov 2021 16:50:07 +0100 Subject: [PATCH 032/109] WIP: Sanitizing tapsmofrq and scipy's slepians - it's better to remove the free parameter nTape, as this can completely ruin the frequency smoothing, regardless of tapsmofrq settings. Better to always set it automatically similary to how it's now done if nTaper=None. - scaling for the scipy dpss then is NW = tapsmofrq * N /(2 fs), see also #58 --- dev_frontend.py | 8 ++-- dev_tapsmofrq.py | 44 +++++++++++++++++++ syncopy/connectivity/AV_compRoutines.py | 21 +++++---- syncopy/connectivity/ST_compRoutines.py | 2 +- syncopy/connectivity/connectivity_analysis.py | 21 ++++++--- syncopy/tests/backend/test_timefreq.py | 4 +- 6 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 dev_tapsmofrq.py diff --git a/dev_frontend.py b/dev_frontend.py index 8532787af..48c35e628 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -15,7 +15,7 @@ from syncopy.tests.misc import generate_artificial_data tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=25) -foilim = [30, 50] +foilim = [1, 100] # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} # this gives type(tsel) = list @@ -27,7 +27,9 @@ # connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') # print('no selection') -coherence = connectivityanalysis(data=tdat, keeptrials=False, foi = np.arange(1, 70)) +coherence = connectivityanalysis(data=tdat, + keeptrials=False, foilim=foilim, + output='pow') # csd = connectivityanalysis(data=tdat, keeptrials=False)#, select=sdict2) # connectivityanalysis(data=tdat, foilim = [20, 80]) @@ -42,7 +44,7 @@ method='mtmfft', samplerate=tdat.samplerate, order_max=20, - foi=np.arange(1, 150, 5), + foilim=foilim, output='abs', # polyremoval=1, t_ftimwin=0.5, diff --git a/dev_tapsmofrq.py b/dev_tapsmofrq.py new file mode 100644 index 000000000..f2ade2c39 --- /dev/null +++ b/dev_tapsmofrq.py @@ -0,0 +1,44 @@ +from syncopy.specest import mtmfft +import numpy as np +import matplotlib.pyplot as ppl +from scipy.signal import windows + +fs = 1000 +# superposition 40Hz and 100Hz oscillations A1:A2 for 1s +f1, f2 = 40, 100 +A1, A2 = 5, 3 +tvec = np.arange(0, 2, 1 / 1000) + +signal = A1 * np.cos(2 * np.pi * f1 * tvec) +signal += A2 * np.cos(2 * np.pi * f2 * tvec) + +# the transforms have shape (nTaper, nFreq, nChannel) +ftr, freqs = mtmfft.mtmfft(signal, fs, taper=None) + +# average over potential tapers (only 1 here) +spec = np.real(ftr * ftr.conj()).mean(axis=0) +amplitudes = np.sqrt(spec)[:, 0] # only 1 channel + +N = len(tvec) +minBw = 2 * fs / N +Bw = 10 +W = Bw / 2 # Hz +NW = W * N / fs +Kmax = int(2 * NW - 1) + +taperopt = {'Kmax' : Kmax, 'NW' : NW} +ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) +# average over tapers +dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) +dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel +# check for amplitudes (and taper normalisation) + + +fig, ax = ppl.subplots() +ax.set_title(f"Amplitude spectrum {A1} x {f1}Hz + {A2} x {f2}Hz, {Bw}Hz smoothing") +ax.plot(freqs[:250], amplitudes[:250], label="No taper", lw=2) +ax.set_xlabel('frequency (Hz)') +ax.set_ylabel('amplitude (a.u.)') + +ax.plot(freqs[:250], dpss_amplitudes[:250], label="Slepian", lw=2) +ax.legend() diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 5d2e3bbb0..d7105269e 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -27,7 +27,7 @@ @unwrap_io -def normalize_csd_cF(trl_dat, +def normalize_csd_cF(trl_av_dat, output='abs', chunkShape=None, noCompute=False): @@ -47,15 +47,15 @@ def normalize_csd_cF(trl_dat, Parameters ---------- - trl_dat : (1, nFreq, N, N) :class:`numpy.ndarray` + trl_av_dat : (1, nFreq, N, N) :class:`numpy.ndarray` Cross-spectral densities for `N` x `N` channels - and `nFreq` frequencies. - output : {'abs', 'pow', 'fourier'}, default: 'abs' + and `nFreq` frequencies averaged over trials. + output : {'abs', 'pow', 'fourier', 'corr'}, default: 'abs' Also after normalization the coherency is still complex (`'fourier'`), to get the real valued coherence 0 < C_ij(f) < 1 one can either take the absolute (`'abs'`) or the absolute squared (`'pow'`) values of the coherencies. The definitions are not uniform in the literature, - hence multiple output types are supported. + hence multiple output types are supported. noCompute : bool Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output @@ -92,7 +92,7 @@ def normalize_csd_cF(trl_dat, """ # it's the same as the input shape! - outShape = trl_dat.shape + outShape = trl_av_dat.shape # For initialization of computational routine, # just return output shape and dtype @@ -101,7 +101,7 @@ def normalize_csd_cF(trl_dat, return outShape, spectralDTypes[output] # re-shape to (nChannels x nChannels x nFreq) - CS_ij = trl_dat.transpose(0, 2, 3, 1)[0, ...] + CS_ij = trl_av_dat.transpose(0, 2, 3, 1)[0, ...] # main diagonal has shape (nChannels x nFreq): the auto spectra diag = CS_ij.diagonal() @@ -142,12 +142,17 @@ class Normalize_CrossMeasure(ComputationalRoutine): method_keys = {} cF_keys = list(signature(normalize_csd_cF).parameters.keys())[1:] - def pre_check(self): + def check_input(self): ''' Make sure we have a trial average, so the input data only consists of `1 trial`. Can only be performed after initialization! ''' + + if self.numTrials is None: + lgl = 'Initialize the computational Routine first!' + act = 'ComputationalRoutine not initialized!' + raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) if self.numTrials != 1: lgl = "1 trial: normalizations can only be done on averaged quantities!" diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index fc550fcce..56bcfb6ca 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -248,7 +248,7 @@ def process_metadata(self, data, out): def cross_covariance_cF(trl_dat, samplerate=1, padding_opt={}, - polyremoval=False, + polyremoval=0, timeAxis=0, norm=False, chunkShape=None, diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index b8fa86fbf..c5ae546e5 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -187,9 +187,20 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", SPYInfo(msg) foi = freqs + # Warn user about DPSS only settings + if taper != "dpss": + if tapsmofrq is not None: + msg = "`tapsmofrq` is only used if `taper` is `dpss`!" + SPYWarning(msg) + + # parallel computation over trials st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, + taper=taper, + taperopt=taperopt, + polyremoval=polyremoval, + timeAxis=timeAxis, foi=foi) # hard coded as class attribute @@ -212,9 +223,9 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", keeptrials=False) # we need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) - # -------------------------------------------------------- - # Sanitize output and call the chosen ComputationalRoutine - # -------------------------------------------------------- + # ---------------------------------------------------------------------------------- + # Sanitize output and call the chosen ComputationalRoutine on the averaged ST output + # ---------------------------------------------------------------------------------- # If provided, make sure output object is appropriate if out is not None: @@ -228,10 +239,10 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", else: out = CrossSpectralData(dimord=st_dimord) new_out = True - + # now take the trial average from the single trial CR as input av_compRoutine.initialize(st_out, chan_per_worker=None) - av_compRoutine.pre_check() # make sure we got a trial_average + av_compRoutine.check_input() # make sure we got a trial_average av_compRoutine.compute(st_out, out, parallel=False) # Either return newly created output object or simply quit diff --git a/syncopy/tests/backend/test_timefreq.py b/syncopy/tests/backend/test_timefreq.py index 9e1866479..d2ea8ed87 100644 --- a/syncopy/tests/backend/test_timefreq.py +++ b/syncopy/tests/backend/test_timefreq.py @@ -340,8 +340,8 @@ def test_mtmfft(): A1, A2 = 5, 3 tvec = np.arange(0, 1, 1 / 1000) - signal = A1 * np.cos(2 * np.pi * 40 * tvec) - signal += A2 * np.cos(2 * np.pi * 100 * tvec) + signal = A1 * np.cos(2 * np.pi * f1 * tvec) + signal += A2 * np.cos(2 * np.pi * f2 * tvec) # -------------------- # -- test untapered -- From c8ebf79b257cc127ede2779d84abacc602ecb375 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 12 Nov 2021 16:13:34 +0100 Subject: [PATCH 033/109] FIX: Multi-taper frequency smoothing and genereal input validators - fixed tapsmofrq setting for the dpss windows, such that the optimal number of tapers is automatically chosen and tapsmofreq now indeed is the spectral smoothing bandwidth in Hz. Users can still input nTaper manually but receive a stern warning, see also #58 - as now the number of automatically chosen tapers changed, had to adjust the tests accordingly - implemented 2 general input validators to be used by all frontends which support foi or taper selection, such as to follow DRY - increased the number of supported tapers Changes to be committed: modified: dev_frontend.py modified: dev_tapsmofrq.py modified: syncopy/connectivity/connectivity_analysis.py new file: syncopy/shared/input_validators.py modified: syncopy/specest/compRoutines.py modified: syncopy/specest/const_def.py modified: syncopy/specest/freqanalysis.py modified: syncopy/tests/test_specest.py --- dev_frontend.py | 16 +- dev_tapsmofrq.py | 35 ++-- syncopy/connectivity/connectivity_analysis.py | 2 +- syncopy/shared/input_validators.py | 158 ++++++++++++++++++ syncopy/specest/compRoutines.py | 57 +++---- syncopy/specest/const_def.py | 6 +- syncopy/specest/freqanalysis.py | 123 +++----------- syncopy/tests/test_specest.py | 22 +-- 8 files changed, 252 insertions(+), 167 deletions(-) create mode 100644 syncopy/shared/input_validators.py diff --git a/dev_frontend.py b/dev_frontend.py index 48c35e628..e8f206eb3 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -15,7 +15,7 @@ from syncopy.tests.misc import generate_artificial_data tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=25) -foilim = [1, 100] +foilim = [1, 30] # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} # this gives type(tsel) = list @@ -38,16 +38,20 @@ # CrossSpectralData() # CrossSpectralData(dimord=dimord) # SpectralData() -print('s') res = freqanalysis(data=tdat, method='mtmfft', samplerate=tdat.samplerate, - order_max=20, - foilim=foilim, - output='abs', +# order_max=20, +# foilim=foilim, +# foi=np.arange(502), + output='pow', # polyremoval=1, - t_ftimwin=0.5, +# t_ftimwin=0.5, keeptrials=False, + taper='dpss', + nTaper = 11, + tapsmofrq=5, +# t_ftimwin=0.5, parallel=False, # try this!!!!!! select={"trials" : [0,1]}) diff --git a/dev_tapsmofrq.py b/dev_tapsmofrq.py index f2ade2c39..c3397daa4 100644 --- a/dev_tapsmofrq.py +++ b/dev_tapsmofrq.py @@ -7,10 +7,10 @@ # superposition 40Hz and 100Hz oscillations A1:A2 for 1s f1, f2 = 40, 100 A1, A2 = 5, 3 -tvec = np.arange(0, 2, 1 / 1000) +tvec = np.arange(0, 4.096, 1 / 1000) signal = A1 * np.cos(2 * np.pi * f1 * tvec) -signal += A2 * np.cos(2 * np.pi * f2 * tvec) +#signal += A2 * np.cos(2 * np.pi * f2 * tvec) # the transforms have shape (nTaper, nFreq, nChannel) ftr, freqs = mtmfft.mtmfft(signal, fs, taper=None) @@ -19,26 +19,37 @@ spec = np.real(ftr * ftr.conj()).mean(axis=0) amplitudes = np.sqrt(spec)[:, 0] # only 1 channel +fig, ax = ppl.subplots() +ax.set_xlabel('frequency (Hz)') +ax.set_ylabel('amplitude (a.u.)') + +ax.plot(freqs[:400], amplitudes[:400], label="No taper", lw=2) + N = len(tvec) minBw = 2 * fs / N -Bw = 10 +Bw = 15 W = Bw / 2 # Hz NW = W * N / fs Kmax = int(2 * NW - 1) - taperopt = {'Kmax' : Kmax, 'NW' : NW} +print(taperopt, N) ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) # average over tapers dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel # check for amplitudes (and taper normalisation) - - -fig, ax = ppl.subplots() -ax.set_title(f"Amplitude spectrum {A1} x {f1}Hz + {A2} x {f2}Hz, {Bw}Hz smoothing") -ax.plot(freqs[:250], amplitudes[:250], label="No taper", lw=2) -ax.set_xlabel('frequency (Hz)') -ax.set_ylabel('amplitude (a.u.)') -ax.plot(freqs[:250], dpss_amplitudes[:250], label="Slepian", lw=2) +ax.plot(freqs[:400], dpss_amplitudes[:400], label=f"{Kmax} Slepians (auto)", lw=2) +Kmax = 30 +taperopt = {'Kmax' : Kmax, 'NW' : NW} +print(taperopt, N) +ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) +# average over tapers +dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) +dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel +ax.plot(freqs[:400], dpss_amplitudes[:400], label=f"{Kmax} Slepians (manual)", lw=2) + +ax.vlines([f1 - Bw/2, f1 + Bw/2], -0.1, 5.5, color='k', ls='--', lw = 2, + label="$\pm$ 7.5Hz") ax.legend() +ax.set_title(f"Amplitude spectrum {A1} x {f1}Hz with {Bw}Hz smoothing") diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index c5ae546e5..56f415286 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -198,7 +198,7 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, taper=taper, - taperopt=taperopt, + taperopt={}, polyremoval=polyremoval, timeAxis=timeAxis, foi=foi) diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py new file mode 100644 index 000000000..4a98801f6 --- /dev/null +++ b/syncopy/shared/input_validators.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# +# Validators for user submitted frontend arguments like foi, taper, etc. +# + +# Builtin/3rd party package imports +import numpy as np + +from syncopy.shared.errors import SPYValueError, SPYWarning, SPYInfo +from syncopy.shared.parsers import data_parser, scalar_parser, array_parser + + +def validate_foi(foi, foilim, samplerate): + + # Basic sanitization of frequency specifications + if foi is not None and foilim is not None: + lgl = "either `foi` or `foilim` specification" + act = "both" + raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) + + if foi is not None: + if isinstance(foi, str): + if foi == "all": + foi = None + else: + raise SPYValueError(legal="'all' or `None` or list/array", + varname="foi", actual=foi) + else: + try: + array_parser(foi, varname="foi", hasinf=False, hasnan=False, + lims=[0, samplerate/2], dims=(None,)) + except Exception as exc: + raise exc + foi = np.array(foi, dtype="float") + + if foilim is not None: + if isinstance(foilim, str): + if foilim == "all": + foilim = None + else: + raise SPYValueError(legal="'all' or `None` or `[fmin, fmax]`", + varname="foilim", actual=foilim) + else: + try: + array_parser(foilim, varname="foilim", hasinf=False, hasnan=False, + lims=[0, samplerate/2], dims=(2,)) + except Exception as exc: + raise exc + # foilim is of shape (2,) + if foilim[0] > foilim[1]: + msg = "Sorting foilim low to high.." + SPYInfo(msg) + foilim = np.sort(foilim) + + return foi, foilim + + +def validate_taper(taper, + tapsmofrq, + nTaper, + keeptapers, + foimax, + fs, + nSamples): + + ''' + General taper validation and Slepian/dpss input sanitization. + We always want to max out nTaper to achieve the desired frequency + smoothing bandwidth. For details about the Slepion settings see + + "The Effective Bandwidth of a Multitaper Spectral Estimator, + A. T. Walden, E. J. McCoy and D. B. Percival" + + ''' + + # Warn user about DPSS only settings + if taper != "dpss": + if tapsmofrq is not None: + msg = "`tapsmofrq` is only used if `taper` is `dpss`!" + SPYWarning(msg) + if nTaper is not None: + msg = "`nTaper` is only used if `taper` is `dpss`!" + SPYWarning(msg) + if keeptapers: + msg = "`keeptapers` is only used if `taper` is `dpss`!" + SPYWarning(msg) + + # empty taperopt, only Slepians have options + return {} + + # Set/get `tapsmofrq` if we're working w/Slepian tapers + elif taper == "dpss": + + # minimal smoothing bandwidth in Hz + # if sampling rate is given in Hz + minBw = 2 * fs / nSamples + + # Try to derive "sane" settings by using 3/4 octave + # smoothing of highest `foi` + # following Hill et al. "Oscillatory Synchronization in Large-Scale + # Cortical Networks Predicts Perception", Neuron, 2011 + # FIX ME: This "sane setting" seems quite excessive + + if tapsmofrq is None: + tapsmofrq = (foimax * 2**(3 / 4 / 2) - foimax * 2**(-3 / 4 / 2)) / 2 + if tapsmofrq < minBw: # *should* not happen but just in case + tapsmofrq = minBw + msg = f'Automatic setting of `tapsmofrq` to {tapsmofrq:.2f}' + SPYInfo(msg) + + # user set tapsmofrq directly + elif tapsmofrq is not None: + try: + scalar_parser(tapsmofrq, varname="tapsmofrq", lims=[0, np.inf]) + except Exception as exc: + raise exc + + if tapsmofrq < minBw: + msg = f'Setting tapsmofrq to the minimal attainable bandwidth of {minBw:.2f}Hz' + SPYInfo(msg) + tapsmofrq = minBw + + # -------------------------------------------- + # set parameters for scipy.signal.windows.dpss + NW = tapsmofrq * nSamples / (2 * fs) + # from the minBw setting NW always is at least 1 + Kmax = int(2 * NW - 1) # optimal number of tapers + # -------------------------------------------- + + # the recommended way: + # set nTaper automatically to maximize effective smoothing bandwidth + if nTaper is None: + msg = f'Using {Kmax} taper(s) for multi-tapering' + SPYInfo(msg) + taperopt = {'NW' : NW, 'Kmax' : Kmax} + return taperopt + + elif nTaper is not None: + try: + scalar_parser(nTaper, + varname="nTaper", + ntype="int_like", lims=[1, np.inf]) + except Exception as exc: + raise exc + + if nTaper != Kmax: + msg = f''' + Manually setting the number of tapers is not recommended + and may (strongly) distort the spectral estimation!\n + The optimal number of tapers is {Kmax}, you have chosen to use {nTaper}. + ''' + SPYWarning(msg) + + taperopt = {'NW' : NW, 'Kmax' : nTaper} + return taperopt + + + diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index d48906e9d..8ff8e4311 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -47,12 +47,10 @@ # ----------------------- @unwrap_io -def mtmfft_cF(trl_dat, foi=None, timeAxis=0, - keeptapers=True, nTaper=None, tapsmofrq=None, +def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, pad="nextpow2", padtype="zero", padlength=None, polyremoval=None, output_fmt="pow", - noCompute=False, chunkShape=None, - method_kwargs=None): + noCompute=False, chunkShape=None, method_kwargs=None): """ Compute (multi-)tapered Fourier transform of multi-channel time series data @@ -67,19 +65,10 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, data length and padding) are used. timeAxis : int Index of running time axis in `trl_dat` (0 or 1) - tapsmofrq : float - The amount of spectral smoothing through multi-tapering (Hz) for Slepian - tapers (`taper`="dpss"). keeptapers : bool If `True`, return spectral estimates for each taper. Otherwise power spectrum is averaged across tapers, only valid spectral estimate if `output_fmt` is `pow`. - nTaper : int - Only effective if ``taper='dpss'``. Number of orthogonal tapers to use. - tapsmofrq : float - Only effective if ``taper='dpss'``. The amount of spectral smoothing through - multi-tapering (Hz). Note that smoothing frequency specifications are one-sided, - i.e., 4 Hz smoothing means plus-minus 4 Hz, i.e., a 8 Hz smoothing box. pad : str Padding mode; one of `'absolute'`, `'relative'`, `'maxlen'`, or `'nextpow2'`. See :func:`syncopy.padding` for more information. @@ -109,7 +98,6 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, Keyword arguments passed to :func:`~syncopy.specest.mtmfft.mtmfft` controlling the spectral estimation method - Returns ------- spec : :class:`numpy.ndarray` @@ -134,13 +122,6 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, numpy.fft.rfft : NumPy's FFT implementation """ - # Slepian window parameters - if method_kwargs['taper'] == "dpss": - taperopt = {"Kmax" : nTaper, "NW" : tapsmofrq} - else: - taperopt = {} - - method_kwargs['taperopt'] = taperopt # Re-arrange array if necessary and get dimensional information if timeAxis != 0: @@ -159,6 +140,7 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, freqs = np.fft.rfftfreq(nSamples, 1 / method_kwargs["samplerate"]) _, freq_idx = best_match(freqs, foi, squash_duplicates=True) nFreq = freq_idx.size + nTaper = method_kwargs["taperopt"].get('Kmax', 1) outShape = (1, max(1, nTaper * keeptapers), nFreq, nChannels) # For initialization of computational routine, @@ -203,9 +185,10 @@ class MultiTaperFFT(ComputationalRoutine): method = "mtmfft" # 1st argument,the data, gets omitted - method_keys = list(signature(mtmfft).parameters.keys())[1:] - # here also last argument, the method_kwargs, are omitted - cF_keys = list(signature(mtmfft_cF).parameters.keys())[1:-1] + valid_kws = list(signature(mtmfft_cF).parameters.keys())[1:] + valid_kws += list(signature(mtmfft).parameters.keys())[1:] + # hardcode some parameter names which got digested from the frontend + valid_kws += ['tapsmofrq', 'nTaper'] def process_metadata(self, data, out): @@ -318,6 +301,9 @@ def mtmconvol_cF( chunkShape : None or tuple If not `None`, represents shape of output object `spec` (respecting provided values of `nTaper`, `keeptapers` etc.) + method_kwargs : dict + Keyword arguments passed to :func:`~syncopy.specest.mtmconvol.mtmconvol` + controlling the spectral estimation method Returns ------- @@ -356,14 +342,6 @@ def mtmconvol_cF( dat = padding(dat, "zero", pad="relative", padlength=None, prepadlength=padbegin, postpadlength=padend) - # Slepian window parameters - if method_kwargs['taper'] == "dpss": - taperopt = {"Kmax" : nTaper, "NW" : tapsmofrq} - else: - taperopt = {} - - method_kwargs['taperopt'] = taperopt - # Get shape of output for dry-run phase nChannels = dat.shape[1] if isinstance(toi, np.ndarray): # `toi` is an array of time-points @@ -438,6 +416,11 @@ class MultiTaperFFTConvol(ComputationalRoutine): """ computeFunction = staticmethod(mtmconvol_cF) + # 1st argument,the data, gets omitted + valid_kws = list(signature(mtmconvol_cF).parameters.keys())[1:] + valid_kws += list(signature(mtmconvol).parameters.keys())[1:] + # hardcode some parameter names which got digested from the frontend + valid_kws += ['tapsmofrq', 't_ftimwin', 'nTaper'] def process_metadata(self, data, out): @@ -605,9 +588,9 @@ class WaveletTransform(ComputationalRoutine): method = "wavelet" # 1st argument,the data, gets omitted - method_keys = list(signature(wavelet).parameters.keys())[1:] + valid_kws = list(signature(wavelet).parameters.keys())[1:] # here also last argument, the method_kwargs, are omitted - cF_keys = list(signature(wavelet_cF).parameters.keys())[1:-1] + valid_kws += list(signature(wavelet_cF).parameters.keys())[1:-1] def process_metadata(self, data, out): @@ -771,9 +754,9 @@ class SuperletTransform(ComputationalRoutine): method = "superlet" # 1st argument,the data, gets omitted - method_keys = list(signature(superlet).parameters.keys())[1:] - # here also last argument, the method_kwargs, are omitted - cF_keys = list(signature(superlet_cF).parameters.keys())[1:-1] + + valid_kws = list(signature(superlet).parameters.keys())[1:] + valid_kws += list(signature(superlet_cF).parameters.keys())[1:-1] def process_metadata(self, data, out): diff --git a/syncopy/specest/const_def.py b/syncopy/specest/const_def.py index 886f2dd22..3caf80a29 100644 --- a/syncopy/specest/const_def.py +++ b/syncopy/specest/const_def.py @@ -5,6 +5,7 @@ # Builtin/3rd party package imports import numpy as np +from scipy.signal import windows # Module-wide output specs spectralDTypes = {"pow": np.float32, @@ -20,7 +21,10 @@ availableOutputs = tuple(spectralConversions.keys()) #: available tapers of :func:`~syncopy.freqanalysis` -availableTapers = ("hann", "dpss") +all_windows = windows.__all__ +all_windows.remove("exponential") # not symmetric +all_windows.remove("hanning") # deprecated +availableTapers = all_windows #: available wavelet functions of :func:`~syncopy.freqanalysis` availableWavelets = ("Morlet", "Paul", "DOG", "Ricker", "Marr", "Mexican_hat") diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 586a58d04..38a318185 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -16,6 +16,7 @@ from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) from syncopy.shared.tools import best_match +from syncopy.shared.input_validators import validate_taper, validate_foi # method specific imports - they should go! import syncopy.specest.wavelets as spywave @@ -417,47 +418,9 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Shortcut to data sampling interval dt = 1 / data.samplerate + + foi, foilim = validate_foi(foi, foilim, data.samplerate) - # Basic sanitization of frequency specifications - if foi is not None: - if isinstance(foi, str): - if foi == "all": - foi = None - else: - raise SPYValueError(legal="'all' or `None` or list/array", - varname="foi", actual=foi) - else: - try: - array_parser(foi, varname="foi", hasinf=False, hasnan=False, - lims=[0, data.samplerate/2], dims=(None,)) - except Exception as exc: - raise exc - foi = np.array(foi, dtype="float") - if foilim is not None: - if isinstance(foilim, str): - if foilim == "all": - foilim = None - else: - raise SPYValueError(legal="'all' or `None` or `[fmin, fmax]`", - varname="foilim", actual=foilim) - else: - try: - array_parser(foilim, varname="foilim", hasinf=False, hasnan=False, - lims=[0, data.samplerate/2], dims=(2,)) - except Exception as exc: - raise exc - # foilim is of shape (2,) - if foilim[0] > foilim[1]: - msg = "Sorting foilim low to high.." - SPYInfo(msg) - foilim = np.sort(foilim) - - if foi is not None and foilim is not None: - lgl = "either `foi` or `foilim` specification" - act = "both" - raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) - - # FIXME: implement detrending # see also https://docs.obspy.org/_modules/obspy/signal/detrend.html#polynomial if polyremoval is not None: try: @@ -559,6 +522,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', scalar_parser(t_ftimwin, varname="t_ftimwin", lims=[dt, minTrialLength]) except Exception as exc: + SPYInfo("Please specify 't_ftimwin' parameter.. exiting!") raise exc # this is the effective sliding window FFT sample size @@ -591,58 +555,19 @@ def freqanalysis(data, method='mtmfft', output='fourier', lgl = "'" + "or '".join(opt + "' " for opt in availableTapers) raise SPYValueError(legal=lgl, varname="taper", actual=taper) - # Warn user about DPSS only settings - if taper != "dpss": - if tapsmofrq is not None: - msg = "`tapsmofrq` is only used if `taper` is `dpss`!" - SPYWarning(msg) - if nTaper is not None: - msg = "`nTaper` is only used if `taper` is `dpss`!" - SPYWarning(msg) - if keeptapers: - msg = "`keeptapers` is only used if `taper` is `dpss`!" - SPYWarning(msg) - - # Set/get `tapsmofrq` if we're working w/Slepian tapers - if taper == "dpss": - - # direct mtm estimate (averaging) only valid for spectral power - if not keeptapers and output != "pow": - lgl = "'pow', the only valid option for taper averaging" - raise SPYValueError(legal=lgl, varname="output", actual=output) + # direct mtm estimate (averaging) only valid for spectral power + if taper == "dpss" and not keeptapers and output != "pow": + lgl = "'pow', the only valid option for taper averaging" + raise SPYValueError(legal=lgl, varname="output", actual=output) - # Try to derive "sane" settings by using 3/4 octave - # smoothing of highest `foi` - # following Hill et al. "Oscillatory Synchronization in Large-Scale - # Cortical Networks Predicts Perception", Neuron, 2011 - if tapsmofrq is None: - foimax = foi.max() - tapsmofrq = (foimax * 2**(3/4/2) - foimax * 2**(-3/4/2)) / 2 - msg = f'Automatic setting of `tapsmofrq` to {tapsmofrq:.2f}' - SPYInfo(msg) - - else: - try: - scalar_parser(tapsmofrq, varname="tapsmofrq", lims=[1, np.inf]) - except Exception as exc: - raise exc - - # Get/compute number of tapers to use (at least 1 and max. 50) - if not nTaper: - nTaper = int(max(2, min(50, np.floor(tapsmofrq * minSampleNum * dt)))) - msg = f'Automatic setting of `nTaper` to {nTaper}' - SPYInfo(msg) - else: - try: - scalar_parser(nTaper, - varname="nTaper", - ntype="int_like", lims=[1, np.inf]) - except Exception as exc: - raise exc - - # only taper with frontend supported options is DPSS - else: - nTaper = 1 + # sanitize taper selection and retrieve dpss settings + taperopt = validate_taper(taper, + tapsmofrq, + nTaper, + keeptapers, + foimax=foi.max(), + fs=data.samplerate, + nSamples=minSampleNum) # Update `log_dct` w/method-specific options (use `lcls` to get actually # provided keyword values, not defaults set in here) @@ -660,26 +585,26 @@ def freqanalysis(data, method='mtmfft', output='fourier', # method specific parameters method_kwargs = { 'samplerate' : data.samplerate, - 'taper' : taper + 'taper' : taper, + 'taperopt' : taperopt } # Set up compute-class specestMethod = MultiTaperFFT( - samplerate=data.samplerate, foi=foi, timeAxis=timeAxis, pad=pad, padtype=padtype, padlength=padlength, keeptapers=keeptapers, - nTaper = nTaper, - tapsmofrq = tapsmofrq, polyremoval=polyremoval, output_fmt=output, method_kwargs=method_kwargs) elif method == "mtmconvol": + _check_effective_parameters(MultiTaperFFTConvol, defaults, lcls) + # Process `toi` for sliding window multi taper fft, # we have to account for three scenarios: (1) center sliding # windows on all samples in (selected) trials (2) `toi` was provided as @@ -810,7 +735,8 @@ def freqanalysis(data, method='mtmfft', output='fourier', method_kwargs = {"samplerate": data.samplerate, "nperseg": nperseg, "noverlap": noverlap, - "taper" : taper} + "taper" : taper, + "taperopt" : taperopt} # Set up compute-class specestMethod = MultiTaperFFTConvol( @@ -821,9 +747,6 @@ def freqanalysis(data, method='mtmfft', output='fourier', equidistant=equidistant, toi=toi, foi=foi, - taper=taper, - nTaper=nTaper, - tapsmofrq=tapsmofrq, timeAxis=timeAxis, keeptapers=keeptapers, polyremoval=polyremoval, @@ -1042,7 +965,7 @@ def _check_effective_parameters(CR, defaults, lcls): Result of `locals()`, all names and values of the local name space ''' # list of possible parameter names of the CR - expected = CR.method_keys + CR.cF_keys + ["parallel", "select"] + expected = CR.valid_kws + ["parallel", "select"] relevant = [name for name in defaults if name not in generalParameters] for name in relevant: if name not in expected and (lcls[name] != defaults[name]): diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index f4db5fff0..7aa5e9aab 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -262,7 +262,8 @@ def test_dpss(self): # specify tapers spec = freqanalysis(self.adata, method="mtmfft", taper="dpss", tapsmofrq=7, keeptapers=True, select=select) - assert spec.taper.size == 7 + # automatic nTaper settings, can be removed?! + # assert spec.taper.size == 7 assert spec.channel.size == len(chanList) # non-equidistant data w/multiple tapers @@ -722,7 +723,8 @@ def test_tf_toi(self): cfg.toi = "all" cfg.t_ftimwin = 0.05 tfSpec = freqanalysis(cfg, self.tfData) - assert tfSpec.taper.size > 1 + print(self.tfData) + assert tfSpec.taper.size >= 1 dt = 1/self.tfData.samplerate timeArr = np.arange(cfg.select["toilim"][0], cfg.select["toilim"][1] + dt, dt) assert np.allclose(tfSpec.time[0], timeArr) @@ -767,7 +769,7 @@ def test_tf_irregular_trials(self): artdata = generate_artificial_data(nTrials=5, nChannels=16, equidistant=True, inmemory=False) tfSpec = freqanalysis(artdata, **cfg) - assert tfSpec.taper.size > 1 + assert tfSpec.taper.size >= 1 for tk, origTime in enumerate(artdata.time): assert np.array_equal(np.unique(np.floor(origTime)), tfSpec.time[tk]) @@ -784,7 +786,7 @@ def test_tf_irregular_trials(self): artdata = generate_artificial_data(nTrials=5, nChannels=8, equidistant=False, inmemory=False) tfSpec = freqanalysis(artdata, **cfg) - assert tfSpec.taper.size > 1 + assert tfSpec.taper.size >= 1 for tk, origTime in enumerate(artdata.time): assert np.array_equal(np.unique(np.floor(origTime)), tfSpec.time[tk]) cfg.toi = "all" @@ -798,7 +800,7 @@ def test_tf_irregular_trials(self): equidistant=False, inmemory=False, dimord=AnalogData._defaultDimord[::-1]) tfSpec = freqanalysis(cfg) - assert tfSpec.taper.size > 1 + assert tfSpec.taper.size >= 1 for tk, origTime in enumerate(cfg.data.time): assert np.array_equal(np.unique(np.floor(origTime)), tfSpec.time[tk]) cfg.toi = "all" @@ -809,11 +811,11 @@ def test_tf_irregular_trials(self): # same + overlapping trials cfg.toi = 0.0 cfg.data = generate_artificial_data(nTrials=5, nChannels=4, - equidistant=False, inmemory=False, - dimord=AnalogData._defaultDimord[::-1], - overlapping=True) + equidistant=False, inmemory=False, + dimord=AnalogData._defaultDimord[::-1], + overlapping=True) tfSpec = freqanalysis(cfg) - assert tfSpec.taper.size > 1 + assert tfSpec.taper.size >= 1 for tk, origTime in enumerate(cfg.data.time): assert np.array_equal(np.unique(np.floor(origTime)), tfSpec.time[tk]) cfg.toi = "all" @@ -873,7 +875,7 @@ def test_tf_parallel(self, testcluster): inmemory=False) for chan_per_worker in enumerate([None, chanPerWrkr]): tfSpec = freqanalysis(artdata, cfg) - assert tfSpec.taper.size > 1 + assert tfSpec.taper.size >= 1 # overlapping trial spacing, throw away trials and tapers cfg.keeptapers = False From 0b267097a8058f4cdecc98ba9607b9e22e20d8d6 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 15 Nov 2021 12:06:55 +0100 Subject: [PATCH 034/109] NEW : Package wide definitions - things like `availableTapers` are shared between different submodules. To make sure these are always the same created `.shared.const_def` to contain such top-level constants and assure seamless inter-operatibilty between SyNCoPy submodules. - sub-modules still contain their own local `.const_def` for specific constants like `availableMethods` - .shared.const_def could also be named sth like `.shared.global_const`, open for discussion here --- dev_frontend.py | 4 +- syncopy/connectivity/AV_compRoutines.py | 2 +- syncopy/connectivity/ST_compRoutines.py | 4 +- syncopy/connectivity/connectivity_analysis.py | 90 ++++++------------- syncopy/connectivity/const_def.py | 31 +------ syncopy/shared/const_def.py | 29 ++++++ syncopy/shared/input_validators.py | 65 ++++++++++++-- syncopy/specest/compRoutines.py | 2 +- syncopy/specest/const_def.py | 25 +----- syncopy/specest/freqanalysis.py | 13 +-- syncopy/specest/mtmfft.py | 7 +- 11 files changed, 137 insertions(+), 135 deletions(-) create mode 100644 syncopy/shared/const_def.py diff --git a/dev_frontend.py b/dev_frontend.py index e8f206eb3..5f65de5c0 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -47,9 +47,9 @@ # foi=np.arange(502), output='pow', # polyremoval=1, -# t_ftimwin=0.5, + t_ftimwin=0.5, keeptrials=False, - taper='dpss', + taper='hann', nTaper = 11, tapsmofrq=5, # t_ftimwin=0.5, diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index d7105269e..7fa5ed3de 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -15,7 +15,7 @@ from inspect import signature # syncopy imports -from syncopy.specest.const_def import spectralDTypes, spectralConversions +from syncopy.shared.const_def import spectralDTypes, spectralConversions from syncopy.shared.errors import SPYWarning from syncopy.shared.computational_routine import ComputationalRoutine from syncopy.shared.kwarg_decorators import unwrap_io diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 56bcfb6ca..89e24acc5 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -12,7 +12,7 @@ # syncopy imports from syncopy.specest.mtmfft import mtmfft -from syncopy.specest.const_def import spectralDTypes +from syncopy.shared.const_def import spectralDTypes from syncopy.shared.errors import SPYWarning from syncopy.datatype import padding from syncopy.shared.tools import best_match @@ -26,7 +26,7 @@ def cross_spectra_cF(trl_dat, foi=None, padding_opt={}, taper="hann", - taperopt={}, + taperopt=None, polyremoval=False, timeAxis=0, norm=False, diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 56f415286..ac41126de 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -10,40 +10,37 @@ # Syncopy imports from syncopy.shared.parsers import data_parser, scalar_parser, array_parser from syncopy.shared.tools import get_defaults -from syncopy.datatype import CrossSpectralData, padding +from syncopy.datatype import CrossSpectralData from syncopy.datatype.methods.padding import _nextpow2 -from syncopy.shared.tools import best_match from syncopy.shared.errors import ( SPYValueError, SPYTypeError, SPYWarning, SPYInfo) - from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) +from syncopy.shared.tools import best_match +from syncopy.shared.input_validators import validate_taper, validate_foi +from syncopy.shared.const_def import ( + spectralConversions, + availableTapers, + generalParameters +) # Local imports from .const_def import ( - availableTapers, availableMethods, - generalParameters, - nextpow2 ) +from .ST_compRoutines import ST_CrossSpectra +from .AV_compRoutines import Normalize_CrossMeasure -# CRs still missing, CFs are already there -from .ST_compRoutines import ( - ST_CrossSpectra -) -from .AV_compRoutines import ( - Normalize_CrossMeasure -) __all__ = ["connectivityanalysis"] @unwrap_cfg @unwrap_select @detect_parallel_client -def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", +def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, nTaper=None, toi="all", out=None, @@ -126,58 +123,23 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", 'pad' : 'nextpow2' } # after padding - nSamples = nextpow2(int(lenTrials.min())) + nSamples = _nextpow2(int(lenTrials.min())) # no padding else: padding_opt = None nSamples = int(lenTrials.min()) - # --- foi sanitization --- - - if foi is not None: - if isinstance(foi, str): - if foi == "all": - foi = None - else: - raise SPYValueError(legal="'all' or `None` or list/array", - varname="foi", actual=foi) - else: - try: - array_parser(foi, varname="foi", hasinf=False, hasnan=False, - lims=[0, data.samplerate/2], dims=(None,)) - except Exception as exc: - raise exc - foi = np.array(foi, dtype="float") - - if foilim is not None: - if isinstance(foilim, str): - if foilim == "all": - foilim = None - else: - raise SPYValueError(legal="'all' or `None` or `[fmin, fmax]`", - varname="foilim", actual=foilim) - else: - try: - array_parser(foilim, varname="foilim", hasinf=False, hasnan=False, - lims=[0, data.samplerate/2], dims=(2,)) - except Exception as exc: - raise exc - # foilim is of shape (2,) - if foilim[0] > foilim[1]: - msg = "Sorting foilim low to high.." - SPYInfo(msg) - foilim = np.sort(foilim) - - if foi is not None and foilim is not None: - lgl = "either `foi` or `foilim` specification" - act = "both" - raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) + # --- Basic foi sanitization --- + + foi, foilim = validate_foi(foi, foilim, data.samplerate) # only now set foi array for foilim in 1Hz steps if foilim: foi = np.arange(foilim[0], foilim[1] + 1) - - if method == 'csd': + + # --- Settingn up specific Methods --- + + if method == 'coh': if foi is None and foilim is None: # Construct array of maximally attainable frequencies @@ -187,12 +149,14 @@ def connectivityanalysis(data, method="csd", keeptrials=False, output="abs", SPYInfo(msg) foi = freqs - # Warn user about DPSS only settings - if taper != "dpss": - if tapsmofrq is not None: - msg = "`tapsmofrq` is only used if `taper` is `dpss`!" - SPYWarning(msg) - + # sanitize taper selection and retrieve dpss settings + taperopt = validate_taper(taper, + tapsmofrq, + nTaper, + keeptapers=False, + foimax=foi.max(), + samplerate=data.samplerate, + nSamples=nSamples) # parallel computation over trials st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py index 6b9400fa4..fdab9d9bf 100644 --- a/syncopy/connectivity/const_def.py +++ b/syncopy/connectivity/const_def.py @@ -1,35 +1,8 @@ # -*- coding: utf-8 -*- # -# Constant definitions and helper functions for spectral estimations +# Constant definitions specific for connectivity # -# Builtin/3rd party package imports -import numpy as np - -# Module-wide output specs -spectralDTypes = {"pow": np.float32, - "fourier": np.complex64, - "abs": np.float32} - -#: output conversion of complex fourier coefficients -spectralConversions = {"pow": lambda x: (x * np.conj(x)).real.astype(np.float32), - "fourier": lambda x: x.astype(np.complex64), - "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} - -#: available tapers of :func:`~syncopy.connectivity_analysis` -availableTapers = ("hann", "dpss") - #: available spectral estimation methods of :func:`~syncopy.connectivity_analysis` -availableMethods = ("csd", "corr") - -#: general, method agnostic, parameters of :func:`~syncopy.connectivity_analysis` -generalParameters = ("method", "output", "keeptrials", - "foi", "foilim", "polyremoval", "out") - +availableMethods = ("coh", "corr") -# auxiliary functions -def nextpow2(number): - n = 1 - while n < number: - n *= 2 - return n diff --git a/syncopy/shared/const_def.py b/syncopy/shared/const_def.py new file mode 100644 index 000000000..2033be140 --- /dev/null +++ b/syncopy/shared/const_def.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# Constant definitions used throughout SyNCoPy +# + +# Builtin/3rd party package imports +import numpy as np +from scipy.signal import windows + +# Module-wide output specs +spectralDTypes = {"pow": np.float32, + "fourier": np.complex64, + "abs": np.float32} + +#: output conversion of complex fourier coefficients +spectralConversions = {"pow": lambda x: (x * np.conj(x)).real.astype(np.float32), + "fourier": lambda x: x.astype(np.complex64), + "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} + + +#: available tapers of :func:`~syncopy.freqanalysis` +all_windows = windows.__all__ +all_windows.remove("exponential") # not symmetric +all_windows.remove("hanning") # deprecated +availableTapers = all_windows + +#: general, method agnostic, parameters of :func:`~syncopy.freqanalysis` +generalParameters = ("method", "output", "keeptrials", + "foi", "foilim", "polyremoval", "out") diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index 4a98801f6..e29e1cf9b 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # # Validators for user submitted frontend arguments like foi, taper, etc. +# Input args are the parameters to check for validity + auxiliary parameters +# needed for the checks. # # Builtin/3rd party package imports @@ -11,8 +13,32 @@ def validate_foi(foi, foilim, samplerate): + + ''' + + Parameters to check + ------------------- + foi : 'all' or array like or None + frequencies of interest + foilim : 2-element sequence or None + foi limits + + Auxiliary arguments + ------------------- + samplerate : float + the samplerate in Hz + + Returns + ------- + foi, foilim : tuple + + Notes + ----- + Setting both `foi` and `foilim` to `None` is valid, the + subsequent analysis methods should all have a default way to + select a standard set of frequencies (e.g. np.fft.fftfreq). + ''' - # Basic sanitization of frequency specifications if foi is not None and foilim is not None: lgl = "either `foi` or `foilim` specification" act = "both" @@ -60,18 +86,43 @@ def validate_taper(taper, nTaper, keeptapers, foimax, - fs, + samplerate, nSamples): ''' General taper validation and Slepian/dpss input sanitization. - We always want to max out nTaper to achieve the desired frequency + The default is to max out `nTaper` to achieve the desired frequency smoothing bandwidth. For details about the Slepion settings see "The Effective Bandwidth of a Multitaper Spectral Estimator, A. T. Walden, E. J. McCoy and D. B. Percival" - ''' + Parameters to check + ------------------- + taper : str + Windowing function, one of :data:`~syncopy.shared.const_def.availableTapers` + tapsmofrq : float or None + Taper smoothing bandwidth for `taper='dpss'` + nTaper : int_like or None + Number of tapers to user for multi-tapering (not recommended) + + Auxiliary arguments + ------------------- + keeptapers : bool + foimax : float + Maximum frequency for the analysis + samplerate : float + the samplerate in Hz + nSamples : int + Number of samples + + Returns + ------- + taperopt : dict + For multi-tapering (`taper='dpss'`) contains the + parameters `NW` and `Kmax` for `scipy.signal.windows.dpss`. + For all other tapers this is an empty dictionary. + ''' # Warn user about DPSS only settings if taper != "dpss": @@ -93,13 +144,13 @@ def validate_taper(taper, # minimal smoothing bandwidth in Hz # if sampling rate is given in Hz - minBw = 2 * fs / nSamples + minBw = 2 * samplerate / nSamples # Try to derive "sane" settings by using 3/4 octave # smoothing of highest `foi` # following Hill et al. "Oscillatory Synchronization in Large-Scale # Cortical Networks Predicts Perception", Neuron, 2011 - # FIX ME: This "sane setting" seems quite excessive + # FIX ME: This "sane setting" seems quite excessive (huuuge bwidths) if tapsmofrq is None: tapsmofrq = (foimax * 2**(3 / 4 / 2) - foimax * 2**(-3 / 4 / 2)) / 2 @@ -122,7 +173,7 @@ def validate_taper(taper, # -------------------------------------------- # set parameters for scipy.signal.windows.dpss - NW = tapsmofrq * nSamples / (2 * fs) + NW = tapsmofrq * nSamples / (2 * samplerate) # from the minBw setting NW always is at least 1 Kmax = int(2 * NW - 1) # optimal number of tapers # -------------------------------------------- diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index 8ff8e4311..969a58cf8 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -36,7 +36,7 @@ from syncopy.shared.tools import best_match from syncopy.shared.computational_routine import ComputationalRoutine from syncopy.shared.kwarg_decorators import unwrap_io -from syncopy.specest.const_def import ( +from syncopy.shared.const_def import ( spectralConversions, spectralDTypes, ) diff --git a/syncopy/specest/const_def.py b/syncopy/specest/const_def.py index 3caf80a29..a15cf744f 100644 --- a/syncopy/specest/const_def.py +++ b/syncopy/specest/const_def.py @@ -1,37 +1,16 @@ # -*- coding: utf-8 -*- # -# Constant definitions and helper functions for spectral estimations +# Constant definitions specific for spectral estimations # -# Builtin/3rd party package imports -import numpy as np -from scipy.signal import windows - -# Module-wide output specs -spectralDTypes = {"pow": np.float32, - "fourier": np.complex64, - "abs": np.float32} - -#: output conversion of complex fourier coefficients -spectralConversions = {"pow": lambda x: (x * np.conj(x)).real.astype(np.float32), - "fourier": lambda x: x.astype(np.complex64), - "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} +from syncopy.shared.const_def import spectralConversions #: available outputs of :func:`~syncopy.freqanalysis` availableOutputs = tuple(spectralConversions.keys()) -#: available tapers of :func:`~syncopy.freqanalysis` -all_windows = windows.__all__ -all_windows.remove("exponential") # not symmetric -all_windows.remove("hanning") # deprecated -availableTapers = all_windows - #: available wavelet functions of :func:`~syncopy.freqanalysis` availableWavelets = ("Morlet", "Paul", "DOG", "Ricker", "Marr", "Mexican_hat") #: available spectral estimation methods of :func:`~syncopy.freqanalysis` availableMethods = ("mtmfft", "mtmconvol", "wavelet", "superlet") -#: general, method agnostic, parameters of :func:`~syncopy.freqanalysis` -generalParameters = ("method", "output", "keeptrials", - "foi", "foilim", "polyremoval", "out") diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 38a318185..3a1420de3 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -16,6 +16,12 @@ from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) from syncopy.shared.tools import best_match +from syncopy.shared.const_def import ( + spectralConversions, + availableTapers, + generalParameters +) + from syncopy.shared.input_validators import validate_taper, validate_foi # method specific imports - they should go! @@ -25,11 +31,8 @@ # Local imports from .const_def import ( - spectralConversions, - availableTapers, availableWavelets, - availableMethods, - generalParameters + availableMethods, ) from .compRoutines import ( @@ -566,7 +569,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', nTaper, keeptapers, foimax=foi.max(), - fs=data.samplerate, + samplerate=data.samplerate, nSamples=minSampleNum) # Update `log_dct` w/method-specific options (use `lcls` to get actually diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 94f4f2a67..622a3435d 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -8,7 +8,7 @@ from scipy import signal -def mtmfft(data_arr, samplerate, taper="hann", taperopt={}): +def mtmfft(data_arr, samplerate, taper="hann", taperopt=None): ''' (Multi-)tapered fast Fourier transform. Returns @@ -25,7 +25,7 @@ def mtmfft(data_arr, samplerate, taper="hann", taperopt={}): taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. - taperopt : dict + taperopt : dict or None Additional keyword arguments passed to the `taper` function. For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. @@ -66,6 +66,9 @@ def mtmfft(data_arr, samplerate, taper="hann", taperopt={}): if taper is None: taper = 'boxcar' + if taperopt is None: + taperopt = {} + taper_func = getattr(signal.windows, taper) # only really 2d if taper='dpss' with Kmax > 1 windows = np.atleast_2d(taper_func(nSamples, **taperopt)) From 4b582cb7f0db82efc77d18a63125aff5c69e406c Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 15 Nov 2021 12:33:23 +0100 Subject: [PATCH 035/109] NEW: Completed taper validation refactoring - all checks regarding tapers have been moved to `.shared.input_validators.validate_taper` Changes to be committed: modified: dev_frontend.py modified: syncopy/connectivity/ST_compRoutines.py modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/shared/input_validators.py modified: syncopy/specest/compRoutines.py modified: syncopy/specest/freqanalysis.py modified: syncopy/specest/mtmconvol.py modified: syncopy/specest/mtmfft.py --- dev_frontend.py | 4 +-- syncopy/connectivity/ST_compRoutines.py | 6 ++-- syncopy/connectivity/connectivity_analysis.py | 19 ++++++----- syncopy/shared/input_validators.py | 33 ++++++++++++------- syncopy/specest/compRoutines.py | 10 +++--- syncopy/specest/freqanalysis.py | 19 +++-------- syncopy/specest/mtmconvol.py | 10 +++--- syncopy/specest/mtmfft.py | 12 +++---- 8 files changed, 58 insertions(+), 55 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 5f65de5c0..9e7ccff41 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -49,9 +49,9 @@ # polyremoval=1, t_ftimwin=0.5, keeptrials=False, - taper='hann', + taper='dpss', nTaper = 11, tapsmofrq=5, -# t_ftimwin=0.5, + keeptapers=True, parallel=False, # try this!!!!!! select={"trials" : [0,1]}) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 89e24acc5..67cc45968 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -26,7 +26,7 @@ def cross_spectra_cF(trl_dat, foi=None, padding_opt={}, taper="hann", - taperopt=None, + taper_opt=None, polyremoval=False, timeAxis=0, norm=False, @@ -73,7 +73,7 @@ def cross_spectra_cF(trl_dat, taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. - taperopt : dict, optional + taper_opt : dict, optional Additional keyword arguments passed to the `taper` function. For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. @@ -163,7 +163,7 @@ def cross_spectra_cF(trl_dat, # compute the individual spectra # specs have shape (nTapers x nFreq x nChannels) - specs, freqs = mtmfft(dat, samplerate, taper, taperopt) + specs, freqs = mtmfft(dat, samplerate, taper, taper_opt) # outer product along channel axes # has shape (nTapers x nFreq x nChannels x nChannels) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index ac41126de..b447ad6f3 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -150,19 +150,20 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", foi = freqs # sanitize taper selection and retrieve dpss settings - taperopt = validate_taper(taper, - tapsmofrq, - nTaper, - keeptapers=False, - foimax=foi.max(), - samplerate=data.samplerate, - nSamples=nSamples) - + taper_opt = validate_taper(taper, + tapsmofrq, + nTaper, + keeptapers=False, # ST_CSD's always average tapers + foimax=foi.max(), + samplerate=data.samplerate, + nSamples=nSamples, + output="pow") # ST_CSD's always have this unit/norm + # parallel computation over trials st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, taper=taper, - taperopt={}, + taper_opt=taper_opt, polyremoval=polyremoval, timeAxis=timeAxis, foi=foi) diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index e29e1cf9b..c96b01538 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -10,6 +10,7 @@ from syncopy.shared.errors import SPYValueError, SPYWarning, SPYInfo from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.const_def import availableTapers def validate_foi(foi, foilim, samplerate): @@ -87,7 +88,8 @@ def validate_taper(taper, keeptapers, foimax, samplerate, - nSamples): + nSamples, + output): ''' General taper validation and Slepian/dpss input sanitization. @@ -115,14 +117,21 @@ def validate_taper(taper, the samplerate in Hz nSamples : int Number of samples + output : str, one of {'abs', 'pow', 'fourier'} + Fourier transformation output type Returns ------- - taperopt : dict + dpss_opt : dict For multi-tapering (`taper='dpss'`) contains the parameters `NW` and `Kmax` for `scipy.signal.windows.dpss`. For all other tapers this is an empty dictionary. ''' + + # See if taper choice is supported + if taper not in availableTapers: + lgl = "'" + "or '".join(opt + "' " for opt in availableTapers) + raise SPYValueError(legal=lgl, varname="taper", actual=taper) # Warn user about DPSS only settings if taper != "dpss": @@ -136,9 +145,14 @@ def validate_taper(taper, msg = "`keeptapers` is only used if `taper` is `dpss`!" SPYWarning(msg) - # empty taperopt, only Slepians have options + # empty dpss_opt, only Slepians have options return {} - + + # direct mtm estimate (averaging) only valid for spectral power + if taper == "dpss" and not keeptapers and output != "pow": + lgl = "'pow', the only valid option for taper averaging" + raise SPYValueError(legal=lgl, varname="output", actual=output) + # Set/get `tapsmofrq` if we're working w/Slepian tapers elif taper == "dpss": @@ -183,8 +197,8 @@ def validate_taper(taper, if nTaper is None: msg = f'Using {Kmax} taper(s) for multi-tapering' SPYInfo(msg) - taperopt = {'NW' : NW, 'Kmax' : Kmax} - return taperopt + dpss_opt = {'NW' : NW, 'Kmax' : Kmax} + return dpss_opt elif nTaper is not None: try: @@ -202,8 +216,5 @@ def validate_taper(taper, ''' SPYWarning(msg) - taperopt = {'NW' : NW, 'Kmax' : nTaper} - return taperopt - - - + dpss_opt = {'NW' : NW, 'Kmax' : nTaper} + return dpss_opt diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index 969a58cf8..ab7a4d3e3 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -140,7 +140,7 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, freqs = np.fft.rfftfreq(nSamples, 1 / method_kwargs["samplerate"]) _, freq_idx = best_match(freqs, foi, squash_duplicates=True) nFreq = freq_idx.size - nTaper = method_kwargs["taperopt"].get('Kmax', 1) + nTaper = method_kwargs["taper_opt"].get('Kmax', 1) outShape = (1, max(1, nTaper * keeptapers), nFreq, nChannels) # For initialization of computational routine, @@ -279,7 +279,7 @@ def mtmconvol_cF( Index of running time axis in `trl_dat` (0 or 1) taper : callable Taper function to use, one of :data:`~syncopy.specest.const_def.availableTapers` - taperopt : dict + taper_opt : dict Additional keyword arguments passed to `taper` (see above). For further details, please refer to the `SciPy docs `_ @@ -381,18 +381,18 @@ def mtmconvol_cF( # every individual soi, so we can use mtmfft! samplerate = method_kwargs['samplerate'] taper = method_kwargs['taper'] - taperopt = method_kwargs['taperopt'] + taper_opt = method_kwargs['taper_opt'] # In case tapers aren't preserved allocate `spec` "too big" # and average afterwards spec = np.full((nTime, nTaper, nFreq, nChannels), np.nan, dtype=spectralDTypes[output_fmt]) - ftr, freqs = mtmfft(dat[soi[0], :], samplerate, taper, taperopt) + ftr, freqs = mtmfft(dat[soi[0], :], samplerate, taper, taper_opt) _, fIdx = best_match(freqs, foi, squash_duplicates=True) spec[0, ...] = spectralConversions[output_fmt](ftr[:, fIdx, :]) # loop over remaining soi to center windows on for tk in range(1, len(soi)): - ftr, freqs = mtmfft(dat[soi[tk], :], samplerate, taper, taperopt) + ftr, freqs = mtmfft(dat[soi[tk], :], samplerate, taper, taper_opt) spec[tk, ...] = spectralConversions[output_fmt](ftr[:, fIdx, :]) # Average across tapers if wanted diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 3a1420de3..25d9af937 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -553,24 +553,15 @@ def freqanalysis(data, method='mtmfft', output='fourier', act = "empty frequency selection" raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) - # See if taper choice is supported - if taper not in availableTapers: - lgl = "'" + "or '".join(opt + "' " for opt in availableTapers) - raise SPYValueError(legal=lgl, varname="taper", actual=taper) - - # direct mtm estimate (averaging) only valid for spectral power - if taper == "dpss" and not keeptapers and output != "pow": - lgl = "'pow', the only valid option for taper averaging" - raise SPYValueError(legal=lgl, varname="output", actual=output) - # sanitize taper selection and retrieve dpss settings - taperopt = validate_taper(taper, + taper_opt = validate_taper(taper, tapsmofrq, nTaper, keeptapers, foimax=foi.max(), samplerate=data.samplerate, - nSamples=minSampleNum) + nSamples=minSampleNum, + output=output) # Update `log_dct` w/method-specific options (use `lcls` to get actually # provided keyword values, not defaults set in here) @@ -589,7 +580,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', method_kwargs = { 'samplerate' : data.samplerate, 'taper' : taper, - 'taperopt' : taperopt + 'taper_opt' : taper_opt } # Set up compute-class @@ -739,7 +730,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', "nperseg": nperseg, "noverlap": noverlap, "taper" : taper, - "taperopt" : taperopt} + "taper_opt" : taper_opt} # Set up compute-class specestMethod = MultiTaperFFTConvol( diff --git a/syncopy/specest/mtmconvol.py b/syncopy/specest/mtmconvol.py index 951ece961..42b844cfd 100644 --- a/syncopy/specest/mtmconvol.py +++ b/syncopy/specest/mtmconvol.py @@ -9,7 +9,7 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", - taperopt={}, boundary='zeros', padded=True, detrend=False): + taper_opt={}, boundary='zeros', padded=True, detrend=False): ''' (Multi-)tapered short time fast Fourier transform. Returns @@ -31,7 +31,7 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. - taperopt : dict + taper_opt : dict Additional keyword arguments passed to the `taper` function. For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. @@ -84,14 +84,14 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", # -> normalizes with win.sum() :/ # see also https://github.com/scipy/scipy/issues/14740 if taper == 'dpss': - taperopt['sym'] = False + taper_opt['sym'] = False # only truly 2d for multi-taper "dpss" - windows = np.atleast_2d(taper_func(nperseg, **taperopt)) + windows = np.atleast_2d(taper_func(nperseg, **taper_opt)) # Slepian normalization if taper == 'dpss': - windows = windows * np.sqrt(taperopt.get('Kmax', 1)) / np.sqrt(nperseg) + windows = windows * np.sqrt(taper_opt.get('Kmax', 1)) / np.sqrt(nperseg) # number of time points in the output if boundary is None: diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 622a3435d..778c19b15 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -8,7 +8,7 @@ from scipy import signal -def mtmfft(data_arr, samplerate, taper="hann", taperopt=None): +def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): ''' (Multi-)tapered fast Fourier transform. Returns @@ -25,7 +25,7 @@ def mtmfft(data_arr, samplerate, taper="hann", taperopt=None): taper : str or None Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. - taperopt : dict or None + taper_opt : dict or None Additional keyword arguments passed to the `taper` function. For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. @@ -66,18 +66,18 @@ def mtmfft(data_arr, samplerate, taper="hann", taperopt=None): if taper is None: taper = 'boxcar' - if taperopt is None: - taperopt = {} + if taper_opt is None: + taper_opt = {} taper_func = getattr(signal.windows, taper) # only really 2d if taper='dpss' with Kmax > 1 - windows = np.atleast_2d(taper_func(nSamples, **taperopt)) + windows = np.atleast_2d(taper_func(nSamples, **taper_opt)) # only(!!) slepian windows are already normalized # still have to normalize by number of tapers # such that taper-averaging yields correct amplitudes if taper == 'dpss': - windows = windows * np.sqrt(taperopt.get('Kmax', 1)) + windows = windows * np.sqrt(taper_opt.get('Kmax', 1)) # per pedes L2 normalisation for all other tapers else: windows = windows * np.sqrt(nSamples) / np.sum(windows) From ba71a9f4ba1c4a687fe42446d74ee042ce5a6102 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 15 Nov 2021 15:48:52 +0100 Subject: [PATCH 036/109] FIX: Mask channel property of CrossSpectralData - basically only to have sth meaningful for __str__() we return an info string pointing to `channel_i` and `channel_j` when `channel` is queried Changes to be committed: modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/continuous_data.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index bdee99fcc..05bb83f10 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -708,15 +708,11 @@ class CrossSpectralData(ContinuousData): # override channel property to avoid accidental access @property def channel(self): - pass - # msg = f"CrossSpectralData has no 'channel' but dimord: {self._dimord}" - # SPYWarning(msg) - # raise NotImplementedError(msg) + return "see channel_i and channel_j" @channel.setter def channel(self, channel): if channel is None: - # print('channel None setter called') pass else: msg = f"CrossSpectralData has no 'channel' to set but dimord: {self._dimord}" From bfc5fa2446d05dbc2926be6bb3ec94a913c2676e Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 15 Nov 2021 17:16:30 +0100 Subject: [PATCH 037/109] WIP: Wilson's algorithm for Granger-Geweke causality Changes to be committed: modified: dev_csd.py deleted: dev_tapsmofrq.py new file: syncopy/connectivity/wilson_sf.py --- dev_csd.py | 63 +++++++------------- dev_tapsmofrq.py | 55 ------------------ syncopy/connectivity/wilson_sf.py | 95 +++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 99 deletions(-) delete mode 100644 dev_tapsmofrq.py create mode 100644 syncopy/connectivity/wilson_sf.py diff --git a/dev_csd.py b/dev_csd.py index 9be3130a4..3fbd31fe2 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -1,8 +1,9 @@ import numpy as np from scipy.signal import csd as sci_csd import scipy.signal as sci -from syncopy.connectivity.single_trial_compRoutines import cross_spectra_cF -from syncopy.connectivity.single_trial_compRoutines import cross_covariance_cF +from syncopy.connectivity.ST_compRoutines import cross_spectra_cF +from syncopy.connectivity import wilson_sf +from syncopy.connectivity.ST_compRoutines import cross_covariance_cF import matplotlib.pyplot as ppl @@ -83,45 +84,12 @@ def sci_est(x, y, nper, norm=False): return (freqs1, np.abs(csd1)), (freqs2, np.abs(csd2)) -def mtm_csd_harmonics(): - - omegas = np.array([30, 80]) * 2 * np.pi - phase_shifts = np.array([0, np.pi / 2, np.pi]) - - NN = 50 - res = np.zeros((nSamples // 2 + 1, NN)) - eps = 1 - Kmax = 15 - data = [np.sum([np.cos(om * tvec + ps) for om in omegas], axis=0) for ps in phase_shifts] - data = np.array(data).T - - for i in range(NN): - dataR = 5 * (data + np.random.randn(nSamples, 3) * eps) - - CS, freqs = cross_spectra_cF(data, fs, taper='bartlett') - CS2, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : Kmax, 'NW' : 6}, norm=True) - - res[:, i] = np.abs(CS2[:, 0, 1]) - - q1 = np.percentile(res, 25, axis=1) - q3 = np.percentile(res, 75, axis=1) - med = np.percentile(res, 50, axis=1) - - fig, ax = ppl.subplots(figsize=(6,4), num=None) - ax.set_xlabel('frequency (Hz)') - ax.set_ylabel('coherence') - ax.set_ylim((-.02,1.05)) - ax.set_title(f'MTM coherence, {Kmax} tapers, SNR={1/eps**2}') - - c = 'cornflowerblue' - ax.plot(freqs, med, lw=2, alpha=0.8, c=c) - ax.fill_between(freqs, q1, q3, color=c, alpha=0.3) - - # omegas = np.arange(30, 50, step=1) * 2 * np.pi # data = np.array([np.cos(om * tvec) for om in omegas]).T # dataR = 1 * (data + np.random.randn(*data.shape) * 1) -# CS, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=True) +# CS, freqs = cross_spectra_cF(dataR, fs, taper='dpss', +# taper_opt={'Kmax' : 15, 'NW' : 6}, +# norm=True, fullOutput=True) # CS2, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=False) @@ -141,16 +109,23 @@ def phase_evo(omega0, eps, fs=1000, N=1000): s2 = np.cos(p2) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) data = np.c_[s1, s2] -CS, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=True) -CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=False) +bw = 10 +NW = bw * nSamples / (2 * fs) +Kmax = int(2 * NW - 1) # optimal number of tapers + +CS, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, + norm=True, fullOutput=True) +CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, + norm=False, fullOutput=True) + -fig, ax = ppl.subplots(figsize=(6,4), num=None) +fig, ax = ppl.subplots(figsize=(6,4), num=1) ax.set_xlabel('frequency (Hz)') ax.set_ylabel('$|CSD(f)|$') ax.set_ylim((-.02,1.25)) -ax.plot(freqs, np.abs(CS2[:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) -ax.plot(freqs, np.abs(CS2[:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) -ax.plot(freqs, np.abs(CS2[:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) +ax.plot(freqs, np.abs(CS2[0,:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) +ax.plot(freqs, np.abs(CS2[0,:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) +ax.plot(freqs, np.abs(CS2[0,:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) ax.legend() diff --git a/dev_tapsmofrq.py b/dev_tapsmofrq.py deleted file mode 100644 index c3397daa4..000000000 --- a/dev_tapsmofrq.py +++ /dev/null @@ -1,55 +0,0 @@ -from syncopy.specest import mtmfft -import numpy as np -import matplotlib.pyplot as ppl -from scipy.signal import windows - -fs = 1000 -# superposition 40Hz and 100Hz oscillations A1:A2 for 1s -f1, f2 = 40, 100 -A1, A2 = 5, 3 -tvec = np.arange(0, 4.096, 1 / 1000) - -signal = A1 * np.cos(2 * np.pi * f1 * tvec) -#signal += A2 * np.cos(2 * np.pi * f2 * tvec) - -# the transforms have shape (nTaper, nFreq, nChannel) -ftr, freqs = mtmfft.mtmfft(signal, fs, taper=None) - -# average over potential tapers (only 1 here) -spec = np.real(ftr * ftr.conj()).mean(axis=0) -amplitudes = np.sqrt(spec)[:, 0] # only 1 channel - -fig, ax = ppl.subplots() -ax.set_xlabel('frequency (Hz)') -ax.set_ylabel('amplitude (a.u.)') - -ax.plot(freqs[:400], amplitudes[:400], label="No taper", lw=2) - -N = len(tvec) -minBw = 2 * fs / N -Bw = 15 -W = Bw / 2 # Hz -NW = W * N / fs -Kmax = int(2 * NW - 1) -taperopt = {'Kmax' : Kmax, 'NW' : NW} -print(taperopt, N) -ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) -# average over tapers -dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) -dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel -# check for amplitudes (and taper normalisation) - -ax.plot(freqs[:400], dpss_amplitudes[:400], label=f"{Kmax} Slepians (auto)", lw=2) -Kmax = 30 -taperopt = {'Kmax' : Kmax, 'NW' : NW} -print(taperopt, N) -ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) -# average over tapers -dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) -dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel -ax.plot(freqs[:400], dpss_amplitudes[:400], label=f"{Kmax} Slepians (manual)", lw=2) - -ax.vlines([f1 - Bw/2, f1 + Bw/2], -0.1, 5.5, color='k', ls='--', lw = 2, - label="$\pm$ 7.5Hz") -ax.legend() -ax.set_title(f"Amplitude spectrum {A1} x {f1}Hz with {Bw}Hz smoothing") diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py new file mode 100644 index 000000000..6d180bf4c --- /dev/null +++ b/syncopy/connectivity/wilson_sf.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# +# Performs the numerical inner-outer factorization of a spectral matrix, using +# Wilsons method. This implementation here is a Python version of the original +# Matlab implementation by M. Dhamala (mdhamala@bme.ufl.edu) & G. Rangarajan +# (rangaraj@math.iisc.ernet.in), UF, Aug 3-4, 2006. +# +# The algorithm itself was first presented in: +# The Factorization of Matricial Spectral Densities, SIAM J. Appl. Math, +# Vol. 23, No. 4, pgs 420-426 December 1972 by G T Wilson). + +# Builtin/3rd party package imports +import numpy as np + + +def wilson_sf(CSD, samplerate, nIter=500, tol=1e-9): + + ''' + Wilsons spectral matrix factorization ("analytic method") + + This is a pure backend function and hence no input argument + checking is performed. + + Parameters + ---------- + CSD : (nFreq, N, N) :class:`numpy.ndarray` + Complex cross spectra for all channel combinations i,j. + `N` corresponds to number of input channels. + + Returns + ------- + + ''' + + psi0 = _psi0_initial(CSD) + + g = np.zeros(CSD.shape) + + g = 0 # :D + +def _psi0_initial(CSD): + + ''' + Initialize Wilson's algorithm with the Cholesky + decomposition of the 1st Fourier series component + of the cross spectral density matrix (CSD). This is + explicitly proposed in section 4. of the original paper. + ''' + + nSamples = CSD.shape[1] + + # perform ifft to obtain gammas. + gamma = np.fft.ifft(CSD, axis=0) + gamma0 = gamma[0, ...] + + # Remove any assymetry due to rounding error. + # This also will zero out any imaginary values + # on the diagonal - real diagonals are required for cholesky. + gamma0 = np.real((gamma0 + gamma0.conj()) / 2) + + # check for positive definiteness + eivals = np.linalg.eigvals(gamma0) + if np.all(np.imag(eivals) == 0): + psi0 = np.linalg.cholesky(gamma0) + # otherwise initialize with 1's as a fallback + else: + psi0 = np.ones((nSamples, nSamples)) + + return psi0 + + +def _plusOperator(g): + + ''' + The []+ operator from definition 1.2, + given by explicit Fourier transformations + + The time x nChannel x nChannel matrix `g` is given + in the frequency domain. + ''' + + # 'negative lags' from the ifft + nLag = g.shape[0] // 2 + # the series expansion in beta_k + beta = np.fft.ifft(g, axis=0) + + # take half of the zero lag + beta[0, ...] = 0.5 * beta[0, ...] + g0 = beta[0, ...] + + # Zero out negative powers. + beta[:nLag + 1, ..., ...] = 0 + + gp = np.fft.fft(beta, axis=0) + return gp, g0 From bedde4304e0c1d896691f75fe5e467f97a19a483 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 16 Nov 2021 14:59:34 +0100 Subject: [PATCH 038/109] CHG: Updated __str__ method of ContinuousData - included a simple if-clause to not show `channel` property when printing `CrossSpectralData` objects; trying to do this solely via `_infoFileProperties` etc. (as suggested in #150) is tedious and error prone (not all print-worthy attributes are saved as is - e.g., `sampleinfo` or `time`). Closes #150 On branch coherence Changes to be committed: modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/continuous_data.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 05bb83f10..3eead0cfc 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -74,7 +74,8 @@ def __str__(self): ppattrs = [attr for attr in ppattrs if not (inspect.ismethod(getattr(self, attr)) or isinstance(getattr(self, attr), Iterator))] - + if self.__class__.__name__ == "CrossSpectralData": + ppattrs.remove("channel") ppattrs.sort() # Construct string for pretty-printing class attributes @@ -717,7 +718,7 @@ def channel(self, channel): else: msg = f"CrossSpectralData has no 'channel' to set but dimord: {self._dimord}" raise NotImplementedError(msg) - + @property def channel_i(self): """ :class:`numpy.ndarray` : list of recording channel names """ @@ -726,7 +727,7 @@ def channel_i(self): nChannel = self.data.shape[self.dimord.index("channel_i")] return np.array(["channel_i-" + str(i + 1).zfill(len(str(nChannel))) for i in range(nChannel)]) - + return self._channel_i @channel_i.setter @@ -756,7 +757,7 @@ def channel_j(self): nChannel = self.data.shape[self.dimord.index("channel_j")] return np.array(["channel_j-" + str(i + 1).zfill(len(str(nChannel))) for i in range(nChannel)]) - + return self._channel_j @channel_j.setter @@ -777,13 +778,13 @@ def channel_j(self, channel_j): raise exc self._channel_j = np.array(channel_j) - + @property def freq(self): """:class:`numpy.ndarray`: frequency axis in Hz """ # if data exists but no user-defined frequency axis, # create a dummy one on the fly - + if self._freq is None and self._data is not None: return np.arange(self.data.shape[self.dimord.index("freq")]) return self._freq @@ -887,4 +888,4 @@ def __init__(self, samplerate=samplerate, freq=freq, dimord=dimord) - + From 6a9a4adb8312fae4d6dbdafa66cf81753304e910 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 16 Nov 2021 16:03:09 +0100 Subject: [PATCH 039/109] NEW: Incorporated in-place selection support - a new keyword (`inplace`) has been added to `selectdata` to permit the storage of user-provided selections in an object's `_selection` attribute. This allows to prepare an object for in-place selections which can then be used by arithmetic operators (closes #142) On branch coherence Changes to be committed: modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/methods/selectdata.py --- syncopy/datatype/continuous_data.py | 6 ++-- syncopy/datatype/methods/selectdata.py | 42 +++++++++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 3eead0cfc..1e5bbf760 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -15,15 +15,13 @@ import numpy as np from abc import ABC from collections.abc import Iterator -from numpy.lib.arraysetops import isin -from numpy.lib.format import open_memmap # Local imports -from .base_data import BaseData, FauxTrial, Selector +from .base_data import BaseData, FauxTrial from .methods.definetrial import definetrial from .methods.selectdata import selectdata from syncopy.shared.parsers import scalar_parser, array_parser -from syncopy.shared.errors import SPYValueError, SPYWarning +from syncopy.shared.errors import SPYValueError from syncopy.shared.tools import best_match from syncopy.plotting import _plot_analog from syncopy.plotting import _plot_spectral diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index bcc78da54..29e40ca13 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -6,6 +6,7 @@ # Local imports from syncopy.shared.parsers import data_parser from syncopy.shared.tools import get_defaults +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYInfo from syncopy.shared.kwarg_decorators import unwrap_cfg, unwrap_io, detect_parallel_client from syncopy.shared.computational_routine import ComputationalRoutine @@ -16,7 +17,7 @@ @detect_parallel_client def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None, foilim=None, tapers=None, units=None, eventids=None, - out=None, **kwargs): + out=None, inplace=False, **kwargs): """ Create a new Syncopy object from a selection @@ -168,6 +169,11 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None can be unsorted and may include repetitions but must match exactly, be finite and not NaN. If `eventids` is `None` or ``eventids = "all"``, all events are selected. + inplace : bool + If `inplace` is `True` **no** new object is created. Instead the provided + selection is stored in the input object's `_selection` attribute for later + use. By default `inplace` is `False` and all calls to `selectdata` create + a new Syncopy data object. Returns ------- @@ -242,18 +248,27 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None except Exception as exc: raise exc + # Vet the only input not checked by `Selector` + if not isinstance(inplace, bool): + raise SPYTypeError(inplace, varname="inplace", expected="Boolean") + # If provided, make sure output object is appropriate - if out is not None: - try: - data_parser(out, varname="out", writable=True, empty=True, - dataclass=data.__class__.__name__, - dimord=data.dimord) - except Exception as exc: - raise exc - new_out = False + if not inplace: + if out is not None: + try: + data_parser(out, varname="out", writable=True, empty=True, + dataclass=data.__class__.__name__, + dimord=data.dimord) + except Exception as exc: + raise exc + new_out = False + else: + out = data.__class__(dimord=data.dimord) + new_out = True else: - out = data.__class__(dimord=data.dimord) - new_out = True + if out is not None: + lgl = "no output object for in-place selection" + raise SPYValueError(lgl, varname="out", actual=out.__class__.__name__) # Pass provided selections on to `Selector` class which performs error checking data._selection = {"trials": trials, @@ -266,6 +281,11 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None "units": units, "eventids": eventids} + # If an in-place selection was requested we're done + if inplace: + SPYInfo("In-place selection attached to data object: {}".format(data._selection)) + return + # Create inventory of all available selectors and actually provided values # to create a bookkeeping dict for logging provided = locals() From e93b65b7df3ceae26864f4e0b1e2a80c796d98f4 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 16 Nov 2021 16:24:02 +0100 Subject: [PATCH 040/109] WIP: Cross-Correlation CRs - basic ST_CR and AV_CR are written, yet the sample/trialdefinitions are off. Probably as the time axis here actually is the lag axis.. --- dev_csd.py | 26 ++-- dev_frontend.py | 58 +++---- syncopy/connectivity/AV_compRoutines.py | 146 ++++++++++++++++-- syncopy/connectivity/ST_compRoutines.py | 59 ++++++- syncopy/connectivity/connectivity_analysis.py | 32 +++- 5 files changed, 256 insertions(+), 65 deletions(-) diff --git a/dev_csd.py b/dev_csd.py index 3fbd31fe2..2985e1f20 100644 --- a/dev_csd.py +++ b/dev_csd.py @@ -4,6 +4,7 @@ from syncopy.connectivity.ST_compRoutines import cross_spectra_cF from syncopy.connectivity import wilson_sf from syncopy.connectivity.ST_compRoutines import cross_covariance_cF +from syncopy.connectivity.AV_compRoutines import normalize_ccov_cF import matplotlib.pyplot as ppl @@ -101,12 +102,12 @@ def phase_evo(omega0, eps, fs=1000, N=1000): return phase -eps = 0.3 * fs +eps = 0.0003 * fs omega = 50 * 2 * np.pi p1 = phase_evo(omega, eps, N=nSamples) p2 = phase_evo(omega, eps, N=nSamples) -s1 = np.cos(p1) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) -s2 = np.cos(p2) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) +s1 = 3 * np.cos(p1) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) +s2 = 20 * np.cos(p2) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) data = np.c_[s1, s2] bw = 10 @@ -119,13 +120,16 @@ def phase_evo(omega0, eps, fs=1000, N=1000): norm=False, fullOutput=True) -fig, ax = ppl.subplots(figsize=(6,4), num=1) -ax.set_xlabel('frequency (Hz)') -ax.set_ylabel('$|CSD(f)|$') -ax.set_ylim((-.02,1.25)) +# fig, ax = ppl.subplots(figsize=(6,4), num=1) +# ax.set_xlabel('frequency (Hz)') +# ax.set_ylabel('$|CSD(f)|$') +# ax.set_ylim((-.02,1.25)) -ax.plot(freqs, np.abs(CS2[0,:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) -ax.plot(freqs, np.abs(CS2[0,:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) -ax.plot(freqs, np.abs(CS2[0,:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) +# ax.plot(freqs, np.abs(CS2[0,:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) +# ax.plot(freqs, np.abs(CS2[0,:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) +# ax.plot(freqs, np.abs(CS2[0,:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) -ax.legend() +# ax.legend() + +CCov, lags = cross_covariance_cF(data, fs, norm=False, fullOutput=True) +Corr = normalize_ccov_cF(CCov) diff --git a/dev_frontend.py b/dev_frontend.py index 9e7ccff41..20191ad88 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -13,45 +13,37 @@ from syncopy.datatype import SpectralData, padding from syncopy.tests.misc import generate_artificial_data -tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=25) +tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=5) foilim = [1, 30] # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} -# this gives type(tsel) = list -# sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.array([0, 0.3, 1])} -sdict2 = {"trials": [0], 'toilim' : [-1, 0]} -print('sdict1') -# connectivityanalysis(data=tdat, select=sdict1, pad_to_length=4200) -# connectivityanalysis(data=tdat, select=sdict1, pad_to_length='nextpow2') - -# print('no selection') +# no problems here.. coherence = connectivityanalysis(data=tdat, - keeptrials=False, foilim=foilim, - output='pow') -# csd = connectivityanalysis(data=tdat, keeptrials=False)#, select=sdict2) -# connectivityanalysis(data=tdat, foilim = [20, 80]) + foilim=None, + output='pow') + +# a lot of problems here.. +# correlation = connectivityanalysis(data=tdat, method='corr', keeptrials=False) + # the hard wired dimord of the cF dimord = ['None', 'freq', 'channel_i', 'channel_j'] -# CrossSpectralData() -# CrossSpectralData(dimord=dimord) -# SpectralData() - -res = freqanalysis(data=tdat, - method='mtmfft', - samplerate=tdat.samplerate, -# order_max=20, -# foilim=foilim, -# foi=np.arange(502), - output='pow', -# polyremoval=1, - t_ftimwin=0.5, - keeptrials=False, - taper='dpss', - nTaper = 11, - tapsmofrq=5, - keeptapers=True, - parallel=False, # try this!!!!!! - select={"trials" : [0,1]}) + +# res = freqanalysis(data=tdat, +# method='mtmfft', +# samplerate=tdat.samplerate, +# # order_max=20, +# # foilim=foilim, +# # foi=np.arange(502), +# output='pow', +# # polyremoval=1, +# t_ftimwin=0.5, +# keeptrials=False, +# taper='dpss', +# nTaper = 11, +# tapsmofrq=5, +# keeptapers=True, +# parallel=False, # try this!!!!!! +# select={"trials" : [0,1]}) diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 7fa5ed3de..3e4fb9b5d 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -50,7 +50,7 @@ def normalize_csd_cF(trl_av_dat, trl_av_dat : (1, nFreq, N, N) :class:`numpy.ndarray` Cross-spectral densities for `N` x `N` channels and `nFreq` frequencies averaged over trials. - output : {'abs', 'pow', 'fourier', 'corr'}, default: 'abs' + output : {'abs', 'pow', 'fourier'}, default: 'abs' Also after normalization the coherency is still complex (`'fourier'`), to get the real valued coherence 0 < C_ij(f) < 1 one can either take the absolute (`'abs'`) or the absolute squared (`'pow'`) values of the @@ -70,8 +70,6 @@ def normalize_csd_cF(trl_av_dat, Notes ----- - This function also normalizes cross-covariances to cross-correlations. - This method is intended to be used as :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. @@ -103,8 +101,9 @@ def normalize_csd_cF(trl_av_dat, # re-shape to (nChannels x nChannels x nFreq) CS_ij = trl_av_dat.transpose(0, 2, 3, 1)[0, ...] - # main diagonal has shape (nChannels x nFreq): the auto spectra + # main diagonal has shape (nFreq x nChannels): the auto spectra diag = CS_ij.diagonal() + # get the needed product pairs of the autospectra Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj @@ -115,13 +114,12 @@ def normalize_csd_cF(trl_av_dat, return CS_ij[None, ...].transpose(0, 3, 1, 2) -class Normalize_CrossMeasure(ComputationalRoutine): +class NormalizeCrossSpectra(ComputationalRoutine): """ - Compute class that normalizes trial averaged quantities + Compute class that normalizes trial averaged csd's of :class:`~syncopy.CrossSpectralData` objects - like cross-spectra or cross-covariances to arrive at - coherencies or cross-correlations respectively. + to arrive at the respective coherencies. Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute @@ -139,10 +137,9 @@ class Normalize_CrossMeasure(ComputationalRoutine): method = "" # there is no backend # 1st argument,the data, gets omitted - method_keys = {} - cF_keys = list(signature(normalize_csd_cF).parameters.keys())[1:] + valid_kws = list(signature(normalize_csd_cF).parameters.keys())[1:] - def check_input(self): + def pre_check(self): ''' Make sure we have a trial average, so the input data only consists of `1 trial`. @@ -186,3 +183,130 @@ def process_metadata(self, data, out): out.channel_i = np.array(data.channel_i[chanSec]) out.channel_j = np.array(data.channel_j[chanSec]) out.freq = data.freq + + +@unwrap_io +def normalize_ccov_cF(trl_av_dat, + chunkShape=None, + noCompute=False): + + """ + Given the trial averaged cross-covariances, + we normalize with the 0-lag auto-covariances + (~averaged single trial variances) + to arrive at the cross-correlations. + + Parameters + ---------- + trl_av_dat : (nLag, 1, N, N) :class:`numpy.ndarray` + Cross-covariances for `N` x `N` channels + and `nLag` epochs averaged over trials. + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + + Returns + ------- + Corr_ij : (nLag, 1, N, N) :class:`numpy.ndarray` + Cross-correlations for all channel combinations i,j. + `N` corresponds to number of input channels. + + Notes + ----- + + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + See also + -------- + cross_covariance_cF : :func:`~syncopy.connectivity.ST_compRoutines.cross_covariance_cF` + Single trial cross covariances. + + """ + print('AV call, input shape', trl_av_dat.shape) + # it's the same as the input shape! + outShape = trl_av_dat.shape + + # For initialization of computational routine, + # just return output shape and dtype + # cross spectra are complex! + if noCompute: + return outShape, spectralDTypes['abs'] + + # re-shape to (nLag x nChannels x nChannels) + CCov_ij = trl_av_dat[:, 0, ...] + + # main diagonal has shape (nChannels x nChannels): + # the auto-covariances at 0-lag (~stds) + diag = trl_av_dat[0, 0, ...].diagonal() + + # get the needed product pairs + Ciijj = np.sqrt(diag[:, None] * diag[None, :]).T + CCov_ij = CCov_ij / Ciijj + + # re-attach dummy freq axis + return CCov_ij[:, None, ...] + + +class NormalizeCrossCov(ComputationalRoutine): + + """ + Compute class that normalizes trial averaged + cross-covariances of :class:`~syncopy.CrossSpectralData` objects + to arrive at the respective correlations + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.connectivityanalysis : parent metafunction + """ + + # the hard wired dimord of the cF + dimord = ['time', 'freq', 'channel_i', 'channel_j'] + + computeFunction = staticmethod(normalize_ccov_cF) + + method = "" # there is no backend + # 1st argument,the data, gets omitted + valid_kws = list(signature(normalize_ccov_cF).parameters.keys())[1:] + + def pre_check(self): + ''' + Make sure we have a trial average, + so the input data only consists of `1 trial`. + Can only be performed after initialization! + ''' + + if self.numTrials is None: + lgl = 'Initialize the computational Routine first!' + act = 'ComputationalRoutine not initialized!' + raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) + + if self.numTrials != 1: + lgl = "1 trial: normalizations can only be done on averaged quantities!" + act = f"DataSet contains {self.numTrials} trials" + raise SPYValueError(legal=lgl, varname="data", actual=act) + + def process_metadata(self, data, out): + + # Get trialdef array + channels from source + if data._selection is not None: + chanSec = data._selection.channel + trl = data._selection.trialdefinition + else: + chanSec = slice(None) + trl = data.trialdefinition + + out.trialdefinition = trl + # Attach remaining meta-data + out.samplerate = data.samplerate + out.channel_i = np.array(data.channel_i[chanSec]) + out.channel_j = np.array(data.channel_j[chanSec]) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 67cc45968..b635d4af6 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -311,7 +311,7 @@ def cross_covariance_cF(trl_dat, under the assumption that all inputs have been externally validated and cross-checked. """ - + print('ST call, input shape:', trl_dat.shape) # Re-arrange array if necessary and get dimensional information if timeAxis != 0: dat = trl_dat.T # does not copy but creates view of `trl_dat` @@ -354,10 +354,10 @@ def cross_covariance_cF(trl_dat, for i in range(nChannels): for j in range(i + 1): cc12 = fftconvolve(dat[:, i], dat[::-1, j], mode='same') - CC[:,0, i, j] = cc12[nSamples // 2:] / norm_overlap + CC[:, 0, i, j] = cc12[nSamples // 2:] / norm_overlap if i != j: - # cross-correlation is NOT symmetric.. - cc21 = fftconvolve(dat[:, j], dat[::-1, i], mode='same') + # cross-correlation is symmetric with C(tau) = C(-tau)^T + cc21 = cc12[::-1] CC[:, 0, j, i] = cc21[nSamples // 2:] / norm_overlap # normalize with products of std @@ -366,7 +366,58 @@ def cross_covariance_cF(trl_dat, N = STDs[:, None] * STDs[None, :] CC = CC / N + print("ST done, output shape:", CC.shape) if not fullOutput: return CC else: return CC, lags + + +class ST_CrossCovariance(ComputationalRoutine): + + """ + Compute class that calculates single-trial cross-covariances + of :class:`~syncopy.AnalogData` objects + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.connectivityanalysis : parent metafunction + """ + + # the hard wired dimord of the cF + dimord = ['time', 'freq', 'channel_i', 'channel_j'] + + computeFunction = staticmethod(cross_covariance_cF) + + method = "" # there is no backend + # 1st argument,the data, gets omitted + valid_kws = list(signature(cross_covariance_cF).parameters.keys())[1:] + + def process_metadata(self, data, out): + + # Get trialdef array + channels from source + if data._selection is not None: + chanSec = data._selection.channel + trl = data._selection.trialdefinition + else: + chanSec = slice(None) + trl = data.trialdefinition + + # If trial-averaging was requested, use the first trial as reference + # (all trials had to have identical lengths), and average onset timings + if not self.keeptrials: + t0 = trl[:, 2].mean() + trl = trl[[0], :] + trl[:, 2] = t0 + + out.trialdefinition = trl + # Attach remaining meta-data + out.samplerate = data.samplerate + out.channel_i = np.array(data.channel[chanSec]) + out.channel_j = np.array(data.channel[chanSec]) + + diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index b447ad6f3..1ab6cd3cb 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -31,8 +31,8 @@ from .const_def import ( availableMethods, ) -from .ST_compRoutines import ST_CrossSpectra -from .AV_compRoutines import Normalize_CrossMeasure +from .ST_compRoutines import ST_CrossSpectra, ST_CrossCovariance +from .AV_compRoutines import NormalizeCrossSpectra, NormalizeCrossCov __all__ = ["connectivityanalysis"] @@ -172,8 +172,20 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", st_dimord = ST_CrossSpectra.dimord # final normalization after trial averaging - av_compRoutine = Normalize_CrossMeasure(output=output) + av_compRoutine = NormalizeCrossSpectra(output=output) + if method == 'corr': + + # parallel computation over trials + st_compRoutine = ST_CrossCovariance(samplerate=data.samplerate, + padding_opt=padding_opt, + polyremoval=polyremoval, + timeAxis=timeAxis) + # hard coded as class attribute + st_dimord = ST_CrossCovariance.dimord + + av_compRoutine = NormalizeCrossCov() + # ------------------------------------------------- # Call the chosen single trial ComputationalRoutine # ------------------------------------------------- @@ -185,9 +197,17 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # Perform the trial-parallelized computation of the matrix quantity st_compRoutine.initialize(data, chan_per_worker=None, # no parallelisation over channel possible - keeptrials=False) # we need trial averaging! + keeptrials=keeptrials) # we need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) + # for debugging ccov + # print(5*'#',' after st_compRoutine call! ', 5*'#') + # print(st_out) + # print(st_out.trialdefinition) + # print(len(st_out.trials)) + # print(st_out.sampleinfo) + # return st_out + # ---------------------------------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine on the averaged ST output # ---------------------------------------------------------------------------------- @@ -207,9 +227,9 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # now take the trial average from the single trial CR as input av_compRoutine.initialize(st_out, chan_per_worker=None) - av_compRoutine.check_input() # make sure we got a trial_average + av_compRoutine.pre_check() # make sure we got a trial_average av_compRoutine.compute(st_out, out, parallel=False) - + # Either return newly created output object or simply quit return out if new_out else None From 795e6fb0c7c0a4af2924054a62a8bedf6656bed8 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 16 Nov 2021 21:56:52 +0100 Subject: [PATCH 041/109] WIP: First proof-of-concept implementation of `show` - make `_preview_trial` an abstract class method (required for all children) - attach `selectdata` to `BaseData` and only overload if necessary (currently only for `CrossSpectralData`: channel selection is not yet supported) - define `show` class method already in `BaseData`: only `_preview_trial` and `selectdata` are required (which are `BaseData` class methods anyway) On branch coherence Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/base_data.py | 56 ++++++++++++++++++++++++++--- syncopy/datatype/continuous_data.py | 47 +++++++++++++++++------- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index d129c403c..1b4e39dc0 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -18,16 +18,18 @@ from functools import reduce import shutil import numpy as np +from numpy.lib.arraysetops import isin from numpy.lib.format import open_memmap, read_magic import h5py import scipy as sp # Local imports import syncopy as spy +from .methods.selectdata import selectdata from syncopy.shared.tools import StructDict from syncopy.shared.parsers import (scalar_parser, array_parser, io_parser, filename_parser, data_parser) -from syncopy.shared.errors import SPYTypeError, SPYValueError, SPYError +from syncopy.shared.errors import SPYInfo, SPYTypeError, SPYValueError, SPYError from syncopy.datatype.methods.definetrial import definetrial as _definetrial from syncopy import __version__, __storage__, __acme__, __sessionid__, __storagelimit__ if __acme__: @@ -71,6 +73,8 @@ class BaseData(ABC): # Set caller for `SPYWarning` to not have it show up as '' _spwCaller = "BaseData.{}" + selectdata = selectdata + # Initialize hidden attributes used by all children _cfg = {} _filename = None @@ -528,18 +532,60 @@ def trialinfo(self): def trialinfo(self, trl): raise SPYError("Cannot set trialinfo. Use `BaseData._trialdefinition` or `syncopy.definetrial` instead.") - # Selector method - @abstractmethod - def selectdata(self, trials=None, deepcopy=False, **kwargs): + # # Selector method + # @abstractmethod + # def selectdata(self, trials=None, deepcopy=False, **kwargs): + # """ + # Docstring mostly pointing to ``selectdata`` + # """ + + # Show subsets of data + def show(self, **kwargs): """ - Docstring mostly pointing to ``selectdata`` + Coming soon... """ + # Account for pathological cases + if self.data is None: + SPYInfo("Empty object") + return + + # Leverage `selectdata` to sanitize input and perform subset picking + self.selectdata(inplace=True, **kwargs) + + SPYInfo("Showing{}".format(self._selection.__str__().partition("with")[-1])) + + idxList = [] + for trlno in self._selection.trials: + idxList.append(self._preview_trial(trlno).idx) + + singleIdx = [False] * len(idxList[0]) + returnIdx = list(idxList[0]) + for sk, selectors in enumerate(zip(*idxList)): + if np.unique(selectors).size == 1: + singleIdx[sk] = True + else: + if all(isinstance(sel, slice) for sel in selectors): + gaps = [selectors[k + 1].start - selectors[k].stop for k in range(len(selectors) - 1)] + if all(gap == 0 for gap in gaps): + singleIdx[sk] = True + returnIdx[sk] = slice(None) + + if all(si == True for si in singleIdx): + return self.data[tuple(returnIdx)] + else: + return [self.data[idx] for idx in idxList] + # Helper function that grabs a single trial @abstractmethod def _get_trial(self, trialno): pass + # Helper function that creates a `FauxTrial` object given actual trial information + @abstractmethod + def _preview_trial(self, trialno): + pass + # Convenience function, wiping contents of backing device from memory def clear(self): """Clear loaded data from memory diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 1e5bbf760..89ae76c83 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -426,22 +426,43 @@ def hdr(self): """ return self._hdr - # Selector method FIXME: use plotting-routine-like patching? - def selectdata(self, trials=None, channels=None, toi=None, toilim=None): - """ - Create new `AnalogData` object from selection + # # Selector method + # def selectdata(self, trials=None, channels=None, toi=None, toilim=None, inplace=False): + # """ + # Create new `AnalogData` object from selection - Please refer to :func:`syncopy.selectdata` for detailed usage information. + # Please refer to :func:`syncopy.selectdata` for detailed usage information. + + # Examples + # -------- + # >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) + + # See also + # -------- + # syncopy.selectdata : create new objects via deep-copy selections + # """ + # return selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim, inplace=inplace) + + # # Show data subsets + # def show(self, trials=None, channels=None, toi=None, toilim=None): + # """ + # FIXME!!!!!!!!!!!! + # Create new `AnalogData` object from selection + + # Please refer to :func:`syncopy.selectdata` for detailed usage information. + + # Examples + # -------- + # >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) + + # See also + # -------- + # syncopy.selectdata : create new objects via deep-copy selections + # """ + + # selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim, inplace=True) - Examples - -------- - >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) - See also - -------- - syncopy.selectdata : create new objects via deep-copy selections - """ - return selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim) # "Constructor" def __init__(self, From b32650540ccbc7b7bad64eb94520aa6ba5cb7c31 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 17 Nov 2021 11:48:21 +0100 Subject: [PATCH 042/109] CHG: New package function show - made `show` a stand-alone function + included complete docstring - cleaned up class method setup of `selectdata`: the routine is now attached to the `BaseData` class and thus inherently usable as class method in all children of `BaseData`. Currently, only `CrossSpectralData` overloads `selectdata` (due to missing `channel_i`/`channel_j` selection functionality) - fixed a bug that stumbled upon missing `parallel` keyword arg in `selectdata` - fixed a bug in `unwrap_cfg` that prevent the decorator to correctly parse functions that only use positional args + anonymous keywords (e.g., `f(x, **kwargs)`) - reset subset selection after picking data in `show` On branch coherence Changes to be committed: modified: doc/source/user/data_handling.rst modified: syncopy/datatype/__init__.py modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/discrete_data.py modified: syncopy/datatype/methods/selectdata.py new file: syncopy/datatype/methods/show.py modified: syncopy/shared/kwarg_decorators.py modified: syncopy/tests/spy_setup.py --- doc/source/user/data_handling.rst | 9 +- syncopy/datatype/__init__.py | 6 +- syncopy/datatype/base_data.py | 47 +----- syncopy/datatype/continuous_data.py | 57 -------- syncopy/datatype/discrete_data.py | 191 ++++++++++--------------- syncopy/datatype/methods/selectdata.py | 7 +- syncopy/datatype/methods/show.py | 131 +++++++++++++++++ syncopy/shared/kwarg_decorators.py | 8 +- syncopy/tests/spy_setup.py | 1 + 9 files changed, 230 insertions(+), 227 deletions(-) create mode 100644 syncopy/datatype/methods/show.py diff --git a/doc/source/user/data_handling.rst b/doc/source/user/data_handling.rst index ce85eb996..efe789edf 100644 --- a/doc/source/user/data_handling.rst +++ b/doc/source/user/data_handling.rst @@ -22,19 +22,20 @@ Reading and writing data with Syncopy syncopy.load syncopy.save -Functions for Editing Syncopy Data Objects -------------------------------------------- -Defining trials, data selection and padding. +Functions for Inspecting/Editing Syncopy Data Objects +----------------------------------------------------- +Defining trials, data selection and padding. .. autosummary:: syncopy.definetrial + syncopy.show syncopy.selectdata syncopy.padding Advanced Topics --------------- -More information about Syncopy's data class structure and file format. +More information about Syncopy's data class structure and file format. .. toctree:: diff --git a/syncopy/datatype/__init__.py b/syncopy/datatype/__init__.py index ae1d8b6af..03e33882a 100644 --- a/syncopy/datatype/__init__.py +++ b/syncopy/datatype/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Populate namespace with datatype routines and classes -# +# # Import __all__ routines from local modules from . import base_data, continuous_data, discrete_data, methods, statistical_data @@ -12,6 +12,7 @@ from .methods.definetrial import * from .methods.padding import * from .methods.selectdata import * +from .methods.show import * # Populate local __all__ namespace __all__ = [] @@ -22,3 +23,4 @@ __all__.extend(methods.definetrial.__all__) __all__.extend(methods.padding.__all__) __all__.extend(methods.selectdata.__all__) +__all__.extend(methods.show.__all__) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 1b4e39dc0..699e0f293 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -26,6 +26,7 @@ # Local imports import syncopy as spy from .methods.selectdata import selectdata +from .methods.show import show from syncopy.shared.tools import StructDict from syncopy.shared.parsers import (scalar_parser, array_parser, io_parser, filename_parser, data_parser) @@ -73,7 +74,9 @@ class BaseData(ABC): # Set caller for `SPYWarning` to not have it show up as '' _spwCaller = "BaseData.{}" + # Attach data selection and output routines to make them available as class methods selectdata = selectdata + show = show # Initialize hidden attributes used by all children _cfg = {} @@ -532,50 +535,6 @@ def trialinfo(self): def trialinfo(self, trl): raise SPYError("Cannot set trialinfo. Use `BaseData._trialdefinition` or `syncopy.definetrial` instead.") - # # Selector method - # @abstractmethod - # def selectdata(self, trials=None, deepcopy=False, **kwargs): - # """ - # Docstring mostly pointing to ``selectdata`` - # """ - - # Show subsets of data - def show(self, **kwargs): - """ - Coming soon... - """ - - # Account for pathological cases - if self.data is None: - SPYInfo("Empty object") - return - - # Leverage `selectdata` to sanitize input and perform subset picking - self.selectdata(inplace=True, **kwargs) - - SPYInfo("Showing{}".format(self._selection.__str__().partition("with")[-1])) - - idxList = [] - for trlno in self._selection.trials: - idxList.append(self._preview_trial(trlno).idx) - - singleIdx = [False] * len(idxList[0]) - returnIdx = list(idxList[0]) - for sk, selectors in enumerate(zip(*idxList)): - if np.unique(selectors).size == 1: - singleIdx[sk] = True - else: - if all(isinstance(sel, slice) for sel in selectors): - gaps = [selectors[k + 1].start - selectors[k].stop for k in range(len(selectors) - 1)] - if all(gap == 0 for gap in gaps): - singleIdx[sk] = True - returnIdx[sk] = slice(None) - - if all(si == True for si in singleIdx): - return self.data[tuple(returnIdx)] - else: - return [self.data[idx] for idx in idxList] - # Helper function that grabs a single trial @abstractmethod def _get_trial(self, trialno): diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 89ae76c83..65769bef1 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -426,44 +426,6 @@ def hdr(self): """ return self._hdr - # # Selector method - # def selectdata(self, trials=None, channels=None, toi=None, toilim=None, inplace=False): - # """ - # Create new `AnalogData` object from selection - - # Please refer to :func:`syncopy.selectdata` for detailed usage information. - - # Examples - # -------- - # >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) - - # See also - # -------- - # syncopy.selectdata : create new objects via deep-copy selections - # """ - # return selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim, inplace=inplace) - - # # Show data subsets - # def show(self, trials=None, channels=None, toi=None, toilim=None): - # """ - # FIXME!!!!!!!!!!!! - # Create new `AnalogData` object from selection - - # Please refer to :func:`syncopy.selectdata` for detailed usage information. - - # Examples - # -------- - # >>> ang2chan = ang.selectdata(channels=["channel01", "channel02"]) - - # See also - # -------- - # syncopy.selectdata : create new objects via deep-copy selections - # """ - - # selectdata(self, trials=trials, channels=channels, toi=toi, toilim=toilim, inplace=True) - - - # "Constructor" def __init__(self, data=None, @@ -620,25 +582,6 @@ def freq(self, freq): self._freq = np.array(freq) - # Selector method - def selectdata(self, trials=None, channels=None, toi=None, toilim=None, - foi=None, foilim=None, tapers=None): - """ - Create new `SpectralData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - - Examples - -------- - >>> spcBand = spc.selectdata(foilim=[10, 40]) - - See also - -------- - syncopy.selectdata : create new objects via deep-copy selections - """ - return selectdata(self, trials=trials, channels=channels, toi=toi, - toilim=toilim, foi=foi, foilim=foilim, tapers=tapers) - # Helper function that extracts frequency-related indices def _get_freq(self, foi=None, foilim=None): """ diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index bdb1bd53a..758f7e3b9 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Syncopy's abstract base class for discrete data + regular children -# +# # Builtin/3rd party package imports import numpy as np @@ -36,7 +36,7 @@ class DiscreteData(BaseData, ABC): @property def data(self): """array-like object representing data without trials - + Trials are concatenated along the time axis. """ @@ -47,7 +47,7 @@ def data(self): raise SPYValueError(legal=lgl, actual=act.format(self.filename), varname="data") return self._data - + @data.setter def data(self, inData): @@ -56,20 +56,20 @@ def data(self, inData): if inData is None: return - def __str__(self): + def __str__(self): # Get list of print-worthy attributes ppattrs = [attr for attr in self.__dir__() if not (attr.startswith("_") or attr in ["log", "trialdefinition", "hdr"])] ppattrs = [attr for attr in ppattrs if not (inspect.ismethod(getattr(self, attr)) or isinstance(getattr(self, attr), Iterator))] - + ppattrs.sort() # Construct string for pretty-printing class attributes dinfo = " '" + self._classname_to_extension()[1:] + "' x " dsep = "'-'" - + hdstr = "Syncopy {clname:s} object with fields\n\n" ppstr = hdstr.format(diminfo=dinfo + "'" + \ dsep.join(dim for dim in self.dimord) + "' " if self.dimord is not None else "Empty ", @@ -112,7 +112,7 @@ def __str__(self): valueString = str(value) ppstr += printString.format(attr, valueString) ppstr += "\nUse `.log` to see object history" - return ppstr + return ppstr @property def hdr(self): @@ -139,7 +139,7 @@ def samplerate(self, sr): if sr is None: self._samplerate = None return - + try: scalar_parser(sr, varname="samplerate", lims=[1, np.inf]) except Exception as exc: @@ -156,7 +156,7 @@ def trialid(self, trlid): if trlid is None: self._trialid = None return - + if self.data is None: print("SyNCoPy core - trialid: Cannot assign `trialid` without data. " + "Please assing data first") @@ -190,69 +190,69 @@ def trialtime(self): # Helper function that grabs a single trial def _get_trial(self, trialno): return self._data[self.trialid == trialno, :] - - # Helper function that spawns a `FauxTrial` object given actual trial information + + # Helper function that spawns a `FauxTrial` object given actual trial information def _preview_trial(self, trialno): """ Generate a `FauxTrial` instance of a trial - + Parameters ---------- trialno : int Number of trial the `FauxTrial` object is intended to mimic - + Returns ------- faux_trl : :class:`syncopy.datatype.base_data.FauxTrial` An instance of :class:`syncopy.datatype.base_data.FauxTrial` mainly - intended to be used in `noCompute` runs of + intended to be used in `noCompute` runs of :meth:`syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` - to avoid loading actual trial-data into memory. - + to avoid loading actual trial-data into memory. + See also -------- syncopy.datatype.base_data.FauxTrial : class definition and further details syncopy.shared.computational_routine.ComputationalRoutine : Syncopy compute engine """ - + trialIdx = np.where(self.trialid == trialno)[0] nCol = len(self.dimord) idx = [trialIdx.tolist(), slice(0, nCol)] if self._selection is not None: # selections are harmonized, just take `.time` idx[0] = trialIdx[self._selection.time[self._selection.trials.index(trialno)]].tolist() shp = [len(idx[0]), nCol] - + return FauxTrial(shp, tuple(idx), self.data.dtype, self.dimord) - + # Helper function that extracts by-trial timing-related indices def _get_time(self, trials, toi=None, toilim=None): """ Get relative by-trial indices of time-selections - + Parameters ---------- trials : list List of trial-indices to perform selection on toi : None or list - Time-points to be selected (in seconds) on a by-trial scale. + Time-points to be selected (in seconds) on a by-trial scale. toilim : None or list Time-window to be selected (in seconds) on a by-trial scale - + Returns ------- timing : list of lists - List of by-trial sample-indices corresponding to provided + List of by-trial sample-indices corresponding to provided time-selection. If both `toi` and `toilim` are `None`, `timing` - is a list of universal (i.e., ``slice(None)``) selectors. - + is a list of universal (i.e., ``slice(None)``) selectors. + Notes ----- - This class method is intended to be solely used by - :class:`syncopy.datatype.base_data.Selector` objects and thus has purely + This class method is intended to be solely used by + :class:`syncopy.datatype.base_data.Selector` objects and thus has purely auxiliary character. Therefore, all input sanitization and error checking - is left to :class:`syncopy.datatype.base_data.Selector` and not - performed here. - + is left to :class:`syncopy.datatype.base_data.Selector` and not + performed here. + See also -------- syncopy.datatype.base_data.Selector : Syncopy data selectors @@ -275,7 +275,7 @@ def _get_time(self, trials, toi=None, toilim=None): if sampSteps.min() == sampSteps.max() == 1: idxList = slice(idxList[0], idxList[-1] + 1, 1) timing.append(idxList) - + elif toi is not None: allTrials = self.trialtime for trlno in trials: @@ -296,17 +296,17 @@ def _get_time(self, trials, toi=None, toilim=None): if sampSteps.min() == sampSteps.max() == 1: idxList = slice(idxList[0], idxList[-1] + 1, 1) timing.append(idxList) - + else: timing = [slice(None)] * len(trials) - + return timing def __init__(self, data=None, samplerate=None, trialid=None, **kwargs): # Assign (default) values self._trialid = None - self._samplerate = None + self._samplerate = None self._hdr = None self._data = None @@ -316,7 +316,7 @@ def __init__(self, data=None, samplerate=None, trialid=None, **kwargs): self.samplerate = samplerate self.trialid = trialid self.data = data - + if self.data is not None: # In case of manual data allocation (reading routine would leave a @@ -332,7 +332,7 @@ class SpikeData(DiscreteData): This class can be used for representing spike trains. The data is always stored as a two-dimensional [nSpikes x 3] array on disk with the columns - being ``["sample", "channel", "unit"]``. + being ``["sample", "channel", "unit"]``. Data is only read from disk on demand, similar to memory maps and HDF5 files. @@ -342,16 +342,16 @@ class SpikeData(DiscreteData): _infoFileProperties = DiscreteData._infoFileProperties + ("channel", "unit",) _hdfFileAttributeProperties = DiscreteData._hdfFileAttributeProperties + ("channel",) _defaultDimord = ["sample", "channel", "unit"] - + @property def channel(self): - """ :class:`numpy.ndarray` : list of original channel names for each unit""" + """ :class:`numpy.ndarray` : list of original channel names for each unit""" # if data exists but no user-defined channel labels, create them on the fly if self._channel is None and self._data is not None: channelNumbers = np.unique(self.data[:, self.dimord.index("channel")]) return np.array(["channel" + str(int(i + 1)).zfill(len(str(channelNumbers.max() + 1))) for i in channelNumbers]) - + return self._channel @channel.setter @@ -361,12 +361,12 @@ def channel(self, chan): return if self.data is None: raise SPYValueError("Syncopy: Cannot assign `channels` without data. " + - "Please assign data first") + "Please assign data first") try: array_parser(chan, varname="channel", ntype="str") except Exception as exc: raise exc - + # Remove duplicate entries from channel array but preserve original order # (e.g., `[2, 0, 0, 1]` -> `[2, 0, 1`); allows for complex subset-selections _, idx = np.unique(chan, return_index=True) @@ -376,7 +376,7 @@ def channel(self, chan): lgl = "channel label array of length {0:d}".format(nchan) act = "array of length {0:d}".format(chan.size) raise SPYValueError(legal=lgl, varname="channel", actual=act) - + self._channel = chan @property @@ -393,11 +393,11 @@ def unit(self, unit): if unit is None: self._unit = None return - + if self.data is None: raise SPYValueError("Syncopy - SpikeData - unit: Cannot assign `unit` without data. " + "Please assign data first") - + nunit = np.unique(self.data[:, self.dimord.index("unit")]).size try: array_parser(unit, varname="unit", ntype="str", dims=(nunit,)) @@ -405,51 +405,33 @@ def unit(self, unit): raise exc self._unit = np.array(unit) - # Selector method - def selectdata(self, trials=None, toi=None, toilim=None, units=None, channels=None): - """ - Create new `SpikeData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - - Examples - -------- - >>> spkUnit01 = spk.selectdata(units=[0, 1]) - - See also - -------- - syncopy.selectdata : create new objects via deep-copy selections - """ - return selectdata(self, trials=trials, channels=channels, toi=toi, - toilim=toilim, units=units) - # Helper function that extracts by-trial unit-indices def _get_unit(self, trials, units=None): """ Get relative by-trial indices of unit selections - + Parameters ---------- trials : list List of trial-indices to perform selection on units : None or list List of unit-indices to be selected - + Returns ------- indices : list of lists - List of by-trial sample-indices corresponding to provided - unit-selection. If `units` is `None`, `indices` is a list of universal - (i.e., ``slice(None)``) selectors. - + List of by-trial sample-indices corresponding to provided + unit-selection. If `units` is `None`, `indices` is a list of universal + (i.e., ``slice(None)``) selectors. + Notes ----- - This class method is intended to be solely used by - :class:`syncopy.datatype.base_data.Selector` objects and thus has purely + This class method is intended to be solely used by + :class:`syncopy.datatype.base_data.Selector` objects and thus has purely auxiliary character. Therefore, all input sanitization and error checking - is left to :class:`syncopy.datatype.base_data.Selector` and not - performed here. - + is left to :class:`syncopy.datatype.base_data.Selector` and not + performed here. + See also -------- syncopy.datatype.base_data.Selector : Syncopy data selectors @@ -469,7 +451,7 @@ def _get_unit(self, trials, units=None): indices.append(trialUnits) else: indices = [slice(None)] * len(trials) - + return indices # "Constructor" @@ -489,13 +471,13 @@ def __init__(self, filename : str path to filename or folder (spy container) - trialdefinition : :class:`EventData` object or nTrials x 3 array + trialdefinition : :class:`EventData` object or nTrials x 3 array [start, stop, trigger_offset] sample indices for `M` trials samplerate : float sampling rate in Hz channel : str or list/array(str) original channel names - unit : str or list/array(str) + unit : str or list/array(str) names of all units dimord : list(str) ordered list of dimension labels @@ -514,7 +496,7 @@ def __init__(self, self._unit = None self._channel = None - + # Call parent initializer super().__init__(data=data, filename=filename, @@ -536,61 +518,44 @@ class EventData(DiscreteData): Data is only read from disk on demand, similar to memory maps and HDF5 files. - """ - + """ + _defaultDimord = ["sample", "eventid"] - + @property def eventid(self): """numpy.ndarray(int): integer event code assocated with each event""" if self.data is None: return None return np.unique(self.data[:, self.dimord.index("eventid")]) - - # Selector method - def selectdata(self, trials=None, toi=None, toilim=None, eventids=None): - """ - Create new `EventData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - - Examples - -------- - >>> evtStimOn = evt.selectdata(eventids=[1]) - - See also - -------- - syncopy.selectdata : create new objects via deep-copy selections - """ - return selectdata(self, trials=trials, toi=toi, toilim=toilim, eventids=eventids) # Helper function that extracts by-trial eventid-indices def _get_eventid(self, trials, eventids=None): """ Get relative by-trial indices of event-id selections - + Parameters ---------- trials : list List of trial-indices to perform selection on eventids : None or list List of event-id-indices to be selected - + Returns ------- indices : list of lists - List of by-trial sample-indices corresponding to provided - event-id-selection. If `eventids` is `None`, `indices` is a list of - universal (i.e., ``slice(None)``) selectors. - + List of by-trial sample-indices corresponding to provided + event-id-selection. If `eventids` is `None`, `indices` is a list of + universal (i.e., ``slice(None)``) selectors. + Notes ----- - This class method is intended to be solely used by - :class:`syncopy.datatype.base_data.Selector` objects and thus has purely + This class method is intended to be solely used by + :class:`syncopy.datatype.base_data.Selector` objects and thus has purely auxiliary character. Therefore, all input sanitization and error checking - is left to :class:`syncopy.datatype.base_data.Selector` and not - performed here. - + is left to :class:`syncopy.datatype.base_data.Selector` and not + performed here. + See also -------- syncopy.datatype.base_data.Selector : Syncopy data selectors @@ -610,9 +575,9 @@ def _get_eventid(self, trials, eventids=None): indices.append(trialEvents) else: indices = [slice(None)] * len(trials) - + return indices - + # "Constructor" def __init__(self, data=None, @@ -628,10 +593,10 @@ def __init__(self, filename : str path to filename or folder (spy container) - trialdefinition : :class:`EventData` object or nTrials x 3 array + trialdefinition : :class:`EventData` object or nTrials x 3 array [start, stop, trigger_offset] sample indices for `M` trials samplerate : float - sampling rate in Hz + sampling rate in Hz dimord : list(str) ordered list of dimension labels diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 29e40ca13..1ad35a455 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -236,10 +236,7 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None See also -------- - :meth:`syncopy.AnalogData.selectdata` : corresponding class method - :meth:`syncopy.SpectralData.selectdata` : corresponding class method - :meth:`syncopy.EventData.selectdata` : corresponding class method - :meth:`syncopy.SpikeData.selectdata` : corresponding class method + :func:`syncopy.show` : Show (subsets) of Syncopy objects """ # Ensure our one mandatory input is usable @@ -288,7 +285,7 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None # Create inventory of all available selectors and actually provided values # to create a bookkeeping dict for logging - provided = locals() + provided = {**locals(), **kwargs} available = get_defaults(data.selectdata) actualSelection = {} for key in available: diff --git a/syncopy/datatype/methods/show.py b/syncopy/datatype/methods/show.py new file mode 100644 index 000000000..a5ac125d7 --- /dev/null +++ b/syncopy/datatype/methods/show.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# Syncopy data slicing methods +# + +# Builtin/3rd party package imports +import numpy as np + +# Local imports +from syncopy.shared.errors import SPYInfo +from syncopy.shared.kwarg_decorators import unwrap_cfg + +__all__ = ["show"] + + +@unwrap_cfg +def show(data, **kwargs): + """ + Show (partial) contents of Syncopy object + + **Usage Notice** + + Syncopy uses HDF5 files as on-disk backing device for data storage. This + allows working with larger-than-memory data-sets by streaming only relevant + subsets of data from disk on demand without excessive RAM use. However, using + :func:`~syncopy.show` this mechanism is bypassed and the requested data subset + is loaded into memory at once. Thus, inadvertent usage of :func:`~syncopy.show` + on a large data object can lead to memory overflow or even out-of-memory errors. + + **Usage Summary** + + Data selectors for showing subsets of Syncopy data objects follow the syntax + of :func:`~syncopy.selectdata`. Please refer to :func:`~syncopy.selectdata` + for a list of valid data selectors for respective Syncopy data objects. + + Parameters + ---------- + data : Syncopy data object + As for subset-selection via :func:`~syncopy.selectdata`, the type of `data` + determines which keywords can be used. Some keywords are only valid for + certain types of Syncopy objects, e.g., "freqs" is not a valid selector + for an :class:`~syncopy.AnalogData` object. + **kwargs : keywords + Valid data selectors (e.g., `trials`, `channels`, `toi` etc.). Please + refer to :func:`~syncopy.selectdata` for a full list of available data + selectors. + + Returns + ------- + arr : NumPy nd-array + A (selection) of data retrieved from the `data` input object. + + Notes + ----- + This routine represents a convenience function for quickly inspecting the + contents of Syncopy objects. It is always possible to manually access an object's + numerical data by indexing the underlying HDF5 dataset: `data.data[idx]`. + The dimension labels of the dataset are encoded in `data.dimord`, e.g., if + `data` is a :class:`~syncopy.AnalogData` with `data.dimord` being `['time', 'channel']` + and `data.data.shape` is `(15000, 16)`, then `data.data[:, 3]` returns the + contents of the fourth channel across all time points. + + Examples + -------- + Use :func:`~syncopy.tests.misc.generate_artificial_data` to create a synthetic + :class:`syncopy.AnalogData` object. + + >>> from syncopy.tests.misc import generate_artificial_data + >>> adata = generate_artificial_data(nTrials=10, nChannels=32) + + Show the contents of `'channel02'` across all trials: + + >>> spy.show(adata, channels=['channel02']) + Syncopy INFO: In-place selection attached to data object: Syncopy AnalogData selector with 1 channels, all times, 10 trials + Syncopy INFO: Showing 1 channels, all times, 10 trials + Out[11]: + array([[1.627 ], + [1.7906], + [1.1757], + ..., + [1.1498], + [0.7753], + [1.0457]], dtype=float32) + + Note that this is equivalent to + + >>> adata.show(channels=['channel02']) + + See also + -------- + :func:`syncopy.selectdata` : Create a new Syncopy object from a selection + """ + + # Account for pathological cases + if data.data is None: + SPYInfo("Empty object, nothing to show") + return + + # Leverage `selectdata` to sanitize input and perform subset picking + data.selectdata(inplace=True, **kwargs) + + # Use an object's `_preview_trial` method fetch required indexing tuples + SPYInfo("Showing{}".format(data._selection.__str__().partition("with")[-1])) + idxList = [] + for trlno in data._selection.trials: + idxList.append(data._preview_trial(trlno).idx) + + # Perform some slicing/list-selection gymnastics: ensure that selections + # that result in contiguous slices are actually returned as such (e.g., + # `idxList = [(slice(1,2), [2]), (slice(2,3), [2])` -> `returnIdx = [slice(1,3), [2]]`) + singleIdx = [False] * len(idxList[0]) + returnIdx = list(idxList[0]) + for sk, selectors in enumerate(zip(*idxList)): + if np.unique(selectors).size == 1: + singleIdx[sk] = True + else: + if all(isinstance(sel, slice) for sel in selectors): + gaps = [selectors[k + 1].start - selectors[k].stop for k in range(len(selectors) - 1)] + if all(gap == 0 for gap in gaps): + singleIdx[sk] = True + returnIdx[sk] = slice(selectors[0].start, selectors[-1].stop) + + # Reset in-place subset selection + data._selection = None + + # If possible slice underlying dataset only once, otherwise return a list + # of arrays corresponding to selected trials + if all(si == True for si in singleIdx): + return data.data[tuple(returnIdx)] + else: + return [data.data[idx] for idx in idxList] diff --git a/syncopy/shared/kwarg_decorators.py b/syncopy/shared/kwarg_decorators.py index 41235c25a..316ba9bec 100644 --- a/syncopy/shared/kwarg_decorators.py +++ b/syncopy/shared/kwarg_decorators.py @@ -117,12 +117,16 @@ def unwrap_cfg(func): """ # Perform a little introspection gymnastics to get the name of the first - # positional and keyword argument of `func` + # positional and keyword argument of `func` (if we only find anonymous `**kwargs`, + # come up with an exemplary keyword - `kwarg0` is only used in the generated docstring) funcParams = inspect.signature(func).parameters paramList = list(funcParams) kwargList = [pName for pName, pVal in funcParams.items() if pVal.default != pVal.empty] arg0 = paramList[0] - kwarg0 = kwargList[0] + if len(kwargList) > 0: + kwarg0 = kwargList[0] + else: + kwarg0 = "some_parameter" @functools.wraps(func) def wrapper_cfg(*args, **kwargs): diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index cc2a275ca..1bda9289b 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -25,5 +25,6 @@ # Test stuff within here... artdata = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) + sys.exit() spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") From 759be68a68e925f7d3a9621e5f2b9b66bfeb1c85 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 17 Nov 2021 12:52:55 +0100 Subject: [PATCH 043/109] CHG: Updated testing pipeline - only run memory-intensive specest tests on machine w/sufficient RAM (at least 10 GB) - changes introduced in `selectdata` and new `show` functionality do not interfere w/tests (closes #145) On branch coherence Changes to be committed: modified: syncopy/tests/test_specest.py --- syncopy/tests/test_specest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index 7aa5e9aab..074bef5a5 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -4,13 +4,12 @@ # # Builtin/3rd party package imports -from multiprocessing import Value import os import tempfile import inspect +import psutil import gc import pytest -import time import numpy as np import scipy.signal as scisig from numpy.lib.format import open_memmap @@ -30,6 +29,10 @@ # Decorator to decide whether or not to run dask-related tests skip_without_acme = pytest.mark.skipif(not __acme__, reason="acme not available") +# Decorator to decide whether or not to run memory-intensive tests +availMem = psutil.virtual_memory().total +skip_low_mem = pytest.mark.skipif(availMem < 10 * 1024**3, reason="less than 10GB RAM available") + # Local helper for constructing TF testing signals def _make_tf_signal(nChannels, nTrials, seed, fadeIn=None, fadeOut=None): @@ -824,6 +827,7 @@ def test_tf_irregular_trials(self): assert np.array_equal(origTime, tfSpec.time[tk]) @skip_without_acme + @skip_low_mem def test_tf_parallel(self, testcluster): # collect all tests of current class and repeat them running concurrently client = dd.Client(testcluster) @@ -1114,6 +1118,7 @@ def test_wav_irregular_trials(self): assert np.array_equal(origTime, tfSpec.time[tk]) @skip_without_acme + @skip_low_mem def test_wav_parallel(self, testcluster): # collect all tests of current class and repeat them running concurrently client = dd.Client(testcluster) From bd5f032251089977161475f119a7c92d3a98d3fb Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 17 Nov 2021 13:36:38 +0100 Subject: [PATCH 044/109] FIX: fix backend tests - `taper_opt` dict like `padding_opt` instead of `taperopt` Changes to be committed: modified: test_connectivity.py modified: test_timefreq.py --- syncopy/tests/backend/test_connectivity.py | 8 +++++--- syncopy/tests/backend/test_timefreq.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index b22a896fb..f425c160e 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -102,12 +102,14 @@ def test_csd(): for ps in phase_shifts] data = np.array(data).T data = np.array(data) + np.random.randn(nSamples, len(phase_shifts)) - - Kmax = 8 # multiple tapers for single trial coherence + + bw = 5 #Hz + NW = nSamples * bw / (2 * fs) + Kmax = int(2 * NW - 1) # multiple tapers for single trial coherence CSD, freqs = stCR.cross_spectra_cF(data, fs, polyremoval=1, taper='dpss', - taperopt={'Kmax' : Kmax, 'NW' : 6}, + taper_opt={'Kmax' : Kmax, 'NW' : NW}, norm=True, fullOutput=True) diff --git a/syncopy/tests/backend/test_timefreq.py b/syncopy/tests/backend/test_timefreq.py index d2ea8ed87..559fb8290 100644 --- a/syncopy/tests/backend/test_timefreq.py +++ b/syncopy/tests/backend/test_timefreq.py @@ -153,10 +153,10 @@ def test_mtmconvol(): # ------------------------- taper = 'dpss' - taperopt = {'Kmax' : 10, 'NW' : 2} + taper_opt = {'Kmax' : 10, 'NW' : 2} # the transforms have shape (nTime, nTaper, nFreq, nChannel) ftr2, freqs2 = mtmconvol.mtmconvol(signal, - samplerate=fs, taper=taper, taperopt=taperopt, + samplerate=fs, taper=taper, taper_opt=taper_opt, nperseg=window_size, noverlap=window_size - 1) @@ -371,8 +371,8 @@ def test_mtmfft(): # test multi-taper analysis # ------------------------- - taperopt = {'Kmax' : 8, 'NW' : 1} - ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taperopt=taperopt) + taper_opt = {'Kmax' : 8, 'NW' : 1} + ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taper_opt=taper_opt) # average over tapers dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel @@ -386,8 +386,8 @@ def test_mtmfft(): # test kaiser taper (is boxcar for beta -> inf) # ----------------- - taperopt = {'beta' : 2} - ftr, freqs = mtmfft.mtmfft(signal, fs, taper="kaiser", taperopt=taperopt) + taper_opt = {'beta' : 2} + ftr, freqs = mtmfft.mtmfft(signal, fs, taper="kaiser", taper_opt=taper_opt) # average over tapers (only 1 here) kaiser_spec = np.real(ftr * ftr.conj()).mean(axis=0) kaiser_amplitudes = np.sqrt(kaiser_spec)[:, 0] # only 1 channel @@ -399,7 +399,7 @@ def test_mtmfft(): # ------------------------------- for win in windows.__all__: - taperopt = {} + taper_opt = {} # that guy isn't symmetric if win == 'exponential': continue @@ -407,7 +407,7 @@ def test_mtmfft(): if win == 'hanning': continue try: - ftr, freqs = mtmfft.mtmfft(signal, fs, taper=win, taperopt=taperopt) + ftr, freqs = mtmfft.mtmfft(signal, fs, taper=win, taper_opt=taper_opt) # average over tapers (only 1 here) spec = np.real(ftr * ftr.conj()).mean(axis=0) amplitudes = np.sqrt(spec)[:, 0] # only 1 channel From 15177152e4cd3a476bb5a728a509d4fb4ce863b7 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 17 Nov 2021 18:21:02 +0100 Subject: [PATCH 045/109] WIP: Granger-Geweke: Wilson's Algorithm - Spectral Matrix Factorization - basic vectorized algorithm, converges pretty fast for the test cases so far Changes to be committed: modified: syncopy/connectivity/wilson_sf.py --- syncopy/connectivity/wilson_sf.py | 58 +++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index 6d180bf4c..07cdefd62 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -13,7 +13,7 @@ import numpy as np -def wilson_sf(CSD, samplerate, nIter=500, tol=1e-9): +def wilson_sf(CSD, samplerate, nIter=2, tol=1e-9): ''' Wilsons spectral matrix factorization ("analytic method") @@ -32,12 +32,41 @@ def wilson_sf(CSD, samplerate, nIter=500, tol=1e-9): ''' + nFreq, nChannels = CSD.shape[:2] + + Ident = np.eye(*CSD.shape[1:]) + + # nChannel x nChannel psi0 = _psi0_initial(CSD) + + # initial choice of psi, constant for all z(~f) + psi = np.tile(psi0, (nFreq, 1, 1)) + assert psi.shape == CSD.shape + + for i in range(nIter): + + psi_inv = np.linalg.inv(psi) + + # the bracket of equation 3.1 + g = psi_inv @ CSD # stacked matrix multiplication: np.matmul + g = g @ psi_inv.conj().transpose(0, 2, 1) + gplus, gplus_0 = _plusOperator(g + Ident) + + # the 'any' matrix + S = np.triu(gplus_0) + S = S - S.conj().T # S + S* = 0 + + # the next step psi_{tau+1} + psi = psi @ (gplus + S) + + CSDfac = psi @ psi.conj().transpose(0, 2, 1) + err = np.abs(CSD - CSDfac) + err = err / np.abs(CSD) # relative error + print(err.max()) + + return CSDfac, err - g = np.zeros(CSD.shape) - g = 0 # :D - def _psi0_initial(CSD): ''' @@ -52,7 +81,7 @@ def _psi0_initial(CSD): # perform ifft to obtain gammas. gamma = np.fft.ifft(CSD, axis=0) gamma0 = gamma[0, ...] - + # Remove any assymetry due to rounding error. # This also will zero out any imaginary values # on the diagonal - real diagonals are required for cholesky. @@ -66,7 +95,7 @@ def _psi0_initial(CSD): else: psi0 = np.ones((nSamples, nSamples)) - return psi0 + return psi0.T def _plusOperator(g): @@ -75,21 +104,28 @@ def _plusOperator(g): The []+ operator from definition 1.2, given by explicit Fourier transformations - The time x nChannel x nChannel matrix `g` is given + The nFreq x nChannel x nChannel matrix `g` is given in the frequency domain. ''' # 'negative lags' from the ifft nLag = g.shape[0] // 2 - # the series expansion in beta_k + # the series expansion in beta_k beta = np.fft.ifft(g, axis=0) # take half of the zero lag beta[0, ...] = 0.5 * beta[0, ...] - g0 = beta[0, ...] + g0 = beta[0, ...].copy() - # Zero out negative powers. - beta[:nLag + 1, ..., ...] = 0 + # Zero out negative freqs + beta[nLag + 1:, ...] = 0 gp = np.fft.fft(beta, axis=0) return gp, g0 + + +def _mem_size(arr): + ''' + Gives array size in MB + ''' + return f'{arr.size * arr.itemsize / 1e6:.2f} MB' From 215c386b7084cbae3522bbf42674db90c16cfcf3 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 17 Nov 2021 19:35:18 +0100 Subject: [PATCH 046/109] FIX: Modify stacking logic in ComputationalRoutine - new mandatory positional argument in `initialize` of `ComputationalRoutine`: `out_dimord` provides the output object's dimensional labels. This is used to determine the correct stacking dimension for trials - do not default to the first dimension as stacking dimension: instead use the new `stackingDim` variable (which is determined based on `out_dimord`, see above) - closes #130 On branch comproutine_fix Changes to be committed: modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/datatype/methods/selectdata.py modified: syncopy/shared/computational_routine.py modified: syncopy/specest/freqanalysis.py modified: syncopy/tests/spy_setup.py modified: syncopy/tests/test_computationalroutine.py modified: syncopy/tests/test_specest.py --- syncopy/connectivity/connectivity_analysis.py | 35 ++++++++--------- syncopy/datatype/methods/selectdata.py | 2 +- syncopy/shared/computational_routine.py | 38 +++++++++++++------ syncopy/specest/freqanalysis.py | 15 ++++---- syncopy/tests/spy_setup.py | 1 - syncopy/tests/test_computationalroutine.py | 4 +- syncopy/tests/test_specest.py | 1 + 7 files changed, 57 insertions(+), 39 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 1ab6cd3cb..3177dbee2 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -43,7 +43,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", out=None, + nTaper=None, toi="all", out=None, **kwargs): """ @@ -130,15 +130,15 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", nSamples = int(lenTrials.min()) # --- Basic foi sanitization --- - + foi, foilim = validate_foi(foi, foilim, data.samplerate) # only now set foi array for foilim in 1Hz steps if foilim: foi = np.arange(foilim[0], foilim[1] + 1) - + # --- Settingn up specific Methods --- - + if method == 'coh': if foi is None and foilim is None: @@ -149,7 +149,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", SPYInfo(msg) foi = freqs - # sanitize taper selection and retrieve dpss settings + # sanitize taper selection and retrieve dpss settings taper_opt = validate_taper(taper, tapsmofrq, nTaper, @@ -157,7 +157,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", foimax=foi.max(), samplerate=data.samplerate, nSamples=nSamples, - output="pow") # ST_CSD's always have this unit/norm + output="pow") # ST_CSD's always have this unit/norm # parallel computation over trials st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, @@ -167,10 +167,10 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", polyremoval=polyremoval, timeAxis=timeAxis, foi=foi) - + # hard coded as class attribute st_dimord = ST_CrossSpectra.dimord - + # final normalization after trial averaging av_compRoutine = NormalizeCrossSpectra(output=output) @@ -183,9 +183,9 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", timeAxis=timeAxis) # hard coded as class attribute st_dimord = ST_CrossCovariance.dimord - + av_compRoutine = NormalizeCrossCov() - + # ------------------------------------------------- # Call the chosen single trial ComputationalRoutine # ------------------------------------------------- @@ -196,23 +196,24 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # Perform the trial-parallelized computation of the matrix quantity st_compRoutine.initialize(data, + st_out.dimord, chan_per_worker=None, # no parallelisation over channel possible - keeptrials=keeptrials) # we need trial averaging! + keeptrials=keeptrials) # we need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) # for debugging ccov - # print(5*'#',' after st_compRoutine call! ', 5*'#') - # print(st_out) + # print(5*'#',' after st_compRoutine call! ', 5*'#') + # print(st_out) # print(st_out.trialdefinition) # print(len(st_out.trials)) # print(st_out.sampleinfo) # return st_out - + # ---------------------------------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine on the averaged ST output # ---------------------------------------------------------------------------------- - # If provided, make sure output object is appropriate + # If provided, make sure output object is appropriate if out is not None: try: data_parser(out, varname="out", writable=True, empty=True, @@ -225,8 +226,8 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", out = CrossSpectralData(dimord=st_dimord) new_out = True - # now take the trial average from the single trial CR as input - av_compRoutine.initialize(st_out, chan_per_worker=None) + # now take the trial average from the single trial CR as input + av_compRoutine.initialize(st_out, out.dimord, chan_per_worker=None) av_compRoutine.pre_check() # make sure we got a trial_average av_compRoutine.compute(st_out, out, parallel=False) diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 1ad35a455..5f1b68209 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -293,7 +293,7 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None # Fire up `ComputationalRoutine`-subclass to do the actual selecting/copying selectMethod = DataSelection() - selectMethod.initialize(data, chan_per_worker=kwargs.get("chan_per_worker")) + selectMethod.initialize(data, out.dimord, chan_per_worker=kwargs.get("chan_per_worker")) selectMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=actualSelection) diff --git a/syncopy/shared/computational_routine.py b/syncopy/shared/computational_routine.py index 936f0bf1c..c533f3151 100644 --- a/syncopy/shared/computational_routine.py +++ b/syncopy/shared/computational_routine.py @@ -26,7 +26,7 @@ # Local imports from .tools import get_defaults from syncopy import __storage__, __acme__, __path__ -from syncopy.shared.errors import SPYValueError, SPYWarning, SPYParallelError +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYParallelError, SPYWarning if __acme__: from acme import ParallelMap import dask.distributed as dd @@ -226,7 +226,7 @@ def __init__(self, *argv, **kwargs): self._callMax = 10000 self._callCount = 0 - def initialize(self, data, chan_per_worker=None, keeptrials=True): + def initialize(self, data, out_dimord, chan_per_worker=None, keeptrials=True): """ Perform dry-run of calculation to determine output shape @@ -235,6 +235,8 @@ def initialize(self, data, chan_per_worker=None, keeptrials=True): data : syncopy data object Syncopy data object to be processed (has to be the same object that is passed to :meth:`compute` for the actual calculation). + out_dimord : list + Data dimension labels of output object chan_per_worker : None or int Number of channels to be processed by each worker (only relevant in case of concurrent processing). If `chan_per_worker` is `None` (default) @@ -291,17 +293,34 @@ def initialize(self, data, chan_per_worker=None, keeptrials=True): dtp_list.append(dtype) trials.append(trial) + # Determine trial stacking dimension and compute aggregate shape of output + if "time" in out_dimord: # AnalogData + SpectralData + CrossSpectralData + stackingDim = out_dimord.index("time") + elif "lag" in out_dimord: # CrossSpectralData + stackingDim = out_dimord.index("lag") + elif "sample" in out_dimord: # SpikeData + EventData + stackingDim = out_dimord.index("sample") + else: + msg = "output object with dimord containing valid stacking dimension" + raise SPYTypeError(out_dimord, varname="out_dimord", expected=msg) + totalSize = sum(cShape[stackingDim] for cShape in chk_list) + outputShape = list(chunkShape) + outputShape[stackingDim] = totalSize + # The aggregate shape is computed as max across all chunks chk_arr = np.array(chk_list) - if np.unique(chk_arr[:, 0]).size > 1 and not self.keeptrials: + chunkShape = tuple(chk_arr.max(axis=0)) + if np.unique(chk_arr[:, stackingDim]).size > 1 and not self.keeptrials: + import pdb; pdb.set_trace() err = "Averaging trials of unequal lengths in output currently not supported!" raise NotImplementedError(err) if np.any([dtp_list[0] != dtp for dtp in dtp_list]): lgl = "unique output dtype" act = "{} different output dtypes".format(np.unique(dtp_list).size) raise SPYValueError(legal=lgl, varname="dtype", actual=act) - chunkShape = tuple(chk_arr.max(axis=0)) - self.outputShape = (chk_arr[:, 0].sum(),) + chunkShape[1:] + + # Save determined shapes and data type + self.outputShape = tuple(outputShape) self.cfg["chunkShape"] = chunkShape self.dtype = np.dtype(dtp_list[0]) @@ -383,18 +402,15 @@ def initialize(self, data, chan_per_worker=None, keeptrials=True): sourceLayout.append(trial.idx) # Construct dimensional layout of output - # FIXME: should be targetLayout[0][stackingDim].stop - # FIXME: should be lyt[stackingDim] = slice(stacking, stacking + chkshp[stackingDim]) - # FIXME: should be stacking += chkshp[stackingDim] - stacking = targetLayout[0][0].stop + stacking = targetLayout[0][stackingDim].stop for tk in range(1, self.numTrials): trial = trials[tk] trlArg = tuple(arg[tk] if isinstance(arg, Sized) and len(arg) == self.numTrials \ else arg for arg in self.argv) chkshp = chk_list[tk] lyt = [slice(0, stop) for stop in chkshp] - lyt[0] = slice(stacking, stacking + chkshp[0]) - stacking += chkshp[0] + lyt[stackingDim] = slice(stacking, stacking + chkshp[stackingDim]) + stacking += chkshp[stackingDim] if chan_per_worker is None: targetLayout.append(tuple(lyt)) targetShapes.append(tuple([slc.stop - slc.start for slc in lyt])) diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 25d9af937..36a6de8fd 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -32,7 +32,7 @@ # Local imports from .const_def import ( availableWavelets, - availableMethods, + availableMethods, ) from .compRoutines import ( @@ -198,7 +198,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', Order of polynomial used for de-trending data in the time domain prior to spectral analysis. A value of 0 corresponds to subtracting the mean ("de-meaning"), ``polyremoval = 1`` removes linear trends (subtracting the - least squares fit of a linear polynomial). + least squares fit of a linear polynomial). If `polyremoval` is `None`, no de-trending is performed. Note that for spectral estimation de-meaning is very advisable and hence also the default. @@ -421,7 +421,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Shortcut to data sampling interval dt = 1 / data.samplerate - + foi, foilim = validate_foi(foi, foilim, data.samplerate) # see also https://docs.obspy.org/_modules/obspy/signal/detrend.html#polynomial @@ -553,7 +553,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', act = "empty frequency selection" raise SPYValueError(legal=lgl, varname="foi/foilim", actual=act) - # sanitize taper selection and retrieve dpss settings + # sanitize taper selection and retrieve dpss settings taper_opt = validate_taper(taper, tapsmofrq, nTaper, @@ -579,7 +579,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # method specific parameters method_kwargs = { 'samplerate' : data.samplerate, - 'taper' : taper, + 'taper' : taper, 'taper_opt' : taper_opt } @@ -598,7 +598,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', elif method == "mtmconvol": _check_effective_parameters(MultiTaperFFTConvol, defaults, lcls) - + # Process `toi` for sliding window multi taper fft, # we have to account for three scenarios: (1) center sliding # windows on all samples in (selected) trials (2) `toi` was provided as @@ -906,7 +906,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', postSelect, toi=toi, timeAxis=timeAxis, - polyremoval=polyremoval, + polyremoval=polyremoval, output_fmt=output, method_kwargs=method_kwargs) @@ -929,6 +929,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Perform actual computation specestMethod.initialize(data, + out.dimord, chan_per_worker=kwargs.get("chan_per_worker"), keeptrials=keeptrials) specestMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=log_dct) diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 1bda9289b..cc2a275ca 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -25,6 +25,5 @@ # Test stuff within here... artdata = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) - sys.exit() spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") diff --git a/syncopy/tests/test_computationalroutine.py b/syncopy/tests/test_computationalroutine.py index c5b06e93a..05fe43fbf 100644 --- a/syncopy/tests/test_computationalroutine.py +++ b/syncopy/tests/test_computationalroutine.py @@ -64,12 +64,12 @@ def process_metadata(self, data, out): def filter_manager(data, b=None, a=None, out=None, select=None, chan_per_worker=None, keeptrials=True, parallel=False, parallel_store=None, log_dict=None): - myfilter = LowPassFilter(b, a=a) - myfilter.initialize(data, chan_per_worker=chan_per_worker, keeptrials=keeptrials) newOut = False if out is None: newOut = True out = AnalogData(dimord=AnalogData._defaultDimord) + myfilter = LowPassFilter(b, a=a) + myfilter.initialize(data, out.dimord, chan_per_worker=chan_per_worker, keeptrials=keeptrials) myfilter.compute(data, out, parallel=parallel, parallel_store=parallel_store, diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index 074bef5a5..f6d2a75bc 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -918,6 +918,7 @@ class TestWavelet(): "channels": range(0, int(nChannels / 2)), "toilim": [-20, 60.8]}] + @skip_low_mem def test_wav_solution(self): # Compute TF specturm across entire time-interval (use integer-valued From 1dc4a400ae48fb383c646c36c035fb2ef819c80d Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 18 Nov 2021 12:27:06 +0100 Subject: [PATCH 047/109] WIP: Started putting together arithmetic operators - first foray into supporting base arithmetic w/Syncopy objects (non-functional atm) On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/base_data.py new file: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/base_data.py | 4 + syncopy/datatype/methods/arithmetic.py | 111 +++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 syncopy/datatype/methods/arithmetic.py diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 699e0f293..4d54a5380 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -25,6 +25,7 @@ # Local imports import syncopy as spy +from .methods.arithmetic import _add from .methods.selectdata import selectdata from .methods.show import show from syncopy.shared.tools import StructDict @@ -728,6 +729,9 @@ def __del__(self): shutil.rmtree(os.path.splitext(self.filename)[0], ignore_errors=True) + def __add__(self, other): + return _add(self, other) + # Class "constructor" def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): """ diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py new file mode 100644 index 000000000..4f6c3cef7 --- /dev/null +++ b/syncopy/datatype/methods/arithmetic.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# +# Syncopy object arithmetics +# + +# Builtin/3rd party package imports +import numbers +import numpy as np + +# Local imports +from syncopy.shared.parsers import data_parser +from syncopy.shared.tools import get_defaults +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYInfo, SPYWarning +from syncopy.shared.computational_routine import ComputationalRoutine +from syncopy.shared.kwarg_decorators import unwrap_cfg, unwrap_io, detect_parallel_client +from syncopy.shared.computational_routine import ComputationalRoutine + +__all__ = [] + +def _add(obj1, obj2): + + _parse_input(obj1, obj2, "+") + pass + +def _parse_input(obj1, obj2, operator): + + # Determine which input is a Syncopy object (depending on lef/right application of + # operator, i.e., `data + 1` or `1 + data`). Can be both as well, but we just need + # one `baseObj` to get going + if "BaseData" in str(obj1.__class__.__mro__): + baseObj = obj1 + operand = obj2 + elif "BaseData" in str(obj2.__class__.__mro__): + baseObj = obj2 + operand = obj1 + + # Ensure our base object is not empty + try: + data_parser(baseObj, varname="data", empty=False) + except Exception as exc: + raise exc + + # If only a subset of `data` is worked on, adjust for this + if baseObj._selection is not None: + trialList = baseObj._selection.trials + else: + trialList = list(range(len(baseObj.trials))) + + # Use the `_preview_trial` functionality of Syncopy objects to get each trial's + # shape and dtype (existing selections are taken care of automatically) + baseTrials = [baseObj._preview_trial(trlno) for trlno in trialList] + + # Depending on the what is thrown at `baseObj` perform more or less extensive parsing + # First up: operand is a scalar + if isinstance(operand, numbers.Number): + if np.isinf(operand): + raise SPYValueError("finite scalar", varname="operand", actual=str(operand)) + if operator == "/" and operand == 0: + raise SPYValueError("non-zero scalar for division", varname="operand", actual=str(operand)) + + # Operand is array-like + elif isinstance(operand, (np.ndarray, list)): + + # First, ensure operand is a NumPy array to make things easier + operand = np.array(operand) + + # Ensure complex and real values are not mashed together + if np.all(np.iscomplex(operand)): + sameType = lambda dt : "complex" in dt.name + else: + sameType = lambda dt : "complex" not in dt.name + if not all(sameType(trl.dtype) for trl in baseTrials): + lgl = "array of same numerical type (real/complex)" + raise SPYTypeError(operand, varname="operand", expected=lgl) + + # Ensure shapes match up + if not all(trl.shape == operand.shape for trl in baseTrials): + lgl = "array of compatible shape" + act = "array with shape {}" + raise SPYValueError(lgl, varname="operand", actual=act.format(operand.shape)) + + # All good, nevertheless warn of potential havoc this operation may cause... + msg = "Performing arithmetic with NumPy arrays may cause inconsistency " +\ + "in Syncopy objects (channels, samplerate, trialdefintions etc.)" + SPYWarning(msg, caller=operator) + + # Operand is another Syncopy object + elif "BaseData" in str(operand.__class__.__mro__): + + # First, ensure operand is same object class and has same `dimord` as `baseObj`` + try: + data_parser(operand, varname="operand", + dataclass=baseObj.__class__.__name__, empty=False) + except Exception as exc: + raise exc + + opndTrials = [operand._preview_trial(trlno) for trlno in trialList] + + else: + typeerror + +@unwrap_io +def arithmetic_cF(trl_dat, noCompute=False, chunkShape=None): + """ + Coming soon... + """ + pass + +class SpyArithmetic(ComputationalRoutine): + + computeFunction = staticmethod(arithmetic_cF) \ No newline at end of file From aac7eaf5153252d6169b059402e7f4c1d3aac32f Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 18 Nov 2021 13:24:14 +0100 Subject: [PATCH 048/109] NEW/CHG: Introduced new BaseData property _stackingDim - a new property `_stackingDim` and class attribute `_stackingDimLabel` has been introduced to tie trial stacking mechanics to the respective class definitions - the stacking algorithm in `ComputationalRoutine` has been modified accordingly On branch comproutine_fix Changes to be committed: modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/discrete_data.py modified: syncopy/datatype/methods/selectdata.py modified: syncopy/shared/computational_routine.py modified: syncopy/specest/freqanalysis.py modified: syncopy/tests/test_computationalroutine.py --- syncopy/connectivity/connectivity_analysis.py | 4 ++-- syncopy/datatype/base_data.py | 16 +++++++++++----- syncopy/datatype/continuous_data.py | 3 +++ syncopy/datatype/discrete_data.py | 2 ++ syncopy/datatype/methods/selectdata.py | 2 +- syncopy/shared/computational_routine.py | 19 +++++++------------ syncopy/specest/freqanalysis.py | 2 +- syncopy/tests/test_computationalroutine.py | 2 +- 8 files changed, 28 insertions(+), 22 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 3177dbee2..4433c96fc 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -196,7 +196,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # Perform the trial-parallelized computation of the matrix quantity st_compRoutine.initialize(data, - st_out.dimord, + st_out._stackingDim, chan_per_worker=None, # no parallelisation over channel possible keeptrials=keeptrials) # we need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) @@ -227,7 +227,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", new_out = True # now take the trial average from the single trial CR as input - av_compRoutine.initialize(st_out, out.dimord, chan_per_worker=None) + av_compRoutine.initialize(st_out, out._stackingDim, chan_per_worker=None) av_compRoutine.pre_check() # make sure we got a trial_average av_compRoutine.compute(st_out, out, parallel=False) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 699e0f293..fb741e6f5 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -70,6 +70,7 @@ class BaseData(ABC): # Dummy allocations of class attributes that are actually initialized in subclasses _mode = None + _stackingDimLabel = None # Set caller for `SPYWarning` to not have it show up as '' _spwCaller = "BaseData.{}" @@ -108,6 +109,11 @@ class BaseData(ABC): def _defaultDimord(cls): return NotImplementedError + @property + def _stackingDim(self): + if self._stackingDimLabel is not None and self.dimord is not None: + return self.dimord.index(self._stackingDimLabel) + @property def cfg(self): """Dictionary of previous operations on data""" @@ -353,11 +359,11 @@ def dimord(self, dims): return # this enforces the _defaultDimord - # if set(dims) != set(self._defaultDimord): - # base = "dimensional labels {}" - # lgl = base.format("'" + "' x '".join(str(dim) for dim in self._defaultDimord) + "'") - # act = base.format("'" + "' x '".join(str(dim) for dim in dims) + "'") - # raise SPYValueError(legal=lgl, varname="dimord", actual=act) + if set(dims) != set(self._defaultDimord): + base = "dimensional labels {}" + lgl = base.format("'" + "' x '".join(str(dim) for dim in self._defaultDimord) + "'") + act = base.format("'" + "' x '".join(str(dim) for dim in dims) + "'") + raise SPYValueError(legal=lgl, varname="dimord", actual=act) # this enforces that custom dimords are set for every axis if len(dims) != len(self._defaultDimord): diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 65769bef1..bd6b6c2f1 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -413,6 +413,7 @@ class AnalogData(ContinuousData): _infoFileProperties = ContinuousData._infoFileProperties + ("_hdr",) _defaultDimord = ["time", "channel"] + _stackingDimLabel = "time" # Attach plotting routines to not clutter the core module code singlepanelplot = _plot_analog.singlepanelplot @@ -521,6 +522,7 @@ class SpectralData(ContinuousData): _infoFileProperties = ContinuousData._infoFileProperties + ("taper", "freq",) _defaultDimord = ["time", "taper", "freq", "channel"] + _stackingDimLabel = "time" # Attach plotting routines to not clutter the core module code singlepanelplot = _plot_spectral.singlepanelplot @@ -663,6 +665,7 @@ class CrossSpectralData(ContinuousData): _infoFileProperties = ContinuousData._infoFileProperties + ("freq",) _defaultDimord = ["time", "freq", "channel_i", "channel_j"] + _stackingDimLabel = "time" _channel_i = None _channel_j = None _samplerate = None diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index 758f7e3b9..993036430 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -342,6 +342,7 @@ class SpikeData(DiscreteData): _infoFileProperties = DiscreteData._infoFileProperties + ("channel", "unit",) _hdfFileAttributeProperties = DiscreteData._hdfFileAttributeProperties + ("channel",) _defaultDimord = ["sample", "channel", "unit"] + _stackingDimLabel = "sample" @property def channel(self): @@ -521,6 +522,7 @@ class EventData(DiscreteData): """ _defaultDimord = ["sample", "eventid"] + _stackingDimLabel = "sample" @property def eventid(self): diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 5f1b68209..8aa6c0e88 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -293,7 +293,7 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None # Fire up `ComputationalRoutine`-subclass to do the actual selecting/copying selectMethod = DataSelection() - selectMethod.initialize(data, out.dimord, chan_per_worker=kwargs.get("chan_per_worker")) + selectMethod.initialize(data, out._stackingDim, chan_per_worker=kwargs.get("chan_per_worker")) selectMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=actualSelection) diff --git a/syncopy/shared/computational_routine.py b/syncopy/shared/computational_routine.py index c533f3151..663d4200c 100644 --- a/syncopy/shared/computational_routine.py +++ b/syncopy/shared/computational_routine.py @@ -226,7 +226,7 @@ def __init__(self, *argv, **kwargs): self._callMax = 10000 self._callCount = 0 - def initialize(self, data, out_dimord, chan_per_worker=None, keeptrials=True): + def initialize(self, data, out_stackingdim, chan_per_worker=None, keeptrials=True): """ Perform dry-run of calculation to determine output shape @@ -235,8 +235,8 @@ def initialize(self, data, out_dimord, chan_per_worker=None, keeptrials=True): data : syncopy data object Syncopy data object to be processed (has to be the same object that is passed to :meth:`compute` for the actual calculation). - out_dimord : list - Data dimension labels of output object + out_stackingdim : int + Index of data dimension for stacking trials in output object chan_per_worker : None or int Number of channels to be processed by each worker (only relevant in case of concurrent processing). If `chan_per_worker` is `None` (default) @@ -294,17 +294,12 @@ def initialize(self, data, out_dimord, chan_per_worker=None, keeptrials=True): trials.append(trial) # Determine trial stacking dimension and compute aggregate shape of output - if "time" in out_dimord: # AnalogData + SpectralData + CrossSpectralData - stackingDim = out_dimord.index("time") - elif "lag" in out_dimord: # CrossSpectralData - stackingDim = out_dimord.index("lag") - elif "sample" in out_dimord: # SpikeData + EventData - stackingDim = out_dimord.index("sample") - else: - msg = "output object with dimord containing valid stacking dimension" - raise SPYTypeError(out_dimord, varname="out_dimord", expected=msg) + stackingDim = out_stackingdim totalSize = sum(cShape[stackingDim] for cShape in chk_list) outputShape = list(chunkShape) + if stackingDim < 0 or stackingDim >= len(outputShape): + msg = "valid trial stacking dimension" + raise SPYTypeError(out_stackingdim, varname="out_stackingdim", expected=msg) outputShape[stackingDim] = totalSize # The aggregate shape is computed as max across all chunks diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 36a6de8fd..1b9fcccb4 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -929,7 +929,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Perform actual computation specestMethod.initialize(data, - out.dimord, + out._stackingDim, chan_per_worker=kwargs.get("chan_per_worker"), keeptrials=keeptrials) specestMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=log_dct) diff --git a/syncopy/tests/test_computationalroutine.py b/syncopy/tests/test_computationalroutine.py index 05fe43fbf..df2f0ed7b 100644 --- a/syncopy/tests/test_computationalroutine.py +++ b/syncopy/tests/test_computationalroutine.py @@ -69,7 +69,7 @@ def filter_manager(data, b=None, a=None, newOut = True out = AnalogData(dimord=AnalogData._defaultDimord) myfilter = LowPassFilter(b, a=a) - myfilter.initialize(data, out.dimord, chan_per_worker=chan_per_worker, keeptrials=keeptrials) + myfilter.initialize(data, out._stackingDim, chan_per_worker=chan_per_worker, keeptrials=keeptrials) myfilter.compute(data, out, parallel=parallel, parallel_store=parallel_store, From bdc843e7008f3b7b1a2da6d2624380c38a8e98cf Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 18 Nov 2021 14:58:40 +0100 Subject: [PATCH 049/109] FIX: Repaired (accidentally correct) stackingDim for DiscreteData - both `DiscreteData` classes use 2d arrays for storing data (nSamples x 3); the `stackingDim` in this case *has* to be 0. This has been fixed. - use (potentially non-standard) `stackingDim` for fetching trials of `ContinuousData` objects On branch comproutine_fix Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/base_data.py | 7 +++++-- syncopy/datatype/continuous_data.py | 21 +++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index fb741e6f5..6eea452cb 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -111,8 +111,11 @@ def _defaultDimord(cls): @property def _stackingDim(self): - if self._stackingDimLabel is not None and self.dimord is not None: - return self.dimord.index(self._stackingDimLabel) + if any(["DiscreteData" in str(base) for base in self.__class__.__mro__]): + return 0 + else: + if self._stackingDimLabel is not None and self.dimord is not None: + return self.dimord.index(self._stackingDimLabel) @property def cfg(self): diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index bd6b6c2f1..a59f8f449 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -128,10 +128,9 @@ def __str__(self): @property def _shapes(self): if self.sampleinfo is not None: - sid = self.dimord.index("time") shp = [list(self.data.shape) for k in range(self.sampleinfo.shape[0])] for k, sg in enumerate(self.sampleinfo): - shp[k][sid] = sg[1] - sg[0] + shp[k][self._stackingDim] = sg[1] - sg[0] return [tuple(sp) for sp in shp] @property @@ -225,8 +224,7 @@ def time(self): # Helper function that grabs a single trial def _get_trial(self, trialno): idx = [slice(None)] * len(self.dimord) - sid = self.dimord.index("time") - idx[sid] = slice(int(self.sampleinfo[trialno, 0]), int(self.sampleinfo[trialno, 1])) + idx[self._stackingDim] = slice(int(self.sampleinfo[trialno, 0]), int(self.sampleinfo[trialno, 1])) return self._data[tuple(idx)] def _is_empty(self): @@ -257,11 +255,10 @@ def _preview_trial(self, trialno): """ shp = list(self.data.shape) idx = [slice(None)] * len(self.dimord) - tidx = self.dimord.index("time") stop = int(self.sampleinfo[trialno, 1]) start = int(self.sampleinfo[trialno, 0]) - shp[tidx] = stop - start - idx[tidx] = slice(start, stop) + shp[self._stackingDim] = stop - start + idx[self._stackingDim] = slice(start, stop) # process existing data selections if self._selection is not None: @@ -281,16 +278,16 @@ def _preview_trial(self, trialno): # account for trial offsets an compute slicing index + shape start = start + tstart stop = start + (tstop - tstart) - idx[tidx] = slice(start, stop) - shp[tidx] = stop - start + idx[self._stackingDim] = slice(start, stop) + shp[self._stackingDim] = stop - start else: - idx[tidx] = [tp + start for tp in tsel] - shp[tidx] = len(tsel) + idx[self._stackingDim] = [tp + start for tp in tsel] + shp[self._stackingDim] = len(tsel) # process the rest dims = list(self.dimord) - dims.pop(dims.index("time")) + dims.pop(self._stackingDim) for dim in dims: sel = getattr(self._selection, dim) if sel: From 324c222fae12a8e77020e404b4651f56ded931d0 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 18 Nov 2021 18:19:07 +0100 Subject: [PATCH 050/109] WIP: Stabilising Wilson - CSD matrices are not well conditioned if nSamples/nChannels is somewhat small - brute force epsilon-regularization helps, and the factorization look still very good. But better to still need to investigate further.. Changes to be committed: new file: dev_granger.py modified: syncopy/connectivity/wilson_sf.py --- dev_granger.py | 52 ++++++++++++++++++ syncopy/connectivity/wilson_sf.py | 90 ++++++++++++++++++++++++++----- 2 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 dev_granger.py diff --git a/dev_granger.py b/dev_granger.py new file mode 100644 index 000000000..d5616fb67 --- /dev/null +++ b/dev_granger.py @@ -0,0 +1,52 @@ +import numpy as np +from scipy.signal import csd as sci_csd +import scipy.signal as sci +from syncopy.connectivity.wilson_sf import wilson_sf +from syncopy.connectivity.ST_compRoutines import cross_spectra_cF +#from syncopy.connectivity import wilson_sf +import matplotlib.pyplot as ppl + + +# noisy phase evolution +def phase_evo(omega0, eps, fs=1000, N=1000): + wn = np.random.randn(N) + delta_ts = np.ones(N) * 1 / fs + phase = np.cumsum(omega0 * delta_ts + eps * wn) + return phase + + +def plot_wilson_errs(errs, label='', c='k'): + + fig, ax = ppl.subplots(figsize=(6,4), num=2) + ax.set_xlabel('Iteration Step') + ax.set_ylabel(r'rel. Error $\frac{|CSD - \Psi\Psi^*|}{|CSD|}$') + ax.semilogy() + # ax.plot(errs, '-o', label=label, c=c) + + fig.subplots_adjust(left=0.15, bottom=0.2) + return ax + + +def make_test_data(nChannels=10, nSamples=1000, bw=5): + + # white noise ensemble + fs = 1000 + tvec = np.arange(nSamples) / fs + + data = np.zeros((nSamples, nChannels)) + for i in range(nChannels): + p1 = phase_evo(30 * 2 * np.pi, 0.1, N=nSamples) + p2 = phase_evo(60 * 2 * np.pi, 0.25, N=nSamples) + + data[:, i] = np.cos(p1) + np.sin(p2) + .5 * np.random.randn(nSamples) + #data[:, i] = 2 * np.random.randn(nSamples) + + bw = 5 + NW = bw * nSamples / (2 * fs) + Kmax = int(2 * NW - 1) # optimal number of tapers + + CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, norm=False, fullOutput=True) + + CSD = CS2[0, ...] + + return CSD diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index 07cdefd62..f9e2ddd28 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -13,11 +13,49 @@ import numpy as np -def wilson_sf(CSD, samplerate, nIter=2, tol=1e-9): +def regularize_csd(CSD, cond_max=1e6, reg_max=-3): + + ''' + Brute force regularize CSD matrix + by inspecting the maximal condition number + along the frequency axis. + Multiply with different epsilon * I, + starting with eps = 1e-12 until the + condition number is smaller than `cond_max` + or the maximal regularization factor was reached. + + Inspection/Check of the used regularization constant + epsilon is highly recommended! + ''' + + reg_factors = np.logspace(-12, reg_max, 100) + I = np.eye(CSD.shape[1]) + + CondNum = np.linalg.cond(CSD).max() + + # nothing to be done + if CondNum < cond_max: + return CSD + + for factor in reg_factors: + CSDreg = CSD + factor * I + CondNum = np.linalg.cond(CSDreg).max() + print(f'Factor: {factor}, CN: {CondNum}') + + if CondNum < cond_max: + return CSDreg, factor + + # raise sth.. + + +def wilson_sf(CSD, samplerate, nIter=100, rtol=1e-9): ''' Wilsons spectral matrix factorization ("analytic method") + Converges extremely fast, so the default number of + iterations should be enough in practical situations. + This is a pure backend function and hence no input argument checking is performed. @@ -25,7 +63,7 @@ def wilson_sf(CSD, samplerate, nIter=2, tol=1e-9): ---------- CSD : (nFreq, N, N) :class:`numpy.ndarray` Complex cross spectra for all channel combinations i,j. - `N` corresponds to number of input channels. + `N` corresponds to number of input channels. Returns ------- @@ -43,28 +81,49 @@ def wilson_sf(CSD, samplerate, nIter=2, tol=1e-9): psi = np.tile(psi0, (nFreq, 1, 1)) assert psi.shape == CSD.shape - for i in range(nIter): - - psi_inv = np.linalg.inv(psi) + errs = [] + for _ in range(nIter): + psi_inv = np.linalg.inv(psi) # the bracket of equation 3.1 - g = psi_inv @ CSD # stacked matrix multiplication: np.matmul - g = g @ psi_inv.conj().transpose(0, 2, 1) + g = psi_inv @ CSD @ psi_inv.conj().transpose(0, 2, 1) gplus, gplus_0 = _plusOperator(g + Ident) # the 'any' matrix S = np.triu(gplus_0) S = S - S.conj().T # S + S* = 0 + psi_old = psi # the next step psi_{tau+1} psi = psi @ (gplus + S) - CSDfac = psi @ psi.conj().transpose(0, 2, 1) - err = np.abs(CSD - CSDfac) - err = err / np.abs(CSD) # relative error - print(err.max()) + rel_err = np.abs((psi - psi_old) / np.abs(psi)) + # print(rel_err.max()) + # mean relative error + CSDfac = psi @ psi.conj().transpose(0, 2, 1) + err = np.abs(CSD - CSDfac) + err = err / np.abs(CSD) # relative error + + print('Cond', np.linalg.cond(psi[0])) + print('Error:', err.max(),'\n') + + errs.append(err.max()) + + Aks = np.fft.ifft(psi, axis=0) + A0 = Aks[0, ...] + + # Noise Covariance + Sigma = A0 * A0.T + # strip off remaining imaginary parts + Sigma = np.real(Sigma) - return CSDfac, err + # Transfer function + A0inv = np.linalg.inv(A0) + Hfunc = psi @ A0inv.conj().T + + # print(err.mean()) + + return Hfunc, Sigma, CSDfac, errs def _psi0_initial(CSD): @@ -97,7 +156,7 @@ def _psi0_initial(CSD): return psi0.T - +# from scipy.signal import windows def _plusOperator(g): ''' @@ -117,10 +176,13 @@ def _plusOperator(g): beta[0, ...] = 0.5 * beta[0, ...] g0 = beta[0, ...].copy() - # Zero out negative freqs + # Zero out negative lags beta[nLag + 1:, ...] = 0 + # beta = beta * windows.tukey(len(beta), alpha=0.2)[:, None, None] + gp = np.fft.fft(beta, axis=0) + return gp, g0 From 22f7eba7b6778ebcf617c18c4a72a3bbfb874481 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 18 Nov 2021 19:36:39 +0100 Subject: [PATCH 051/109] WIP: More work on arithmetic - started putting together `computeFunction` and first skeleton of `ComputationalRoutine` - attempt to propagate actual arithmetic operation as lambda; will need to check if dask can serialize/deserialize this... Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 126 +++++++++++++++++++++---- 1 file changed, 110 insertions(+), 16 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 4f6c3cef7..13ff5ff09 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -6,6 +6,7 @@ # Builtin/3rd party package imports import numbers import numpy as np +import h5py # Local imports from syncopy.shared.parsers import data_parser @@ -19,8 +20,8 @@ def _add(obj1, obj2): - _parse_input(obj1, obj2, "+") - pass + operand_dat, opres_type, operand_idxs = _parse_input(obj1, obj2, "+") + operation = lambda x, y : x + y def _parse_input(obj1, obj2, operator): @@ -40,31 +41,40 @@ def _parse_input(obj1, obj2, operator): except Exception as exc: raise exc - # If only a subset of `data` is worked on, adjust for this - if baseObj._selection is not None: - trialList = baseObj._selection.trials - else: - trialList = list(range(len(baseObj.trials))) + # If no active selection is present, create a "fake" all-to-all selection + # to harmonize processing down the road + if baseObj._selection is None: + baseObj.selectdata(inplace=True) + baseTrialList = baseObj._selection.trials # Use the `_preview_trial` functionality of Syncopy objects to get each trial's # shape and dtype (existing selections are taken care of automatically) - baseTrials = [baseObj._preview_trial(trlno) for trlno in trialList] + baseTrials = [baseObj._preview_trial(trlno) for trlno in baseTrialList] # Depending on the what is thrown at `baseObj` perform more or less extensive parsing # First up: operand is a scalar if isinstance(operand, numbers.Number): + + # Don't allow `np.inf` manipulations and catch zero-divisions if np.isinf(operand): raise SPYValueError("finite scalar", varname="operand", actual=str(operand)) if operator == "/" and operand == 0: raise SPYValueError("non-zero scalar for division", varname="operand", actual=str(operand)) + # Determine numeric type of operation's result + opres_type = np.result_type(*(trl.dtype for trl in baseTrials), operand) + + # That's it set output vars + operand_dat = operand + operand_idxs = None + # Operand is array-like elif isinstance(operand, (np.ndarray, list)): # First, ensure operand is a NumPy array to make things easier operand = np.array(operand) - # Ensure complex and real values are not mashed together + # Ensure complex and real values are not mashed up if np.all(np.iscomplex(operand)): sameType = lambda dt : "complex" in dt.name else: @@ -73,12 +83,19 @@ def _parse_input(obj1, obj2, operator): lgl = "array of same numerical type (real/complex)" raise SPYTypeError(operand, varname="operand", expected=lgl) + # Determine the numeric type of the operation's result + opres_type = np.result_type(*(trl.dtype for trl in baseTrials), operand.dtype) + # Ensure shapes match up if not all(trl.shape == operand.shape for trl in baseTrials): lgl = "array of compatible shape" act = "array with shape {}" raise SPYValueError(lgl, varname="operand", actual=act.format(operand.shape)) + # No more info needed, the array is the only quantity we need + operand_dat = operand + operand_idxs = None + # All good, nevertheless warn of potential havoc this operation may cause... msg = "Performing arithmetic with NumPy arrays may cause inconsistency " +\ "in Syncopy objects (channels, samplerate, trialdefintions etc.)" @@ -87,25 +104,102 @@ def _parse_input(obj1, obj2, operator): # Operand is another Syncopy object elif "BaseData" in str(operand.__class__.__mro__): - # First, ensure operand is same object class and has same `dimord` as `baseObj`` + # Ensure operand object class, and `dimord` match up (and it's non-empty) try: - data_parser(operand, varname="operand", + data_parser(operand, varname="operand", dimord=baseObj.dimord, dataclass=baseObj.__class__.__name__, empty=False) except Exception as exc: raise exc - opndTrials = [operand._preview_trial(trlno) for trlno in trialList] + # Make sure samplerates are identical (if present) + baseSr = getattr(baseObj, "samplerate") + opndSr = getattr(operand, "samplerate") + if baseSr != opndSr: + lgl = "Syncopy objects with identical samplerate" + act = "Syncopy object with samplerates {} and {}, respectively" + raise SPYValueError(lgl, varname="operand", + actual=act.format(baseSr, opndSr)) + + # If only a subset of `operand` is selected, adjust for this + if operand._selection is not None: + opndTrialList = operand._selection.trials + else: + opndTrialList = list(range(len(operand.trials))) + + # Ensure the same number of trials is about to be processed + opndTrials = [operand._preview_trial(trlno) for trlno in opndTrialList] + if len(opndTrials) != len(baseTrials): + lgl = "Syncopy object with same number of trials (selected)" + act = "Syncopy object with {} trials (selected)" + raise SPYValueError(lgl, varname="operand", actual=act.format(len(opndTrials))) + + # Ensure complex and real values are not mashed up + baseIsComplex = ["complex" in trl.dtype.name for trl in baseTrials] + opndIsComplex = ["complex" in trl.dtype.name for trl in opndTrials] + if baseIsComplex != opndIsComplex: + lgl = "Syncopy data object of same numerical type (real/complex)" + raise SPYTypeError(operand, varname="operand", expected=lgl) + + # Determine the numeric type of the operation's result + opres_type = np.result_type(*(trl.dtype for trl in baseTrials), + *(trl.dtype for trl in opndTrials)) + + # Finally, ensure shapes align + if not all(baseTrials[k].shape == opndTrials[k].shape for k in range(len(baseTrials))): + lgl = "Syncopy object (selection) of compatible shapes {}" + act = "Syncopy object (selection) with shapes {}" + baseShapes = [trl.shape for trl in baseTrials] + opndShapes = [trl.shape for trl in opndTrials] + raise SPYValueError(lgl.format(baseShapes), varname="operand", + actual=act.format(opndShapes)) + # Propagate indices for fetching data from operand + operand_idxs = [trl.idx for trl in opndTrials] + + # Assemble dict with relevant info for performing operation + operand_dat = {"filename" : operand.filename, + "dsetname" : operand._hdfFileDatasetProperties[0]} + + # If `operand` is anything else it's invalid for performing arithmetic on else: - typeerror + lgl = "Syncopy object, scalar or array-like" + raise SPYTypeError(operand, varname="operand", expected=lgl) + + return operand_dat, opres_type, operand_idxs @unwrap_io -def arithmetic_cF(trl_dat, noCompute=False, chunkShape=None): +def arithmetic_cF(base_dat, operand_dat, operand_idx, operation=None, opres_type = None, + noCompute=False, chunkShape=None): """ Coming soon... """ - pass + + if noCompute: + return base_dat.shape, opres_type + + if isinstance(operand_dat, dict): + with h5py.File(operand_dat["filename"], "r") as h5f: + operand = h5f[operand_dat["dsetname"]][operand_idx] + else: + operand = operand_dat + + return operation(base_dat, operand) class SpyArithmetic(ComputationalRoutine): - computeFunction = staticmethod(arithmetic_cF) \ No newline at end of file + computeFunction = staticmethod(arithmetic_cF) + + def process_metadata(self, data, out): + + # Get/set timing-related selection modifiers + out.trialdefinition = data._selection.trialdefinition + # if data._selection._timeShuffle: # FIXME: should be implemented done the road + # out.time = data._selection.timepoints + if data._selection._samplerate: + out.samplerate = data.samplerate + + # Get/set dimensional attributes changed by selection + for prop in data._selection._dimProps: + selection = getattr(data._selection, prop) + if selection is not None: + setattr(out, prop, getattr(data, prop)[selection]) From 7f0acc1d965eaea7dfe62ba2586e4526c16fbac7 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Fri, 19 Nov 2021 16:43:07 +0100 Subject: [PATCH 052/109] WIP: Granger-Geweke causality: Wilson cleanup - added convergence checking by tracking the relative error of the factorization Changes to be committed: modified: dev_granger.py modified: syncopy/connectivity/wilson_sf.py --- dev_granger.py | 28 +++++- syncopy/connectivity/wilson_sf.py | 162 ++++++++++++++++++------------ 2 files changed, 121 insertions(+), 69 deletions(-) diff --git a/dev_granger.py b/dev_granger.py index d5616fb67..3c1e1a24c 100644 --- a/dev_granger.py +++ b/dev_granger.py @@ -1,7 +1,7 @@ import numpy as np from scipy.signal import csd as sci_csd import scipy.signal as sci -from syncopy.connectivity.wilson_sf import wilson_sf +from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd from syncopy.connectivity.ST_compRoutines import cross_spectra_cF #from syncopy.connectivity import wilson_sf import matplotlib.pyplot as ppl @@ -15,6 +15,13 @@ def phase_evo(omega0, eps, fs=1000, N=1000): return phase +def brown_noise(N): + wn = np.random.randn(N) + xs = np.cumsum(wn) + + return xs + + def plot_wilson_errs(errs, label='', c='k'): fig, ax = ppl.subplots(figsize=(6,4), num=2) @@ -39,7 +46,7 @@ def make_test_data(nChannels=10, nSamples=1000, bw=5): p2 = phase_evo(60 * 2 * np.pi, 0.25, N=nSamples) data[:, i] = np.cos(p1) + np.sin(p2) + .5 * np.random.randn(nSamples) - #data[:, i] = 2 * np.random.randn(nSamples) + # data[:, i] = brown_noise(nSamples) bw = 5 NW = bw * nSamples / (2 * fs) @@ -50,3 +57,20 @@ def make_test_data(nChannels=10, nSamples=1000, bw=5): CSD = CS2[0, ...] return CSD + + +def cond_samples(Ns, nChannels=2): + + cns = [] + + for N in Ns: + cn = np.linalg.cond(make_test_data(bw=5, nSamples=N, nChannels=nChannels)).max() + cns.append(cn) + + ax = ppl.gca() + ax.set_xlabel('nSamples') + ax.set_ylabel('Condition Number') + ax.plot(Ns, cns, '-o', label=f'nChannels={nChannels}') + ax.set_ylim((-1, 5000)) + + diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index f9e2ddd28..3d2e1aef2 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -13,48 +13,13 @@ import numpy as np -def regularize_csd(CSD, cond_max=1e6, reg_max=-3): - - ''' - Brute force regularize CSD matrix - by inspecting the maximal condition number - along the frequency axis. - Multiply with different epsilon * I, - starting with eps = 1e-12 until the - condition number is smaller than `cond_max` - or the maximal regularization factor was reached. - - Inspection/Check of the used regularization constant - epsilon is highly recommended! - ''' - - reg_factors = np.logspace(-12, reg_max, 100) - I = np.eye(CSD.shape[1]) - - CondNum = np.linalg.cond(CSD).max() - - # nothing to be done - if CondNum < cond_max: - return CSD - - for factor in reg_factors: - CSDreg = CSD + factor * I - CondNum = np.linalg.cond(CSDreg).max() - print(f'Factor: {factor}, CN: {CondNum}') - - if CondNum < cond_max: - return CSDreg, factor - - # raise sth.. - - -def wilson_sf(CSD, samplerate, nIter=100, rtol=1e-9): +def wilson_sf(CSD, nIter=100, rtol=1e-9): ''' Wilsons spectral matrix factorization ("analytic method") Converges extremely fast, so the default number of - iterations should be enough in practical situations. + iterations should be more than enough in practical situations. This is a pure backend function and hence no input argument checking is performed. @@ -63,11 +28,25 @@ def wilson_sf(CSD, samplerate, nIter=100, rtol=1e-9): ---------- CSD : (nFreq, N, N) :class:`numpy.ndarray` Complex cross spectra for all channel combinations i,j. - `N` corresponds to number of input channels. + `N` corresponds to number of input channels. Has to be + positive definite and well conditioned. + nIter : int + Maximum Number of iterations, factorization result + is returned also if error tolerance wasn't met. + rtol : float + Tolerance of the relative maximal + error of the factorization. Returns ------- - + Hfunc : (nFreq, N, N) :class:`numpy.ndarray` + The transfer functio + Sigma : (N, N) :class:`numpy.ndarray` + Noise covariance + converged : bool + Indicates wether the algorithm converged. + If `False` result was returned after `nIter` + iterations. ''' nFreq, nChannels = CSD.shape[:2] @@ -81,49 +60,40 @@ def wilson_sf(CSD, samplerate, nIter=100, rtol=1e-9): psi = np.tile(psi0, (nFreq, 1, 1)) assert psi.shape == CSD.shape - errs = [] + converged = False for _ in range(nIter): psi_inv = np.linalg.inv(psi) # the bracket of equation 3.1 g = psi_inv @ CSD @ psi_inv.conj().transpose(0, 2, 1) gplus, gplus_0 = _plusOperator(g + Ident) - + # the 'any' matrix S = np.triu(gplus_0) S = S - S.conj().T # S + S* = 0 - psi_old = psi # the next step psi_{tau+1} psi = psi @ (gplus + S) - - rel_err = np.abs((psi - psi_old) / np.abs(psi)) - # print(rel_err.max()) - # mean relative error + psi0 = psi0 @ (gplus_0 + S) + + # max relative error CSDfac = psi @ psi.conj().transpose(0, 2, 1) err = np.abs(CSD - CSDfac) - err = err / np.abs(CSD) # relative error - - print('Cond', np.linalg.cond(psi[0])) - print('Error:', err.max(),'\n') - - errs.append(err.max()) + err = (err / np.abs(CSD)).max() - Aks = np.fft.ifft(psi, axis=0) - A0 = Aks[0, ...] - + # converged + if err < rtol: + converged = True + break + # Noise Covariance - Sigma = A0 * A0.T - # strip off remaining imaginary parts - Sigma = np.real(Sigma) - + Sigma = psi0 @ psi0.conj().T + # Transfer function - A0inv = np.linalg.inv(A0) - Hfunc = psi @ A0inv.conj().T + psi0_inv = np.linalg.inv(psi0) + Hfunc = psi @ psi0_inv.conj().T - # print(err.mean()) - - return Hfunc, Sigma, CSDfac, errs + return Hfunc, Sigma, converged def _psi0_initial(CSD): @@ -156,7 +126,7 @@ def _psi0_initial(CSD): return psi0.T -# from scipy.signal import windows + def _plusOperator(g): ''' @@ -179,13 +149,71 @@ def _plusOperator(g): # Zero out negative lags beta[nLag + 1:, ...] = 0 - # beta = beta * windows.tukey(len(beta), alpha=0.2)[:, None, None] - gp = np.fft.fft(beta, axis=0) return gp, g0 +# --- End of Wilson's Algorithm --- + + +def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, Nsteps=50): + + ''' + Brute force regularize CSD matrix + by inspecting the maximal condition number + along the frequency axis. + Multiply with different epsilon * I, + starting with epsilon = 1e-10 until the + condition number is smaller than `cond_max`. + Raises a ValueError if the maximal regularization + factor `epx_max` was reached but `cond_max` still not met. + + + Parameters + ---------- + CSD : 3D :class:`numpy.ndarray` + The cross spectral density matrix + with shape (nFreq, nChannel, nChannel) + cond_max : float + The maximal condition number after regularization + eps_max : float + The largest regularization factor to be used. If + also this value does not regularize the CSD up + to `cond_max` a ValueError is raised. + nSteps : int + Number of steps between 1e-10 and eps_max. + + Returns + ------- + CSDreg : 3D :class:`numpy.ndarray` + The regularized CSD matrix with a maximal + condition number of `cond_max` + eps : float + The regularization factor used + + ''' + + epsilons = np.logspace(-10, np.log10(eps_max), 25) + I = np.eye(CSD.shape[1]) + + CondNum = np.linalg.cond(CSD).max() + + # nothing to be done + if CondNum < cond_max: + return CSD, 0 + + for eps in epsilons: + CSDreg = CSD + eps * I + CondNum = np.linalg.cond(CSDreg).max() + + if CondNum < cond_max: + return CSDreg, eps + + msg = f"CSD matrix not regularizable with a max epsilon of {eps_max}!" + raise ValueError(msg) + + def _mem_size(arr): ''' Gives array size in MB From 9057a2e3c8325a1faab5a2c154b0d442001ba0ed Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 19 Nov 2021 16:59:57 +0100 Subject: [PATCH 053/109] NEW: First prototype of Syncopy object arithmetic ready - both sequential and parallel processing of simple arithmetic supported - currently only addition possible (but extensions to -,/,* are straight forward) - chained operations (`data + data + data`) currently only work sequentially On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/methods/arithmetic.py modified: syncopy/tests/spy_setup.py --- syncopy/datatype/base_data.py | 4 +- syncopy/datatype/methods/arithmetic.py | 96 +++++++++++++++++++++----- syncopy/tests/spy_setup.py | 1 + 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 7992aa579..230fe20c4 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -25,7 +25,7 @@ # Local imports import syncopy as spy -from .methods.arithmetic import _add +from .methods.arithmetic import _process_operator from .methods.selectdata import selectdata from .methods.show import show from syncopy.shared.tools import StructDict @@ -739,7 +739,7 @@ def __del__(self): ignore_errors=True) def __add__(self, other): - return _add(self, other) + return _process_operator(self, other, "+") # Class "constructor" def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 13ff5ff09..e7a5a6314 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -9,19 +9,24 @@ import h5py # Local imports +from syncopy import __acme__ from syncopy.shared.parsers import data_parser -from syncopy.shared.tools import get_defaults -from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYInfo, SPYWarning +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning from syncopy.shared.computational_routine import ComputationalRoutine -from syncopy.shared.kwarg_decorators import unwrap_cfg, unwrap_io, detect_parallel_client +from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.computational_routine import ComputationalRoutine +if __acme__: + import dask.distributed as dd __all__ = [] -def _add(obj1, obj2): +def _process_operator(obj1, obj2, operator, **kwargs): + """ + Coming soon... + """ + baseObj, operand, operand_dat, opres_type, operand_idxs = _parse_input(obj1, obj2, operator) + return _perform_computation(baseObj, operand, operand_dat, operand_idxs, opres_type, operator) - operand_dat, opres_type, operand_idxs = _parse_input(obj1, obj2, "+") - operation = lambda x, y : x + y def _parse_input(obj1, obj2, operator): @@ -42,9 +47,10 @@ def _parse_input(obj1, obj2, operator): raise exc # If no active selection is present, create a "fake" all-to-all selection - # to harmonize processing down the road + # to harmonize processing down the road (and attach `_cleanup` attribute for later removal) if baseObj._selection is None: baseObj.selectdata(inplace=True) + baseObj._selection._cleanup = True baseTrialList = baseObj._selection.trials # Use the `_preview_trial` functionality of Syncopy objects to get each trial's @@ -165,10 +171,64 @@ def _parse_input(obj1, obj2, operator): lgl = "Syncopy object, scalar or array-like" raise SPYTypeError(operand, varname="operand", expected=lgl) - return operand_dat, opres_type, operand_idxs + return baseObj, operand, operand_dat, opres_type, operand_idxs + +def _perform_computation(baseObj, + operand, + operand_dat, + operand_idxs, + opres_type, + operator): + """ + Coming soon... + """ + + # Prepare logging info in dictionary: we know that `baseObj` is definitely + # a Syncopy data object, operand may or may not be; account for this + if "BaseData" in str(operand.__class__.__mro__): + opSel = operand._selection + else: + opSel = None + log_dct = {"operator": operator, + "base": baseObj.__class__.__name__, + "base selection": baseObj._selection, + "operand": operand.__class__.__name__, + "operand selection": opSel} + + # Create output object + out = baseObj.__class__(dimord=baseObj.dimord) + + # Wrap operator in lambda function + if operator == "+": + operation = lambda x, y : x + y + + if __acme__: + try: + dd.get_client() + parallel = True + except ValueError: + parallel = False + + # Perform actual computation + opMethod = SpyArithmetic(operand_dat, operand_idxs, operation=operation, + opres_type=opres_type) + opMethod.initialize(baseObj, + out._stackingDim, + chan_per_worker=None, + keeptrials=True) + opMethod.compute(baseObj, out, parallel=parallel, log_dict=log_dct) + + # Delete any created subset selections + if hasattr(baseObj._selection, "_cleanup"): + baseObj._selection = None + if opSel is not None: + operand._selection = None + + return out + @unwrap_io -def arithmetic_cF(base_dat, operand_dat, operand_idx, operation=None, opres_type = None, +def arithmetic_cF(base_dat, operand_dat, operand_idx, operation=None, opres_type=None, noCompute=False, chunkShape=None): """ Coming soon... @@ -189,17 +249,17 @@ class SpyArithmetic(ComputationalRoutine): computeFunction = staticmethod(arithmetic_cF) - def process_metadata(self, data, out): + def process_metadata(self, baseObj, out): # Get/set timing-related selection modifiers - out.trialdefinition = data._selection.trialdefinition - # if data._selection._timeShuffle: # FIXME: should be implemented done the road - # out.time = data._selection.timepoints - if data._selection._samplerate: - out.samplerate = data.samplerate + out.trialdefinition = baseObj._selection.trialdefinition + # if baseObj._selection._timeShuffle: # FIXME: should be implemented done the road + # out.time = baseObj._selection.timepoints + if baseObj._selection._samplerate: + out.samplerate = baseObj.samplerate # Get/set dimensional attributes changed by selection - for prop in data._selection._dimProps: - selection = getattr(data._selection, prop) + for prop in baseObj._selection._dimProps: + selection = getattr(baseObj._selection, prop) if selection is not None: - setattr(out, prop, getattr(data, prop)[selection]) + setattr(out, prop, getattr(baseObj, prop)[selection]) diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index cc2a275ca..1bda9289b 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -25,5 +25,6 @@ # Test stuff within here... artdata = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) + sys.exit() spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") From 242520d0a50cf6e980529f63ec77e38f32dc94c0 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 19 Nov 2021 17:08:47 +0100 Subject: [PATCH 054/109] FIX: Enable chained arithmetic ops in parallel computing mode - use a distributed lock to prevent ACME from executing chained arithmetic operations concurrently; instead, only compute trials in parallel but keep chained ops sequential On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index e7a5a6314..9f10107f0 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -216,7 +216,12 @@ def _perform_computation(baseObj, out._stackingDim, chan_per_worker=None, keeptrials=True) + if __acme__: + lock = dd.lock.Lock(name='arithmetic_ops') + lock.acquire() opMethod.compute(baseObj, out, parallel=parallel, log_dict=log_dct) + if __acme__: + lock.release() # Delete any created subset selections if hasattr(baseObj._selection, "_cleanup"): From 2d5fae796cc5828f77586218570db0378a1527c5 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 22 Nov 2021 09:45:55 +0100 Subject: [PATCH 055/109] NEW/CHG: First draft of Syncopy arithmetic is ready - built mechanics for the (component-wise) operations +, -, /, * and **; currently, in-place arithmetics are not supported yet - only acquire/release distributed lock if computation is actually performed concurrently (merely having ACME present is not enough) On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/methods/arithmetic.py modified: syncopy/tests/spy_setup.py --- syncopy/datatype/base_data.py | 13 +++++++++++++ syncopy/datatype/methods/arithmetic.py | 24 +++++++++++++++++++----- syncopy/tests/spy_setup.py | 1 + 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 230fe20c4..6f1669106 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -738,9 +738,22 @@ def __del__(self): shutil.rmtree(os.path.splitext(self.filename)[0], ignore_errors=True) + # Support for basic arithmetic operations (no in-place computations supported yet) def __add__(self, other): return _process_operator(self, other, "+") + def __sub__(self, other): + return _process_operator(self, other, "-") + + def __mul__(self, other): + return _process_operator(self, other, "*") + + def __truediv__(self, other): + return _process_operator(self, other, "/") + + def __pow__(self, other): + return _process_operator(self, other, "**") + # Class "constructor" def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): """ diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 9f10107f0..951d8feaa 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -20,7 +20,7 @@ __all__ = [] -def _process_operator(obj1, obj2, operator, **kwargs): +def _process_operator(obj1, obj2, operator): """ Coming soon... """ @@ -198,10 +198,22 @@ def _perform_computation(baseObj, # Create output object out = baseObj.__class__(dimord=baseObj.dimord) - # Wrap operator in lambda function + # Now create actual functional operations: wrap operator in lambda if operator == "+": operation = lambda x, y : x + y + elif operator == "-": + operation = lambda x, y : x - y + elif operator == "*": + operation = lambda x, y : x * y + elif operator == "/": + operation = lambda x, y : x / y + elif operator == "**": + operation = lambda x, y : x ** y + else: + raise SPYValueError("supported arithmetic operator", actual=operator) + # If ACME is available, try to attach (already running) parallel computing client + parallel = False if __acme__: try: dd.get_client() @@ -209,18 +221,20 @@ def _perform_computation(baseObj, except ValueError: parallel = False - # Perform actual computation + # Perform actual computation: in case of parallel execution, use a distributed + # lock to prevent ACME from performing chained operations (`x + y + 3``) + # simultaneously (thereby wrecking the underlying HDF5 datasets) opMethod = SpyArithmetic(operand_dat, operand_idxs, operation=operation, opres_type=opres_type) opMethod.initialize(baseObj, out._stackingDim, chan_per_worker=None, keeptrials=True) - if __acme__: + if parallel: lock = dd.lock.Lock(name='arithmetic_ops') lock.acquire() opMethod.compute(baseObj, out, parallel=parallel, log_dict=log_dct) - if __acme__: + if parallel: lock.release() # Delete any created subset selections diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 1bda9289b..3dbb7c3cb 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -25,6 +25,7 @@ # Test stuff within here... artdata = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) + sys.exit() spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") From 626032875f1280cda551f6a8bc28835718f2a61a Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 22 Nov 2021 10:37:33 +0100 Subject: [PATCH 056/109] WIP : Cleanup unneeded dev files Changes to be committed: deleted: dev_csd.py modified: dev_frontend.py deleted: testdev_backend.py deleted: testdev_frontend.py --- dev_csd.py | 135 ----------------------------- dev_frontend.py | 42 ++++----- testdev_backend.py | 205 -------------------------------------------- testdev_frontend.py | 32 ------- 4 files changed, 22 insertions(+), 392 deletions(-) delete mode 100644 dev_csd.py delete mode 100644 testdev_backend.py delete mode 100644 testdev_frontend.py diff --git a/dev_csd.py b/dev_csd.py deleted file mode 100644 index 2985e1f20..000000000 --- a/dev_csd.py +++ /dev/null @@ -1,135 +0,0 @@ -import numpy as np -from scipy.signal import csd as sci_csd -import scipy.signal as sci -from syncopy.connectivity.ST_compRoutines import cross_spectra_cF -from syncopy.connectivity import wilson_sf -from syncopy.connectivity.ST_compRoutines import cross_covariance_cF -from syncopy.connectivity.AV_compRoutines import normalize_ccov_cF -import matplotlib.pyplot as ppl - - -# white noise ensemble -nSamples = 1001 -fs = 1000 -tvec = np.arange(nSamples) / fs -omegas = np.array([21, 42, 59, 78]) * 2 * np.pi -omegas = np.arange(10, 40) * 2 * np.pi -data = np.c_[[1*np.cos(om * tvec) for om in omegas]].T - -nChannels = 5 -data1 = np.random.randn(nSamples, nChannels) - -x1 = data1[:, 0] -y1 = data1[:, 1] - - -def dev_cc(nSamples=1001): - - nSamples = 1001 - fs = 1000 - tvec = np.arange(nSamples) / fs - - sig1 = np.cos(2 * np.pi * 30 * tvec) - sig2 = np.sin(2 * np.pi * 30 * tvec) - - mode = 'same' - t_half = nSamples // 2 - - r = sci.correlate(sig2, sig2, mode=mode, method='fft') - r2 = sci.fftconvolve(sig2, sig2[::-1], mode=mode) - assert np.all(r == r2) - - - lags = np.arange(-nSamples // 2, nSamples // 2) - if nSamples % 2 != 0: - lags = lags + 1 - lags = lags * 1 / fs - - if nSamples % 2 == 0: - half_lags = np.arange(0, nSamples // 2) - else: - half_lags = np.arange(0, nSamples // 2 + 1) - half_lags = half_lags * 1 / fs - - ppl.figure(1) - ppl.xlabel('lag (s)') - ppl.ylabel('convolution result') - - ppl.plot(lags, r2, lw = 1.5) - # ppl.xlim((490,600)) - - - norm = np.arange(nSamples, t_half, step = -1) / 2 - # norm = np.r_[norm, norm[::-1]] - - ppl.figure(2) - ppl.xlabel('lag (s)') - ppl.ylabel('correlation') - ppl.plot(half_lags, r2[nSamples // 2:] / norm, lw = 1.5) - - -def sci_est(x, y, nper, norm=False): - freqs1, csd1 = sci_csd(x, y, fs, window='bartlett', nperseg=nper) - freqs2, csd2 = sci_csd(x, y, fs, window='bartlett', nperseg=nSamples) - - if norm: - # WIP.. - auto1 = sci_csd(x, x, fs, window='bartlett', nperseg=nper) - auto1 *= sci_csd(y, y, fs, window='bartlett', nperseg=nper) - - auto2 = sci_csd(x, y, fs, window='bartlett', nperseg=nSamples) - auto2 *= sci_csd(y, y, fs, window='bartlett', nperseg=nSamples) - - csd1 = csd1 / np.sqrt(auto1 * auto2) - - return (freqs1, np.abs(csd1)), (freqs2, np.abs(csd2)) - - -# omegas = np.arange(30, 50, step=1) * 2 * np.pi -# data = np.array([np.cos(om * tvec) for om in omegas]).T -# dataR = 1 * (data + np.random.randn(*data.shape) * 1) -# CS, freqs = cross_spectra_cF(dataR, fs, taper='dpss', -# taper_opt={'Kmax' : 15, 'NW' : 6}, -# norm=True, fullOutput=True) - -# CS2, freqs = cross_spectra_cF(dataR, fs, taper='dpss', taperopt={'Kmax' : 15, 'NW' : 6}, norm=False) - -# noisy phase evolution -def phase_evo(omega0, eps, fs=1000, N=1000): - wn = np.random.randn(N) * 1 / fs - delta_ts = np.ones(N) * 1 / fs - phase = np.cumsum(omega0 * delta_ts + eps * wn) - return phase - - -eps = 0.0003 * fs -omega = 50 * 2 * np.pi -p1 = phase_evo(omega, eps, N=nSamples) -p2 = phase_evo(omega, eps, N=nSamples) -s1 = 3 * np.cos(p1) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) -s2 = 20 * np.cos(p2) + np.cos(2 * omega * tvec) + .5 * np.random.randn(nSamples) -data = np.c_[s1, s2] - -bw = 10 -NW = bw * nSamples / (2 * fs) -Kmax = int(2 * NW - 1) # optimal number of tapers - -CS, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, - norm=True, fullOutput=True) -CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, - norm=False, fullOutput=True) - - -# fig, ax = ppl.subplots(figsize=(6,4), num=1) -# ax.set_xlabel('frequency (Hz)') -# ax.set_ylabel('$|CSD(f)|$') -# ax.set_ylim((-.02,1.25)) - -# ax.plot(freqs, np.abs(CS2[0,:, 0, 0]), label = '$CSD_{00}$', lw = 2, alpha = 0.7) -# ax.plot(freqs, np.abs(CS2[0,:, 1, 1]), label = '$CSD_{11}$', lw = 2, alpha = 0.7) -# ax.plot(freqs, np.abs(CS2[0,:, 0, 1]), label = '$CSD_{01}$', lw = 2, alpha = 0.7) - -# ax.legend() - -CCov, lags = cross_covariance_cF(data, fs, norm=False, fullOutput=True) -Corr = normalize_ccov_cF(CCov) diff --git a/dev_frontend.py b/dev_frontend.py index 20191ad88..4ab2ea89a 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -1,7 +1,7 @@ import numpy as np import scipy.signal as sci -from syncopy.datatype import CrossSpectralData, padding, SpectralData +from syncopy.datatype import CrossSpectralData, padding, SpectralData, AnalogData from syncopy.connectivity.ST_compRoutines import cross_spectra_cF, ST_CrossSpectra from syncopy.connectivity.ST_compRoutines import cross_covariance_cF from syncopy.connectivity import connectivityanalysis @@ -21,29 +21,31 @@ # no problems here.. coherence = connectivityanalysis(data=tdat, - foilim=None, - output='pow') + foilim=None, + output='pow') + +# D = SpectralData(dimord=['freq','test1','test2','taper']) +# D2 = AnalogData(dimord=['freq','test1']) # a lot of problems here.. # correlation = connectivityanalysis(data=tdat, method='corr', keeptrials=False) - # the hard wired dimord of the cF dimord = ['None', 'freq', 'channel_i', 'channel_j'] -# res = freqanalysis(data=tdat, -# method='mtmfft', -# samplerate=tdat.samplerate, -# # order_max=20, -# # foilim=foilim, -# # foi=np.arange(502), -# output='pow', -# # polyremoval=1, -# t_ftimwin=0.5, -# keeptrials=False, -# taper='dpss', -# nTaper = 11, -# tapsmofrq=5, -# keeptapers=True, -# parallel=False, # try this!!!!!! -# select={"trials" : [0,1]}) +res = freqanalysis(data=tdat, + method='mtmfft', + samplerate=tdat.samplerate, +# order_max=20, +# foilim=foilim, +# foi=np.arange(502), + output='pow', +# polyremoval=1, + t_ftimwin=0.5, + keeptrials=True, + taper='dpss', + nTaper = 11, + tapsmofrq=5, + keeptapers=True, + parallel=False, # try this!!!!!! + select={"trials" : [0,1]}) diff --git a/testdev_backend.py b/testdev_backend.py deleted file mode 100644 index f202ade61..000000000 --- a/testdev_backend.py +++ /dev/null @@ -1,205 +0,0 @@ -''' This is a temporary development file ''' - -import numpy as np -import matplotlib.pyplot as ppl - -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser -from syncopy.specest.wavelet import get_optimal_wavelet_scales, wavelet -from syncopy.specest.superlet import SuperletTransform, MorletSL, cwtSL, _get_superlet_support, superlet, compute_adaptive_order, scale_from_period -from syncopy.specest.wavelets import Morlet -from scipy.signal import fftconvolve - - -def gen_superlet_testdata(freqs=[20, 40, 60], - cycles=11, fs=1000, - eps = 0): - - ''' - Harmonic superposition of multiple - few-cycle oscillations akin to the - example of Figure 3 in Moca et al. 2021 NatComm - ''' - - signal = [] - for freq in freqs: - - # 10 cycles of f1 - tvec = np.arange(cycles / freq, step=1 / fs) - - harmonic = np.cos(2 * np.pi * freq * tvec) - f_neighbor = np.cos(2 * np.pi * (freq + 10) * tvec) - packet = harmonic + f_neighbor - - # 2 cycles time neighbor - delta_t = np.zeros(int(2 / freq * fs)) - - # 5 cycles break - pad = np.zeros(int(5 / freq * fs)) - - signal.extend([pad, packet, delta_t, harmonic]) - - # stack the packets together with some padding - signal.append(pad) - signal = np.concatenate(signal) - - # additive white noise - if eps > 0: - signal = np.random.randn(len(signal)) * eps + signal - - return signal - - -# test the Wavelet transform -fs = 1000 -s1 = 1 * gen_superlet_testdata(fs=fs, eps=0) # 20Hz, 40Hz and 60Hz -data = np.c_[3*s1, 50*s1] -preselect = np.ones(len(s1), dtype=bool) -preselect2 = np.ones((len(s1), 2), dtype=bool) -pads = 0 - -ts = np.arange(-50,50) -morletTC = Morlet() -morletSL = MorletSL(c_i=30) - -# frequencies to look at, 10th freq is around 20Hz -freqs = np.linspace(1, 100, 50) # up to 100Hz -scalesTC = morletTC.scale_from_period(1 / freqs) -# scales are cycle independent! -scalesSL = scale_from_period(1 / freqs) - -# automatic diadic scales -ssTC = get_optimal_wavelet_scales(Morlet().scale_from_period, len(s1), 1/fs) -ssSL = get_optimal_wavelet_scales(scale_from_period, len(s1), 1/fs) -# a multiplicative Superlet - a set of Morlets, order 1 - 30 -c_1 = 1 -cycles = c_1 * np.arange(1, 31) -sl = [MorletSL(c) for c in cycles] - -res = wavelet(data, - preselect, - preselect, - pads, - pads, - samplerate=fs, - # toi='some', - output_fmt="pow", - scales=scalesTC, - wav=Morlet(), - noCompute=False) - - -# unit impulse -# data = np.zeros(500) -# data[248:252] = 1 -spec = superlet(s1, samplerate=fs, scales=scalesSL, - order_max=10, - order_min=5, - adaptive=False) -spec2 = superlet(data, samplerate=fs, scales=scalesSL, order_max=20, adaptive=False) - -# nc = superlet(data, samplerate=fs, scales=scalesSL, order_max=30) - - -def do_slt(data, scales=scalesSL, **slkwargs): - - if scales is None: - scales = get_optimal_wavelet_scales(scale_from_period, - len(data[:, 0]), - 1 / fs) - - spec = superlet(data, samplerate=fs, - scales=scales, - **slkwargs) - - print(spec.max(),spec.shape) - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - ppl.imshow(np.abs(spec[...,0]), cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return spec - - -def show_MorletSL(morletSL, scale): - - cycle = morletSL.c_i - ts = _get_superlet_support(scale, 1/fs, cycle) - ppl.plot(ts, MorletSL(cycle)(ts, scale)) - - -def show_MorletTC(morletTC, scale): - - M = 10 * scale * fs - # times to use, centred at zero - ts = np.arange((-M + 1) / 2.0, (M + 1) / 2.0) / fs - ppl.plot(ts, morletTC(ts, scale)) - - -def do_superlet_cwt(data, wav, scales=None): - - if scales is None: - scales = get_optimal_wavelet_scales(scale_from_period, len(data[:,0]), 1/fs) - - res = cwtSL(data, - wav, - scales=scales, - dt=1 / fs) - - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - channel=0 - ppl.imshow(np.abs(res[:,:, channel]), cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return res.T - - -def do_normal_cwt(data, wav, scales=None): - - if scales is None: - scales = get_optimal_wavelet_scales(wav.scale_from_period, - len(data[:,0]), - 1/fs) - res = wavelet(data, - preselect, - preselect, - pads, - pads, - samplerate=fs, - # toi='some', - output_fmt="pow", - scales=scales, - wav=wav) - - ppl.figure() - extent = [0, len(s1) / fs, freqs[-1], freqs[0]] - ppl.imshow(res[:, 0, :, 0].T, cmap='plasma', aspect='auto', extent=extent) - ppl.plot([0, len(s1) / fs], [20, 20], 'k--') - ppl.plot([0, len(s1) / fs], [40, 40], 'k--') - ppl.plot([0, len(s1) / fs], [60, 60], 'k--') - - return res[:, 0, :, :].T - -# do_cwt(morletSL) - - -def screen_CWT(w0s= [5, 8, 12]): - for w0 in w0s: - morletTC = Morlet(w0) - scales = _get_optimal_wavelet_scales(morletTC, len(s1), 1/fs) - res = wavelet(s1[:, np.newaxis], - preselect, - preselect, - pads, - pads, - samplerate=fs, - toi=np.array([1,2]), - scales=scales, - wav=morletTC) - - ppl.figure() - ppl.imshow(res[:, 0, :, 0].T, cmap='plasma', aspect='auto') diff --git a/testdev_frontend.py b/testdev_frontend.py deleted file mode 100644 index 8e7df5563..000000000 --- a/testdev_frontend.py +++ /dev/null @@ -1,32 +0,0 @@ -''' This is a temporary development file ''' - -import numpy as np -import matplotlib.pyplot as ppl -from syncopy.specest import freqanalysis -from syncopy.tests.misc import generate_artificial_data - -tdat = generate_artificial_data() - -# test mtmfft analysis -r_mtm = freqanalysis(tdat) - -toi_ival = np.linspace(-0.5, 1, 100) - -#toi_ival = [0,0.2,0.5,1] -toi_ival = 'all' -foi = np.logspace(-1, 2.6, 25) -# test classical wavelet analysis -r_wav = freqanalysis(tdat, method="wavelet", - toi=toi_ival, - output='abs', - foi=None) #, foilim=[5, 500]) - -# test superlet analysis -r_sup = freqanalysis(tdat, method="superlet", toi=toi_ival, - order_max=20, output='abs', - order_min=1, - c_1 = 5, - adaptive=True) - -r_sup = freqanalysis(tdat, method="superlet", toi='all', order_max=30, foi=foi, output='abs',order_min=5, adaptive=True) -#res_strials = [t for t in r_sup.trials] From b586eafc180462ad426478842a3be8e10545a5b8 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 22 Nov 2021 11:21:39 +0100 Subject: [PATCH 057/109] NEW: Posthoc addition to support right-acting operators - add support for things like `1 + data` etc. Note: don't allow the use of Syncopy objects as exponents (e.g., `2 ** data`...) On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/base_data.py --- syncopy/datatype/base_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 6f1669106..1ed2e5fcd 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -742,15 +742,27 @@ def __del__(self): def __add__(self, other): return _process_operator(self, other, "+") + def __radd__(self, other): + return _process_operator(self, other, "+") + def __sub__(self, other): return _process_operator(self, other, "-") + def __rsub__(self, other): + return _process_operator(self, other, "-") + def __mul__(self, other): return _process_operator(self, other, "*") + def __rmul__(self, other): + return _process_operator(self, other, "*") + def __truediv__(self, other): return _process_operator(self, other, "/") + def __rtruediv__(self, other): + return _process_operator(self, other, "/") + def __pow__(self, other): return _process_operator(self, other, "**") From 27ccb832513b252413d7d5475407f4b4e12b8663 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 22 Nov 2021 12:39:07 +0100 Subject: [PATCH 058/109] NEW : Wilson's spectral matrix factorization backend test - test for convergence within error tolerance - test also the regularization Changes to be committed: modified: syncopy/connectivity/wilson_sf.py modified: syncopy/tests/backend/test_connectivity.py --- syncopy/connectivity/wilson_sf.py | 2 +- syncopy/tests/backend/test_connectivity.py | 84 ++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index 3d2e1aef2..feabb138f 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -157,7 +157,7 @@ def _plusOperator(g): # --- End of Wilson's Algorithm --- -def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, Nsteps=50): +def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): ''' Brute force regularize CSD matrix diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index f425c160e..2c5f30b5c 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as ppl from syncopy.connectivity import ST_compRoutines as stCR from syncopy.connectivity import AV_compRoutines as avCR +from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd def test_coherence(): @@ -162,3 +163,86 @@ def test_cross_cov(): # cross-correlation (normalized cross-covariance) between # cosine and sine analytically equals minus sine assert np.all(CC[:, 0, 0, 1] + sine[:nLags] < 1e-5) + + +def test_wilson(): + + ''' + Test Wilson's spectral matrix factorization. + + As the routine has relative error-checking + inbuild, we just need to check for convergence. + ''' + + # --- create test data --- + fs = 1000 + nChannels = 10 + nSamples = 1000 + f1, f2 = [30 , 40] # 30Hz and 60Hz + data = np.zeros((nSamples, nChannels)) + for i in range(nChannels): + # more phase diffusion in the 60Hz band + p1 = phase_evo(f1 * 2 * np.pi, eps=0.1, fs=fs, N=nSamples) + p2 = phase_evo(f2 * 2 * np.pi, eps=0.35, fs=fs, N=nSamples) + + data[:, i] = np.cos(p1) + 2 * np.sin(p2) + .5 * np.random.randn(nSamples) + + # --- get the (single trial) CSD --- + + bw = 5 # 5Hz smoothing + NW = bw * nSamples / (2 * fs) + Kmax = int(2 * NW - 1) # optimal number of tapers + + CSD, freqs = stCR.cross_spectra_cF(data, fs, + taper='dpss', + taper_opt={'Kmax' : Kmax, 'NW' : NW}, + norm=False, + fullOutput=True) + # strip off singleton time axis + CSD = CSD[0] + + # get CSD condition number, which is way too large! + CN = np.linalg.cond(CSD).max() + assert CN > 1e6 + + # --- regularize CSD --- + + CSDreg, fac = regularize_csd(CSD, cond_max=1e6, nSteps=25) + CNreg = np.linalg.cond(CSDreg).max() + assert CNreg < 1e6 + # check that 'small' regularization factor is enough + assert fac < 1e-5 + + # --- factorize CSD with Wilson's algorithm --- + + H, Sigma, conv = wilson_sf(CSDreg, rtol=1e-9) + + # converged - \Psi \Psi^* \approx CSD, + # with relative error <= rtol? + assert conv + + # reconstitute + CSDfac = H @ Sigma @ H.conj().transpose(0, 2, 1) + + fig, ax = ppl.subplots(figsize=(6, 4)) + ax.set_xlabel('frequency (Hz)') + ax.set_ylabel(r'$|CSD_{ij}(f)|$') + chan = nChannels // 2 + # show (real) auto-spectra + assert ax.plot(freqs, np.abs(CSD[:, chan, chan]), + '-o', label='original CSD', ms=3) + assert ax.plot(freqs, np.abs(CSDreg[:, chan, chan]), + '-o', label='regularized CSD', ms=3) + assert ax.plot(freqs, np.abs(CSDfac[:, chan, chan]), + '-o', label='factorized CSD', ms=3) + ax.set_xlim((f1 - 5, f2 + 5)) + + +# --- Helper routines --- + +# noisy phase evolution -> phase diffusion +def phase_evo(omega0, eps, fs=1000, N=1000): + wn = np.random.randn(N) + delta_ts = np.ones(N) * 1 / fs + phase = np.cumsum(omega0 * delta_ts + eps * wn) + return phase From 298e3c7199a9023f018c1075b70ac3015fa34c6f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 22 Nov 2021 17:47:37 +0100 Subject: [PATCH 059/109] WIP : Granger-Causality backend - could reproduce AR(2) example of Dhamala 2008 Changes to be committed: modified: ../../dev_granger.py new file: granger_causality.py --- dev_granger.py | 55 +++++++++++++++- syncopy/connectivity/granger_causality.py | 79 +++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 syncopy/connectivity/granger_causality.py diff --git a/dev_granger.py b/dev_granger.py index 3c1e1a24c..77c1533de 100644 --- a/dev_granger.py +++ b/dev_granger.py @@ -48,7 +48,6 @@ def make_test_data(nChannels=10, nSamples=1000, bw=5): data[:, i] = np.cos(p1) + np.sin(p2) + .5 * np.random.randn(nSamples) # data[:, i] = brown_noise(nSamples) - bw = 5 NW = bw * nSamples / (2 * fs) Kmax = int(2 * NW - 1) # optimal number of tapers @@ -58,9 +57,14 @@ def make_test_data(nChannels=10, nSamples=1000, bw=5): return CSD - + def cond_samples(Ns, nChannels=2): + ''' + Screens condition number for CSDs + with different channel Numbers + ''' + cns = [] for N in Ns: @@ -73,4 +77,51 @@ def cond_samples(Ns, nChannels=2): ax.plot(Ns, cns, '-o', label=f'nChannels={nChannels}') ax.set_ylim((-1, 5000)) + +def make_AR2_csd(nSamples=1000, coupling=0.2, fs=200, nTrials=10): + + # both processes have same parameters + alpha1, alpha2 = 0.55, -0.8 + + CSDav = np.zeros((nSamples // 2 + 1, 2, 2), dtype=np.complex64) + for _ in range(nTrials): + sol = np.zeros((nSamples, 2)) + + # pick the 1st values at random + xs_ini = np.random.randn(2,2) + + sol[:2,:] = xs_ini + + for i in range(1, nSamples): + sol[i, 1] = alpha1 * sol[i - 1, 1] + alpha2 * sol[i - 2, 1] + sol[i, 1] += np.random.randn() + + # X2 drives X1 + sol[i, 0] = alpha1 * sol[i - 1, 0] + alpha2 * sol[i - 2, 0] + sol[i, 0] += sol[i - 1, 1] * coupling + sol[i, 0] += np.random.randn() + + # --- get CSD --- + bw = 5 + NW = bw * nSamples / (2 * 1000) + Kmax = int(2 * NW - 1) # optimal number of tapers + CS2, freqs = cross_spectra_cF(sol, fs, + taper='dpss', + taper_opt={'Kmax' : Kmax, 'NW' : NW}, + norm=False, + fullOutput=True) + + CSD = CS2[0, ...] + CSDav += CSD + + print(Kmax) + CSDav /= nTrials + return CSDav, freqs, sol + + +# test run +# CSDav, freqs, data = make_AR2_csd(nSamples=2500, nTrials=250) +# H, Sigma, conv = wilson_sf(CSDav, nIter=20) +# G = granger(CSDav, H, Sigma) + diff --git a/syncopy/connectivity/granger_causality.py b/syncopy/connectivity/granger_causality.py new file mode 100644 index 000000000..b6675aeb8 --- /dev/null +++ b/syncopy/connectivity/granger_causality.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# +# Implementation of Granger-Geweke causality +# +# +# Builtin/3rd party package imports +import numpy as np + + +def granger(CSD, Hfunc, Sigma): + + ''' + Computes the pairwise Granger causalities + for all (non-symmetric!) channel combinations + accoeding to equation 8 in [1]_. + + The transfer functions `Hfunc` and noise covariance + `Sigma` are expected to have been already computed. + + Parameters + ---------- + CSD : (nFreq, N, N) :class:`numpy.ndarray` + Complex cross spectra for all channel combinations i,j. + `N` corresponds to number of input channels. + Hfunc : (nFreq, N, N) :class:`numpy.ndarray` + Spectral transfer functions for all channel combinations i,j. + Sigma : (N, N) :class:`numpy.ndarray` + The noise covariances, should be multiplied by the samplerate + beforehand. + + See also + -------- + wilson_sf : :func:`~syncopy.connectivity.wilson_sf.wilson_sf + Spectral matrix factorization that yields the + transfer functions and noise covariances + from a cross spectral density. + + Notes + ----- + .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms + of time series data." Physical review letters 100.1 (2008): 018701. + + ''' + + nChannels = CSD.shape[1] + auto_spectra = CSD.transpose(1, 2, 0).diagonal() + auto_spectra = np.abs(auto_spectra) # auto-spectra are real + + # we need the stacked auto-spectra of the form (nChannel=3): + # S_11 S_22 S_33 + # Smat(f) = S_11 S_22 S_33 + # S_11 S_22 S_33 + Smat = auto_spectra[:, None, :] * np.ones(nChannels)[:, None] + assert CSD.shape == Smat.shape + + # Granger i->j needs H_ji entry + Hmat = np.abs(Hfunc.transpose(0, 2, 1))**2 + # Granger i->j needs Sigma_ji entry + SigmaIJ = np.abs(Sigma.T)**2 + + # imag part should be 0 + auto_cov = np.abs(Sigma.diagonal()) + # same stacking as for the auto spectra (without freq axis) + SigmaII = auto_cov[:, None] * np.ones(nChannels)[:, None] + + # the denominator + denom = SigmaII.transpose() - SigmaIJ / SigmaII + denom = Smat - denom * Hmat + + # linear causality i -> j + Granger = np.log(Smat / denom) + + return Granger + + + + + From 9547d90f5601b75c1ac7066d063fbb529893fba6 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 22 Nov 2021 18:00:12 +0100 Subject: [PATCH 060/109] FIX: First round of testing bugfixes - be more restrictive w/existing selections in `operand`: don't allow Syncopy objects containing unordered selections or repetitions, e.g., `data1` + `data2` with `data2` containing a selection for trials `[2, 0, 0, 1]`. The "base" object is still allowed to have these selections, but any incoming operand must behave within reason On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 951d8feaa..650f993fa 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -150,7 +150,7 @@ def _parse_input(obj1, obj2, operator): opres_type = np.result_type(*(trl.dtype for trl in baseTrials), *(trl.dtype for trl in opndTrials)) - # Finally, ensure shapes align + # Ensure shapes align if not all(baseTrials[k].shape == opndTrials[k].shape for k in range(len(baseTrials))): lgl = "Syncopy object (selection) of compatible shapes {}" act = "Syncopy object (selection) with shapes {}" @@ -159,6 +159,15 @@ def _parse_input(obj1, obj2, operator): raise SPYValueError(lgl.format(baseShapes), varname="operand", actual=act.format(opndShapes)) + # Avoid things becoming too nasty: if operand contains wild selections + # (unordered lists or index repetitions), abort + for trl in opndTrials: + if any(np.diff(sel).min() <= 0 if isinstance(sel, list) and len(sel) > 1 \ + else False for sel in trl.idx): + lgl = "Syncopy object with ordered unreverberated subset selection" + act = "Syncopy object with selection {}" + raise SPYValueError(lgl, varname="operand", actual=act.format(operand._selection)) + # Propagate indices for fetching data from operand operand_idxs = [trl.idx for trl in opndTrials] From c4ebe1684da146e4ce254f8a8d4a78da39d23fe2 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 22 Nov 2021 18:02:27 +0100 Subject: [PATCH 061/109] WIP: Initial implementation of AnalogData arithmetic tests - first round of tests, bugs still present though... On branch spy_arithmetic Changes to be committed: modified: syncopy/tests/spy_setup.py modified: syncopy/tests/test_continuousdata.py --- syncopy/tests/spy_setup.py | 6 +- syncopy/tests/test_continuousdata.py | 135 ++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 5 deletions(-) diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 3dbb7c3cb..774beea15 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -23,8 +23,10 @@ if __name__ == "__main__": # Test stuff within here... - artdata = generate_artificial_data(nTrials=5, nChannels=16, - equidistant=False, inmemory=False) + data1 = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) + data2 = generate_artificial_data(nTrials=5, nChannels=16, equidistant=True, inmemory=False) + # client = spy.esi_cluster_setup(interactive=False) + # data1 + data2 sys.exit() spec = spy.freqanalysis(artdata, method="mtmfft", taper="dpss", output="pow") diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 6d1bf4db0..ce95ac1d1 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -27,6 +27,12 @@ skip_without_acme = pytest.mark.skipif( not __acme__, reason="acme not available") +arithmetics = [lambda x, y : x + y, + lambda x, y : x - y, + lambda x, y : x * y, + lambda x, y : x / y, + lambda x, y : x ** y] + class TestAnalogData(): @@ -399,14 +405,14 @@ def test_dataselection(self): ] chanSelections = [ ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered - [4, 2, 2, 5, 5], # repetition + unorderd + [4, 2, 2, 5, 5], # repetition + unordered range(5, 8), # narrow range slice(-2, None) # negative-start slice ] toiSelections = [ "all", # non-type-conform string [0.6], # single inexact match - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions + [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetitions ] toilimSelections = [ [0.5, 1.5], # regular range @@ -445,6 +451,128 @@ def test_dataselection(self): assert np.array_equal(cfg.out.data, selected.data) time.sleep(0.05) + # test arithmetic operations + def test_ang_arithmetic(self): + dummy = AnalogData(data=self.data, + trialdefinition=self.trl, + samplerate=self.samplerate) + ymmud = AnalogData(data=self.data.T, + trialdefinition=self.trl, + samplerate=self.samplerate, + dimord=AnalogData._defaultDimord[::-1]) + dummy2 = AnalogData(data=self.data, + trialdefinition=self.trl, + samplerate=self.samplerate) + ymmud2 = AnalogData(data=self.data.T, + trialdefinition=self.trl, + samplerate=self.samplerate, + dimord=AnalogData._defaultDimord[::-1]) + dummyArr = 2 * np.ones((dummy.trials[0].shape)) + ymmudArr = 2 * np.ones((ymmud.trials[0].shape)) + scalarOperands = [2, np.pi] + dummyOperands = [dummyArr, dummyArr.tolist()] + ymmudOperands = [ymmudArr, ymmudArr.tolist()] + + trialSelections = [ + "all", # enforce below selections in all trials of `dummy` + [3, 1] # minimally unordered + ] + chanSelections = [ + ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered + [4, 2, 2, 5, 5], # repetition + unorderd + range(5, 8), # narrow range + slice(-2, None) # negative-start slice + ] + toiSelections = [ + "all", # non-type-conform string + [0.6], # single inexact match + [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions + ] + toilimSelections = [ + [0.5, 1.5], # regular range + [1.5, 2.0], # minimal range (just two-time points) + [1.0, np.inf] # unbounded from above + ] + timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) + + + for operation in arithmetics: + for operand in scalarOperands: + result = operation(dummy, operand) # perform operation from right + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + result2 = operation(operand, dummy) # perform operation from left + assert np.array_equal(result2.data, result.data) + + # same, but swapped `dimord` + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + result2 = operation(operand, ymmud) + assert np.array_equal(result2.data, result.data) + + # Careful: NumPy tries to avoid failure by broadcasting; instead of relying + # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., + # ``np.ones((3,3)) + data`` performs ``1 + data`` nine times, so don't + # test for left/right arithmetics... + for operand in dummyOperands: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + for operand in ymmudOperands: + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + + result = operation(dummy, dummy2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) + + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) + + + for trialSel in trialSelections: + for chanSel in chanSelections: + for timeSel in timeSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["channels"] = chanSel + kwdict[timeSel[0]] = timeSel[1] + + # perform in-place selection and construct array based on new subset + selected = dummy.selectdata(**kwdict) + dummy.selectdata(inplace=True, **kwdict) + arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) + for operand in [np.pi, arr]: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], operand)) + + + dummy2.selectdata(inplace=True, **kwdict) + try: + result = operation(dummy, dummy2) + cleanSelection = True + except SPYValueError: + cleanSelection = False + except: + import pdb; pdb.set_trace() + if cleanSelection: + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], selected.trials[tk])) + + selected = ymmud.selectdata(**kwdict) + ymmud.selectdata(inplace=True, **kwdict) + ymmud2.selectdata(inplace=True, **kwdict) + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], selected.trials[tk])) + + + @skip_without_acme def test_parallel(self, testcluster): # repeat selected test w/parallel processing engine @@ -452,7 +580,8 @@ def test_parallel(self, testcluster): par_tests = ["test_relative_array_padding", "test_absolute_nextpow2_array_padding", "test_object_padding", - "test_dataselection"] + "test_dataselection", + "test_ang_arithmetic"] for test in par_tests: getattr(self, test)() flush_local_cluster(testcluster) From ff6ccf16adfa36dbd084e11778c89fd0fcded6a1 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 09:33:56 +0100 Subject: [PATCH 062/109] FIX: Second subtle bug fix: type checking in ComputationalRoutine - be explicit: do not just check for a cF's positional args to be `Sized` child but specifically ask for a tuple, list or ndarray (dicts have a `len` too...) On branch spy_arithmetic Changes to be committed: modified: syncopy/shared/computational_routine.py --- syncopy/shared/computational_routine.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/syncopy/shared/computational_routine.py b/syncopy/shared/computational_routine.py index 663d4200c..6d66f1854 100644 --- a/syncopy/shared/computational_routine.py +++ b/syncopy/shared/computational_routine.py @@ -8,13 +8,10 @@ import sys import psutil import h5py -import time import numpy as np from itertools import chain from abc import ABC, abstractmethod -from collections.abc import Sized from copy import copy -from glob import glob from numpy.lib.format import open_memmap from tqdm.auto import tqdm if sys.platform == "win32": @@ -284,7 +281,7 @@ def initialize(self, data, out_stackingdim, chan_per_worker=None, keeptrials=Tru trials = [] for tk, trialno in enumerate(self.trialList): trial = data._preview_trial(trialno) - trlArg = tuple(arg[tk] if isinstance(arg, Sized) and len(arg) == self.numTrials \ + trlArg = tuple(arg[tk] if isinstance(arg, (list, tuple, np.ndarray)) and len(arg) == self.numTrials \ else arg for arg in self.argv) chunkShape, dtype = self.computeFunction(trial, *trlArg, @@ -306,7 +303,6 @@ def initialize(self, data, out_stackingdim, chan_per_worker=None, keeptrials=Tru chk_arr = np.array(chk_list) chunkShape = tuple(chk_arr.max(axis=0)) if np.unique(chk_arr[:, stackingDim]).size > 1 and not self.keeptrials: - import pdb; pdb.set_trace() err = "Averaging trials of unequal lengths in output currently not supported!" raise NotImplementedError(err) if np.any([dtp_list[0] != dtp for dtp in dtp_list]): @@ -337,7 +333,7 @@ def initialize(self, data, out_stackingdim, chan_per_worker=None, keeptrials=Tru # Allocate control variables trial = trials[0] - trlArg0 = tuple(arg[0] if isinstance(arg, Sized) and len(arg) == self.numTrials \ + trlArg0 = tuple(arg[0] if isinstance(arg, (list, tuple, np.ndarray)) and len(arg) == self.numTrials \ else arg for arg in self.argv) chunkShape0 = chk_arr[0, :] lyt = [slice(0, stop) for stop in chunkShape0] @@ -400,7 +396,7 @@ def initialize(self, data, out_stackingdim, chan_per_worker=None, keeptrials=Tru stacking = targetLayout[0][stackingDim].stop for tk in range(1, self.numTrials): trial = trials[tk] - trlArg = tuple(arg[tk] if isinstance(arg, Sized) and len(arg) == self.numTrials \ + trlArg = tuple(arg[tk] if isinstance(arg, (list, tuple, np.ndarray)) and len(arg) == self.numTrials \ else arg for arg in self.argv) chkshp = chk_list[tk] lyt = [slice(0, stop) for stop in chkshp] @@ -878,7 +874,7 @@ def compute_sequential(self, data, out): sigrid = self.sourceSelectors[nblock] outgrid = self.targetLayout[nblock] argv = tuple(arg[nblock] \ - if isinstance(arg, Sized) and len(arg) == self.numTrials \ + if isinstance(arg, (list, tuple, np.ndarray)) and len(arg) == self.numTrials \ else arg for arg in self.argv) # Catch empty source-array selections; this workaround is not From 34dc16ed156f6d5fb0c9ac48090438092783e9cc Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 13:32:07 +0100 Subject: [PATCH 063/109] FIX: Another bugfix: close operand's HDF file before computation - if arithmetic is performed b/w two or more Syncopy objects, close the operand's HDF5 file before starting to concurrently read from it - do not remove operand selections: they weren't set by our code but the user On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 650f993fa..284887e3d 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -230,27 +230,38 @@ def _perform_computation(baseObj, except ValueError: parallel = False - # Perform actual computation: in case of parallel execution, use a distributed - # lock to prevent ACME from performing chained operations (`x + y + 3``) - # simultaneously (thereby wrecking the underlying HDF5 datasets) + # Perform actual computation: instantiate `ComputationalRoutine` w/extracted info opMethod = SpyArithmetic(operand_dat, operand_idxs, operation=operation, opres_type=opres_type) opMethod.initialize(baseObj, out._stackingDim, chan_per_worker=None, keeptrials=True) + + # In case of parallel execution, be careful: use a distributed lock to prevent + # ACME from performing chained operations (`x + y + 3``) simultaneously (thereby + # wrecking the underlying HDF5 datasets). Similarly, if `operand` is a Syncopy + # object, close its corresponding dataset(s) before starting to concurrently read + # from them (triggering locking errors) if parallel: lock = dd.lock.Lock(name='arithmetic_ops') lock.acquire() + if "BaseData" in str(operand.__class__.__mro__): + for dsetName in operand._hdfFileDatasetProperties: + dset = getattr(operand, dsetName) + dset.file.close() + opMethod.compute(baseObj, out, parallel=parallel, log_dict=log_dct) + + # Re-open `operand`'s dataset(s) and release distributed lock if parallel: + if "BaseData" in str(operand.__class__.__mro__): + operand.data = operand.filename lock.release() # Delete any created subset selections if hasattr(baseObj._selection, "_cleanup"): baseObj._selection = None - if opSel is not None: - operand._selection = None return out From e0ac0b6c6d412fe3d382af8e027b877b275ead48 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 14:18:32 +0100 Subject: [PATCH 064/109] NEW/CHG: Wrapped up tests for AnalogData arithmetics - included testing suite for `AnalogData` arithmetics - restructured `ContinuousData` test data setup (assemble once, reuse for entire module) On branch spy_arithmetic Changes to be committed: modified: syncopy/tests/test_continuousdata.py --- syncopy/tests/test_continuousdata.py | 196 ++++++++++++--------------- 1 file changed, 89 insertions(+), 107 deletions(-) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index ce95ac1d1..09e64668f 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -27,12 +27,55 @@ skip_without_acme = pytest.mark.skipif( not __acme__, reason="acme not available") +# Collect all supported binary arithmetic operators arithmetics = [lambda x, y : x + y, lambda x, y : x - y, lambda x, y : x * y, lambda x, y : x / y, lambda x, y : x ** y] +# Module-wide set of testing selections +trialSelections = [ + "all", # enforce below selections in all trials of `dummy` + [3, 1, 2] # minimally unordered +] +chanSelections = [ + ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered + [4, 2, 2, 5, 5], # repetition + unorderd + range(5, 8), # narrow range + slice(-2, None) # negative-start slice + ] +toiSelections = [ + "all", # non-type-conform string + [0.6], # single inexact match + [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions + ] +toilimSelections = [ + [0.5, 1.5], # regular range + [1.5, 2.0], # minimal range (just two-time points) + [1.0, np.inf] # unbounded from above + ] +foiSelections = [ + "all", # non-type-conform string + [2.6], # single inexact match + [1.1, 1.9, 2.1, 3.9, 9.2, 11.8, 12.9, 5.1, 13.8] # unordered, inexact, repetions + ] +foilimSelections = [ + [2, 11], # regular range + [1, 2.0], # minimal range (just two-time points) + [1.0, np.inf] # unbounded from above + ] +taperSelections = [ + ["TestTaper_03", "TestTaper_01", "TestTaper_01", "TestTaper_02"], # string selection w/repetition + unordered + [0, 1, 1, 2, 3], # preserve repetition, don't convert to slice + range(2, 5), # narrow range + slice(0, 5, 2), # slice w/non-unitary step-size + ] +timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) +freqSelections = list(zip(["foi"] * len(foiSelections), foiSelections)) \ + + list(zip(["foilim"] * len(foilimSelections), foilimSelections)) + class TestAnalogData(): @@ -396,32 +439,11 @@ def test_object_padding(self): # test data-selection via class method def test_dataselection(self): + + # Create testing object and prepare multi-index dummy = AnalogData(data=self.data, trialdefinition=self.trl, samplerate=self.samplerate) - trialSelections = [ - "all", # enforce below selections in all trials of `dummy` - [3, 1] # minimally unordered - ] - chanSelections = [ - ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered - [4, 2, 2, 5, 5], # repetition + unordered - range(5, 8), # narrow range - slice(-2, None) # negative-start slice - ] - toiSelections = [ - "all", # non-type-conform string - [0.6], # single inexact match - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetitions - ] - toilimSelections = [ - [0.5, 1.5], # regular range - [1.5, 2.0], # minimal range (just two-time points) - [1.0, np.inf] # unbounded from above - ] - timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ - + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) - idx = [slice(None)] * len(dummy.dimord) timeIdx = dummy.dimord.index("time") chanIdx = dummy.dimord.index("channel") @@ -453,6 +475,8 @@ def test_dataselection(self): # test arithmetic operations def test_ang_arithmetic(self): + + # Create testing objects and corresponding arrays to perform arithmetics with dummy = AnalogData(data=self.data, trialdefinition=self.trl, samplerate=self.samplerate) @@ -473,44 +497,26 @@ def test_ang_arithmetic(self): dummyOperands = [dummyArr, dummyArr.tolist()] ymmudOperands = [ymmudArr, ymmudArr.tolist()] - trialSelections = [ - "all", # enforce below selections in all trials of `dummy` - [3, 1] # minimally unordered - ] - chanSelections = [ - ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered - [4, 2, 2, 5, 5], # repetition + unorderd - range(5, 8), # narrow range - slice(-2, None) # negative-start slice - ] - toiSelections = [ - "all", # non-type-conform string - [0.6], # single inexact match - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions - ] - toilimSelections = [ - [0.5, 1.5], # regular range - [1.5, 2.0], # minimal range (just two-time points) - [1.0, np.inf] # unbounded from above - ] - timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ - + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) - - + # Perform basic arithmetic with +, -, *, / and ** (pow) for operation in arithmetics: + + # Scalar algebra must be commutative (except for pow) for operand in scalarOperands: result = operation(dummy, operand) # perform operation from right for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(dummy.trials[tk], operand)) - result2 = operation(operand, dummy) # perform operation from left - assert np.array_equal(result2.data, result.data) + # Don't try to compute `2 ** data`` + if operation(2,3) != 8: + result2 = operation(operand, dummy) # perform operation from left + assert np.array_equal(result2.data, result.data) - # same, but swapped `dimord` + # Same as above, but swapped `dimord` result = operation(ymmud, operand) for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) - result2 = operation(operand, ymmud) - assert np.array_equal(result2.data, result.data) + if operation(2,3) != 8: + result2 = operation(operand, ymmud) + assert np.array_equal(result2.data, result.data) # Careful: NumPy tries to avoid failure by broadcasting; instead of relying # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., @@ -525,15 +531,16 @@ def test_ang_arithmetic(self): for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + # Most severe safety hazard: throw two objects at each other (with regular and + # swapped dimord) result = operation(dummy, dummy2) for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) - result = operation(ymmud, ymmud2) for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) - + # Now the most complicated case: user-defined subset selections are present for trialSel in trialSelections: for chanSel in chanSelections: for timeSel in timeSelections: @@ -542,7 +549,7 @@ def test_ang_arithmetic(self): kwdict["channels"] = chanSel kwdict[timeSel[0]] = timeSel[1] - # perform in-place selection and construct array based on new subset + # Perform in-place selection and construct array based on new subset selected = dummy.selectdata(**kwdict) dummy.selectdata(inplace=True, **kwdict) arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) @@ -551,37 +558,51 @@ def test_ang_arithmetic(self): for tk, trl in enumerate(result.trials): assert np.array_equal(trl, operation(selected.trials[tk], operand)) - + # Most most complicated: subset selection present in base object + # and operand thrown at it: only attempt to do this if the selection + # is "well-behaved", i.e., is ordered and does not contain repetitions + # The operator code checks for this, so catch the corresponding + # `SpyValueError` and only attempt to test if coast is clear dummy2.selectdata(inplace=True, **kwdict) try: result = operation(dummy, dummy2) cleanSelection = True except SPYValueError: cleanSelection = False - except: - import pdb; pdb.set_trace() if cleanSelection: for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], selected.trials[tk])) - + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) selected = ymmud.selectdata(**kwdict) ymmud.selectdata(inplace=True, **kwdict) ymmud2.selectdata(inplace=True, **kwdict) result = operation(ymmud, ymmud2) for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], selected.trials[tk])) + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) + # Very important: clear manually set selections for next iteration + dummy._selection = None + dummy2._selection = None + ymmud._selection = None + ymmud2._selection = None + # Finally, perform a representative chained operation to ensure chaining works + result = (dummy + dummy2) / dummy ** 3 + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, + (dummy.trials[tk] + dummy2.trials[tk]) / dummy.trials[tk] ** 3) @skip_without_acme def test_parallel(self, testcluster): # repeat selected test w/parallel processing engine client = dd.Client(testcluster) - par_tests = ["test_relative_array_padding", - "test_absolute_nextpow2_array_padding", - "test_object_padding", - "test_dataselection", - "test_ang_arithmetic"] + par_tests = ["test_ang_arithmetic"] + # par_tests = ["test_relative_array_padding", + # "test_absolute_nextpow2_array_padding", + # "test_object_padding", + # "test_dataselection", + # "test_ang_arithmetic"] for test in par_tests: getattr(self, test)() flush_local_cluster(testcluster) @@ -706,51 +727,12 @@ def test_sd_saveload(self): # test data-selection via class method def test_sd_dataselection(self): + + # Create testing object and prepare multi-index dummy = SpectralData(data=self.data, trialdefinition=self.trl, samplerate=self.samplerate, taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)]) - trialSelections = [ - "all", # enforce below selections in all trials of `dummy` - [3, 1] # minimally unordered - ] - chanSelections = [ - ["channel03", "channel01", "channel01", "channel02"], # string selection w/repetition + unordered - [4, 2, 2, 5, 5], # repetition + unorderd - range(5, 8), # narrow range - slice(-2, None) # negative-start slice - ] - toiSelections = [ - "all", # non-type-conform string - [0.6], # single inexact match - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions - ] - toilimSelections = [ - [0.5, 1.5], # regular range - [1.5, 2.0], # minimal range (just two-time points) - [1.0, np.inf] # unbounded from above - ] - foiSelections = [ - "all", # non-type-conform string - [2.6], # single inexact match - [1.1, 1.9, 2.1, 3.9, 9.2, 11.8, 12.9, 5.1, 13.8] # unordered, inexact, repetions - ] - foilimSelections = [ - [2, 11], # regular range - [1, 2.0], # minimal range (just two-time points) - [1.0, np.inf] # unbounded from above - ] - taperSelections = [ - ["TestTaper_03", "TestTaper_01", "TestTaper_01", "TestTaper_02"], # string selection w/repetition + unordered - [0, 1, 1, 2, 3], # preserve repetition, don't convert to slice - range(2, 5), # narrow range - slice(0, 5, 2), # slice w/non-unitary step-size - ] - timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ - + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) - freqSelections = list(zip(["foi"] * len(foiSelections), foiSelections)) \ - + list(zip(["foilim"] * len(foilimSelections), foilimSelections)) - idx = [slice(None)] * len(dummy.dimord) timeIdx = dummy.dimord.index("time") chanIdx = dummy.dimord.index("channel") From f53d846332e2c5ee94ec21b9d5884e06f54c471b Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 14:34:44 +0100 Subject: [PATCH 065/109] FIX: Avoid explicit use of data property - use `_hdfFileDatasetProperties` to re-set dataset in `_perform_computation` On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 284887e3d..89988c5d3 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -256,7 +256,8 @@ def _perform_computation(baseObj, # Re-open `operand`'s dataset(s) and release distributed lock if parallel: if "BaseData" in str(operand.__class__.__mro__): - operand.data = operand.filename + for dsetName in operand._hdfFileDatasetProperties: + setattr(operand, dsetName, operand.filename) lock.release() # Delete any created subset selections From 85b4dea1310c478b3b709e2389164815db322cb2 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 14:56:37 +0100 Subject: [PATCH 066/109] FIX: Make complex type checking of arrays more robust - use `np.iscomplexobj` instead of `np.iscomplex` to catch complex arrays w/zero imaginary parts On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 89988c5d3..14e04413d 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -42,7 +42,7 @@ def _parse_input(obj1, obj2, operator): # Ensure our base object is not empty try: - data_parser(baseObj, varname="data", empty=False) + data_parser(baseObj, varname="base", empty=False) except Exception as exc: raise exc @@ -81,7 +81,7 @@ def _parse_input(obj1, obj2, operator): operand = np.array(operand) # Ensure complex and real values are not mashed up - if np.all(np.iscomplex(operand)): + if np.iscomplexobj(operand): sameType = lambda dt : "complex" in dt.name else: sameType = lambda dt : "complex" not in dt.name From af7e50bdab47d79a302a51b9ba31cdd2fd32ddc4 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 15:24:55 +0100 Subject: [PATCH 067/109] NEW: General purpose arithmetic tests - included general purpose arithmetic tests (ensure class types are respected etc.) - included checks for `samplerate` in `BaseData` tests - cleaned up obsolete `BaseData` tests On branch spy_arithmetic Changes to be committed: modified: syncopy/tests/test_basedata.py modified: syncopy/tests/test_continuousdata.py --- syncopy/tests/test_basedata.py | 109 +++++++++++++++++++-------- syncopy/tests/test_continuousdata.py | 5 ++ 2 files changed, 82 insertions(+), 32 deletions(-) diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index fa3ffeb7f..e864b14d8 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Test proper functionality of Syncopy's `BaseData` class + helpers -# +# # Builtin/3rd party package imports import os @@ -24,6 +24,13 @@ skip_in_vm = pytest.mark.skipif(is_win_vm(), reason="running in Win VM") skip_in_slurm = pytest.mark.skipif(is_slurm_node(), reason="running on cluster node") +# Collect all supported binary arithmetic operators +arithmetics = [lambda x, y : x + y, + lambda x, y : x - y, + lambda x, y : x * y, + lambda x, y : x / y, + lambda x, y : x ** y] + class TestVirtualData(): @@ -141,6 +148,7 @@ class TestBaseData(): nSpikes = 50 data = {} trl = {} + samplerate = 1.0 # Generate 2D array simulating an AnalogData array data["AnalogData"] = np.arange(1, nChannels * nSamples + 1).reshape(nSamples, nChannels) @@ -176,35 +184,18 @@ def test_data_alloc(self): hname = os.path.join(tdir, "dummy.h5") for dclass in self.classes: - # attempt allocation with random file - with open(fname, "w") as f: - f.write("dummy") - # with pytest.raises(SPYValueError): - # getattr(spd, dclass)(fname) # allocation with HDF5 file h5f = h5py.File(hname, mode="w") h5f.create_dataset("dummy", data=self.data[dclass]) h5f.close() - - # dummy = getattr(spd, dclass)(filename=hname) - # assert np.array_equal(dummy.data, self.data[dclass]) - # assert dummy.filename == hname - # del dummy # allocation using HDF5 dataset directly dset = h5py.File(hname, mode="r+")["dummy"] dummy = getattr(spd, dclass)(data=dset) assert np.array_equal(dummy.data, self.data[dclass]) assert dummy.mode == "r+", dummy.data.file.mode - del dummy - - # # allocation with memmaped npy file - # np.save(fname, self.data[dclass]) - # dummy = getattr(spd, dclass)(filename=fname) - # assert np.array_equal(dummy.data, self.data[dclass]) - # assert dummy.filename == fname - # del dummy + del dummy # allocation using memmap directly np.save(fname, self.data[dclass]) @@ -231,15 +222,10 @@ def test_data_alloc(self): with pytest.raises(SPYValueError): getattr(spd, dclass)(data=dset) - # # attempt allocation using illegal HDF5 file + # allocate with valid dataset of "illegal" file del h5f["dummy"] h5f.create_dataset("dummy1", data=self.data[dclass]) - # FIXME: unused: h5f.create_dataset("dummy2", data=self.data[dclass]) h5f.close() - # with pytest.raises(SPYValueError): - # getattr(spd, dclass)(hname) - - # allocate with valid dataset of "illegal" file dset = h5py.File(hname, mode="r")["dummy1"] dummy = getattr(spd, dclass)(data=dset, filename=fname) @@ -256,7 +242,7 @@ def test_data_alloc(self): np.save(fname, np.ones((self.nChannels,))) with pytest.raises(SPYValueError): getattr(spd, dclass)(data=open_memmap(fname)) - + time.sleep(0.01) del dummy @@ -264,7 +250,8 @@ def test_data_alloc(self): def test_trialdef(self): for dclass in self.classes: dummy = getattr(spd, dclass)(self.data[dclass], - trialdefinition=self.trl[dclass]) + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) assert np.array_equal(dummy.sampleinfo, self.trl[dclass][:, :2]) assert np.array_equal(dummy._t0, self.trl[dclass][:, 2]) assert np.array_equal(dummy.trialinfo.flatten(), self.trl[dclass][:, 3]) @@ -296,7 +283,7 @@ def test_clear(self): def test_filename(self): # ensure we're salting sufficiently to create at least `numf` # distinct pseudo-random filenames in `__storage__` - numf = 1000 + numf = 10000 dummy = AnalogData() fnames = [] for k in range(numf): @@ -310,13 +297,15 @@ def test_copy(self): # shallow copies are views in memory) for dclass in self.classes: dummy = getattr(spd, dclass)(self.data[dclass], - trialdefinition=self.trl[dclass]) + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) dummy2 = dummy.copy() assert dummy.filename == dummy2.filename assert hash(str(dummy.data)) == hash(str(dummy2.data)) assert hash(str(dummy.sampleinfo)) == hash(str(dummy2.sampleinfo)) assert hash(str(dummy._t0)) == hash(str(dummy2._t0)) assert hash(str(dummy.trialinfo)) == hash(str(dummy2.trialinfo)) + assert hash(str(dummy.samplerate)) == hash(str(dummy2.samplerate)) # test shallow + deep copies of memmaps + HDF5 files with tempfile.TemporaryDirectory() as tdir: @@ -330,13 +319,16 @@ def test_copy(self): mm = open_memmap(fname, mode="r") # hash-matching of shallow-copied memmap - dummy = getattr(spd, dclass)(data=mm, trialdefinition=self.trl[dclass]) + dummy = getattr(spd, dclass)(data=mm, + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) dummy2 = dummy.copy() assert dummy.filename == dummy2.filename assert hash(str(dummy.data)) == hash(str(dummy2.data)) assert hash(str(dummy.sampleinfo)) == hash(str(dummy2.sampleinfo)) assert hash(str(dummy._t0)) == hash(str(dummy2._t0)) assert hash(str(dummy.trialinfo)) == hash(str(dummy2.trialinfo)) + assert hash(str(dummy.samplerate)) == hash(str(dummy2.samplerate)) # test integrity of deep-copy dummy3 = dummy.copy(deep=True) @@ -346,16 +338,19 @@ def test_copy(self): assert np.array_equal(dummy._t0, dummy3._t0) assert np.array_equal(dummy.trialinfo, dummy3.trialinfo) assert np.array_equal(dummy.sampleinfo, dummy3.sampleinfo) + assert dummy.samplerate == dummy3.samplerate # hash-matching of shallow-copied HDF5 dataset dummy = getattr(spd, dclass)(data=h5py.File(hname)["dummy"], - trialdefinition=self.trl[dclass]) + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) dummy2 = dummy.copy() assert dummy.filename == dummy2.filename assert hash(str(dummy.data)) == hash(str(dummy2.data)) assert hash(str(dummy.sampleinfo)) == hash(str(dummy2.sampleinfo)) assert hash(str(dummy._t0)) == hash(str(dummy2._t0)) assert hash(str(dummy.trialinfo)) == hash(str(dummy2.trialinfo)) + assert hash(str(dummy.samplerate)) == hash(str(dummy2.samplerate)) # test integrity of deep-copy dummy3 = dummy.copy(deep=True) @@ -364,6 +359,7 @@ def test_copy(self): assert np.array_equal(dummy._t0, dummy3._t0) assert np.array_equal(dummy.trialinfo, dummy3.trialinfo) assert np.array_equal(dummy.data, dummy3.data) + assert dummy.samplerate == dummy3.samplerate # Delete all open references to file objects b4 closing tmp dir del mm, dummy, dummy2, dummy3 @@ -371,3 +367,52 @@ def test_copy(self): # remove file for next round os.unlink(hname) + + # Test basic error handling of arithmetic ops + def test_arithmetic(self): + + # test shallow copy of data arrays (hashes must match up, since + # shallow copies are views in memory) + for dclass in self.classes: + dummy = getattr(spd, dclass)(self.data[dclass], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + otherClass = list(set(self.classes).difference([dclass]))[0] + other = getattr(spd, otherClass)(self.data[otherClass], + trialdefinition=self.trl[otherClass], + samplerate=self.samplerate) + complexArr = np.complex64(dummy.trials[0]) + + # Start w/the one operator that does not handle zeros well... + with pytest.raises(SPYValueError) as spyval: + dummy / 0 + assert "expected non-zero scalar for division" in str(spyval.value) + + # Go through all supported operators and try to sabotage them + for operation in arithmetics: + + # Completely wrong operand + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, np.sin) + assert "expected Syncopy object, scalar or array-like found ufunc" in str(spytyp.value) + + # Empty object + with pytest.raises(SPYValueError) as spyval: + operation(getattr(spd, dclass)(), np.sin) + assert "expected non-empty Syncopy data object" in str(spyval.value) + + # Unbounded scalar + with pytest.raises(SPYValueError) as spyval: + operation(dummy, np.inf) + assert "'inf'; expected finite scalar" in str(spyval.value) + + # Array w/wrong numeric type + with pytest.raises(SPYTypeError) as spyval: + operation(dummy, complexArr) + assert "array of same numerical type (real/complex) found ndarray" in str(spytyp.value) + + # Syncopy object of different type + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, other) + err = "expected Syncopy {} object found {}" + assert err.format(dclass, otherClass) in str(spytyp.value) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 09e64668f..9535a4379 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -500,6 +500,11 @@ def test_ang_arithmetic(self): # Perform basic arithmetic with +, -, *, / and ** (pow) for operation in arithmetics: + # First, ensure `dimord` is respected + with pytest.raises(SPYValueError) as sypval: + operation(dummy, ymmud) + assert "expected Syncopy 'time' x 'channel' data object" + # Scalar algebra must be commutative (except for pow) for operand in scalarOperands: result = operation(dummy, operand) # perform operation from right From e6212e3bcb2ef92e07e0f37ed0782572191e9309 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 23 Nov 2021 16:09:48 +0100 Subject: [PATCH 068/109] NEW : Granger-Geweke frontend integration - works for the standard artifical test data, which doesn't give meaningful results however - appropriate test-data would be coupled AR(2) processes Changes to be committed: modified: ../../dev_frontend.py modified: ../../dev_granger.py modified: AV_compRoutines.py modified: ST_compRoutines.py modified: connectivity_analysis.py modified: const_def.py renamed: granger_causality.py -> granger.py --- dev_frontend.py | 15 +- dev_granger.py | 7 +- syncopy/connectivity/AV_compRoutines.py | 200 +++++++++++++++++- syncopy/connectivity/ST_compRoutines.py | 6 +- syncopy/connectivity/connectivity_analysis.py | 35 +-- syncopy/connectivity/const_def.py | 4 +- .../{granger_causality.py => granger.py} | 27 +-- 7 files changed, 254 insertions(+), 40 deletions(-) rename syncopy/connectivity/{granger_causality.py => granger.py} (80%) diff --git a/dev_frontend.py b/dev_frontend.py index 4ab2ea89a..2561f42b9 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -13,7 +13,7 @@ from syncopy.datatype import SpectralData, padding from syncopy.tests.misc import generate_artificial_data -tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=5) +tdat = generate_artificial_data(inmemory=True, seed=1230, nTrials=50, nChannels=5) foilim = [1, 30] # this still gives type(tsel) = slice :) @@ -22,7 +22,18 @@ # no problems here.. coherence = connectivityanalysis(data=tdat, foilim=None, - output='pow') + output='pow', + taper='dpss', + tapsmofrq=5, + keeptrials=False) + +granger = connectivityanalysis(data=tdat, + method='granger', + foilim=[0, 50], + output='pow', + taper='dpss', + tapsmofrq=5, + keeptrials=False) # D = SpectralData(dimord=['freq','test1','test2','taper']) # D2 = AnalogData(dimord=['freq','test1']) diff --git a/dev_granger.py b/dev_granger.py index 77c1533de..65b467564 100644 --- a/dev_granger.py +++ b/dev_granger.py @@ -2,6 +2,7 @@ from scipy.signal import csd as sci_csd import scipy.signal as sci from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd +from syncopy.connectivity.granger import granger from syncopy.connectivity.ST_compRoutines import cross_spectra_cF #from syncopy.connectivity import wilson_sf import matplotlib.pyplot as ppl @@ -120,8 +121,8 @@ def make_AR2_csd(nSamples=1000, coupling=0.2, fs=200, nTrials=10): # test run -# CSDav, freqs, data = make_AR2_csd(nSamples=2500, nTrials=250) -# H, Sigma, conv = wilson_sf(CSDav, nIter=20) -# G = granger(CSDav, H, Sigma) +CSDav, freqs, data = make_AR2_csd(nSamples=2500, nTrials=500, coupling=0.25) +H, Sigma, conv = wilson_sf(CSDav, nIter=20) +G = granger(CSDav, H, Sigma) diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 3e4fb9b5d..9d4f0ffe5 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -24,10 +24,12 @@ SPYTypeError, SPYWarning, SPYInfo) +from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd +from syncopy.connectivity.granger import granger @unwrap_io -def normalize_csd_cF(trl_av_dat, +def normalize_csd_cF(csd_av_dat, output='abs', chunkShape=None, noCompute=False): @@ -47,7 +49,7 @@ def normalize_csd_cF(trl_av_dat, Parameters ---------- - trl_av_dat : (1, nFreq, N, N) :class:`numpy.ndarray` + csd_av_dat : (1, nFreq, N, N) :class:`numpy.ndarray` Cross-spectral densities for `N` x `N` channels and `nFreq` frequencies averaged over trials. output : {'abs', 'pow', 'fourier'}, default: 'abs' @@ -90,7 +92,7 @@ def normalize_csd_cF(trl_av_dat, """ # it's the same as the input shape! - outShape = trl_av_dat.shape + outShape = csd_av_dat.shape # For initialization of computational routine, # just return output shape and dtype @@ -99,7 +101,7 @@ def normalize_csd_cF(trl_av_dat, return outShape, spectralDTypes[output] # re-shape to (nChannels x nChannels x nFreq) - CS_ij = trl_av_dat.transpose(0, 2, 3, 1)[0, ...] + CS_ij = csd_av_dat.transpose(0, 2, 3, 1)[0, ...] # main diagonal has shape (nFreq x nChannels): the auto spectra diag = CS_ij.diagonal() @@ -310,3 +312,193 @@ def process_metadata(self, data, out): out.samplerate = data.samplerate out.channel_i = np.array(data.channel_i[chanSec]) out.channel_j = np.array(data.channel_j[chanSec]) + + +@unwrap_io +def granger_cF(csd_av_dat, + rtol=1e-8, + nIter=100, + cond_max=1e6, + chunkShape=None, + noCompute=False): + + """ + Given the trial averaged cross spectral densities, + calculates the pairwise Granger-Geweke causalities + for all (non-symmetric!) channel combinations + following the algorithm proposed in [1]_. + + First the CSD matrix is factorized using Wilson's + algorithm, the resulting transfer functions and + noise covariance matrix is then used to calculate + Granger causality according to Eq. 8 in [1]_. + + Selection of channels and frequencies of interest + can and should be done beforehand when calculating the CSDs. + + Critical numerical parameters for Wilson's algorithm + (`rtol`, `nIter`, `cond_max`) have sensitive defaults, + which were tested for datasets with up to + 5000 samples and 256 channels. Changing them is + recommended for expert users only. + + Parameters + ---------- + csd_av_dat : (1, nFreq, N, N) :class:`numpy.ndarray` + Cross-spectral densities for `N` x `N` channels + and `nFreq` frequencies averaged over trials. + rtol : float + Relative error tolerance for Wilson's algorithm + for spectral matrix factorization. Default should + be fine for most cases, handle with care! + nIter : int + Maximum Number of iterations for CSD factorization. A result + is returned if exhausted also if error tolerance was not met. + cond_max : float + The maximal condition number of the spectral matrix. + The CSD matrix can be almost singular in cases of many channels and + low sample number. In these cases Wilson's factorization fails + to converge, as it relies on positive definiteness of the CSD matrix. + If the condition number is above `cond_max`, a brute force + regularization is performed until the regularized CSD matrix has a + condition number below `cond_max`. + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + + Returns + ------- + Granger : (1, nFreq, N, N) :class:`numpy.ndarray` + Spectral Granger-Geweke causality between all channel + combinations. Directionality follows array + notation: causality from i->j is Granger[0,:,i,j], + causality from j->i is Granger[0,:,j,i] + + Notes + ----- + + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms + of time series data." Physical review letters 100.1 (2008): 018701. + + See also + -------- + cross_spectra_cF : :func:`~syncopy.connectivity.ST_compRoutines.cross_spectra_cF` + Single trial (Multi-)tapered cross spectral densities. Trial averages + can be obtained by calling the respective computational routine + with `keeptrials=False`. + wilson_sf : :func:`~syncopy.connectivity.wilson_sf.wilson_sf + Spectral matrix factorization that yields the + transfer functions and noise covariances + from a cross spectral density. + regularize_csd : :func:`~syncopy.connectivity.wilson_sf.regularize_csd + Brute force regularization scheme for the CSD matrix + granger : :func:`~syncopy.connectivity.granger.granger + Given the results of the spectral matrix + factorization, calculates the granger causalities + """ + + # it's the same as the input shape! + outShape = csd_av_dat.shape + + # For initialization of computational routine, + # just return output shape and dtype + # Granger causalities are real + if noCompute: + return outShape, spectralDTypes['abs'] + + # strip off singleton time dimension + # for the backend calls + CSD = csd_av_dat[0] + + # auto-regularize to `cond_max` condition number + # maximal regularization factor is 1e-3, raises a ValueError + # if this is not enough! + CSDreg, factor = regularize_csd(CSD, cond_max=cond_max, eps_max=1e-3) + # call Wilson + + H, Sigma, conv = wilson_sf(CSDreg, nIter=nIter, rtol=rtol) + + # calculate G-causality + Granger = granger(CSDreg, H, Sigma) + + # reattach dummy time axis + return Granger[None, ...] + + +class GrangerCausality(ComputationalRoutine): + + """ + Compute class that computes pairwise Granger causalities + of :class:`~syncopy.CrossSpectralData` objects. + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.connectivityanalysis : parent metafunction + """ + + # the hard wired dimord of the cF + dimord = ['time', 'freq', 'channel_i', 'channel_j'] + + computeFunction = staticmethod(granger_cF) + + method = "" # there is no backend + # 1st argument,the data, gets omitted + valid_kws = list(signature(granger_cF).parameters.keys())[1:] + + def pre_check(self): + ''' + Make sure we have a trial average, + so the input data only consists of `1 trial`. + Can only be performed after initialization! + ''' + + if self.numTrials is None: + lgl = 'Initialize the computational Routine first!' + act = 'ComputationalRoutine not initialized!' + raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) + + if self.numTrials != 1: + lgl = "1 trial: Granger causality can only be computed on trial averages!" + act = f"DataSet contains {self.numTrials} trials" + raise SPYValueError(legal=lgl, varname="data", actual=act) + + def process_metadata(self, data, out): + + # Some index gymnastics to get trial begin/end "samples" + if data._selection is not None: + chanSec = data._selection.channel + trl = data._selection.trialdefinition + for row in range(trl.shape[0]): + trl[row, :2] = [row, row + 1] + else: + chanSec = slice(None) + time = np.arange(len(data.trials)) + time = time.reshape((time.size, 1)) + trl = np.hstack((time, time + 1, + np.zeros((len(data.trials), 1)), + np.array(data.trialinfo))) + + # Attach constructed trialdef-array (if even necessary) + if self.keeptrials: + out.trialdefinition = trl + else: + out.trialdefinition = np.array([[0, 1, 0]]) + + # Attach remaining meta-data + out.samplerate = data.samplerate + out.channel_i = np.array(data.channel_i[chanSec]) + out.channel_j = np.array(data.channel_j[chanSec]) + out.freq = data.freq diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index b635d4af6..0b467bfda 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -13,7 +13,7 @@ # syncopy imports from syncopy.specest.mtmfft import mtmfft from syncopy.shared.const_def import spectralDTypes -from syncopy.shared.errors import SPYWarning +from syncopy.shared.errors import SPYWarning, SPYValueError from syncopy.datatype import padding from syncopy.shared.tools import best_match from syncopy.shared.computational_routine import ComputationalRoutine @@ -175,7 +175,9 @@ def cross_spectra_cF(trl_dat, if norm: # only meaningful for multi-tapering - assert taper == 'dpss' + if taper != 'dpss': + msg = "Normalization of single trial csd only possible with taper='dpss'" + raise SPYValueError(legal=msg, varname="taper", actual=taper) # main diagonal has shape (nChannels x nFreq): the auto spectra diag = CS_ij.diagonal() # get the needed product pairs of the autospectra diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 4433c96fc..50c2edeb8 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -27,14 +27,11 @@ generalParameters ) -# Local imports -from .const_def import ( - availableMethods, -) from .ST_compRoutines import ST_CrossSpectra, ST_CrossCovariance -from .AV_compRoutines import NormalizeCrossSpectra, NormalizeCrossCov +from .AV_compRoutines import NormalizeCrossSpectra, NormalizeCrossCov, GrangerCausality __all__ = ["connectivityanalysis"] +availableMethods = ("coh", "corr", "granger") @unwrap_cfg @@ -43,8 +40,8 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", out=None, - **kwargs): + nTaper=None, toi="all", rtol=1e-7, nIter=100, cond_max=1e6, + out=None, **kwargs): """ coming soon.. @@ -137,10 +134,17 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", if foilim: foi = np.arange(foilim[0], foilim[1] + 1) - # --- Settingn up specific Methods --- + # --- Setting up specific Methods --- - if method == 'coh': + if method in ['coh', 'granger']: + # --- set up computation of the single trial CSDs --- + + if keeptrials is not False: + lgl = "False, trial averaging needed!" + act = keeptrials + raise SPYValueError(lgl, varname="keeptrials", actual=act) + if foi is None and foilim is None: # Construct array of maximally attainable frequencies freqs = np.fft.rfftfreq(nSamples, 1 / data.samplerate) @@ -167,13 +171,20 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", polyremoval=polyremoval, timeAxis=timeAxis, foi=foi) - # hard coded as class attribute st_dimord = ST_CrossSpectra.dimord + if method == 'coh': # final normalization after trial averaging av_compRoutine = NormalizeCrossSpectra(output=output) + if method == 'granger': + # after trial averaging + av_compRoutine = GrangerCausality(rtol=rtol, + nIter=nIter, + cond_max=cond_max + ) + if method == 'corr': # parallel computation over trials @@ -193,12 +204,11 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # the single trial results need a new DataSet st_out = CrossSpectralData(dimord=st_dimord) - # Perform the trial-parallelized computation of the matrix quantity st_compRoutine.initialize(data, st_out._stackingDim, chan_per_worker=None, # no parallelisation over channel possible - keeptrials=keeptrials) # we need trial averaging! + keeptrials=keeptrials) # we most likely need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) # for debugging ccov @@ -233,4 +243,3 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # Either return newly created output object or simply quit return out if new_out else None - diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py index fdab9d9bf..5684353f5 100644 --- a/syncopy/connectivity/const_def.py +++ b/syncopy/connectivity/const_def.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# Constant definitions specific for connectivity +# Constant definitions specific for connectivity - None so far # #: available spectral estimation methods of :func:`~syncopy.connectivity_analysis` -availableMethods = ("coh", "corr") - diff --git a/syncopy/connectivity/granger_causality.py b/syncopy/connectivity/granger.py similarity index 80% rename from syncopy/connectivity/granger_causality.py rename to syncopy/connectivity/granger.py index b6675aeb8..dbb7c29a2 100644 --- a/syncopy/connectivity/granger_causality.py +++ b/syncopy/connectivity/granger.py @@ -10,9 +10,9 @@ def granger(CSD, Hfunc, Sigma): ''' - Computes the pairwise Granger causalities + Computes the pairwise Granger-Geweke causalities for all (non-symmetric!) channel combinations - accoeding to equation 8 in [1]_. + according to equation 8 in [1]_. The transfer functions `Hfunc` and noise covariance `Sigma` are expected to have been already computed. @@ -20,13 +20,20 @@ def granger(CSD, Hfunc, Sigma): Parameters ---------- CSD : (nFreq, N, N) :class:`numpy.ndarray` - Complex cross spectra for all channel combinations i,j. + Complex cross spectra for all channel combinations i,j `N` corresponds to number of input channels. Hfunc : (nFreq, N, N) :class:`numpy.ndarray` - Spectral transfer functions for all channel combinations i,j. + Spectral transfer functions for all channel combinations i,j Sigma : (N, N) :class:`numpy.ndarray` - The noise covariances, should be multiplied by the samplerate - beforehand. + The noise covariances + + Returns + ------- + Granger : (nFreq, N, N) :class:`numpy.ndarray` + Spectral Granger-Geweke causality between all channel + combinations. Directionality follows array + notation: causality from i->j is Granger[:,i,j], + causality from j->i is Granger[:,j,i] See also -------- @@ -52,7 +59,6 @@ def granger(CSD, Hfunc, Sigma): # Smat(f) = S_11 S_22 S_33 # S_11 S_22 S_33 Smat = auto_spectra[:, None, :] * np.ones(nChannels)[:, None] - assert CSD.shape == Smat.shape # Granger i->j needs H_ji entry Hmat = np.abs(Hfunc.transpose(0, 2, 1))**2 @@ -71,9 +77,4 @@ def granger(CSD, Hfunc, Sigma): # linear causality i -> j Granger = np.log(Smat / denom) - return Granger - - - - - + return Granger From 60bf5b67887d9fb4d9cce9e8d079f71701a9b93e Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 23 Nov 2021 16:39:47 +0100 Subject: [PATCH 069/109] NEW : Granger backend test - two unidirectionally coupled AR(2) processes akin to Dhamala 2008 - There are some artifacts at the boundary of the frequency axis (0Hz, f_Nyquist), should not be a problem in practice (only if the foi is close to one of the boundaries..) but one should inquire if this is 'normal' also for other implementations Changes to be committed: modified: test_connectivity.py --- syncopy/tests/backend/test_connectivity.py | 84 ++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 2c5f30b5c..cfd1cc219 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -5,6 +5,7 @@ from syncopy.connectivity import ST_compRoutines as stCR from syncopy.connectivity import AV_compRoutines as avCR from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd +from syncopy.connectivity.granger import granger def test_coherence(): @@ -21,7 +22,7 @@ def test_coherence(): harm_freq = 40 phase_shifts = np.array([0, np.pi / 2, np.pi]) - nTrials = 50 + nTrials = 100 # shape is (1, nFreq, nChannel, nChannel) nFreq = nSamples // 2 + 1 @@ -46,7 +47,7 @@ def test_coherence(): assert avCSD.shape == CSD.shape avCSD += CSD - # this is the result of the + # this is the trial average avCSD /= nTrials # perform the normalisation on the trial averaged csd's @@ -67,7 +68,7 @@ def test_coherence(): assert ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') # we test for the highest peak sitting at - # the vicinity (± 5Hz) of one the harmonic + # the vicinity (± 5Hz) of the harmonic peak_val = np.max(coh) peak_idx = np.argmax(coh) peak_freq = freqs[peak_idx] @@ -237,9 +238,84 @@ def test_wilson(): '-o', label='factorized CSD', ms=3) ax.set_xlim((f1 - 5, f2 + 5)) + +def test_granger(): + + ''' + Test the granger causality measure + with uni-directionally coupled AR(2) + processes akin to the source publication: + + Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms + of time series data." Physical review letters 100.1 (2008): 018701. + ''' + + fs = 200 # Hz + nSamples = 2500 + nTrials = 50 + # both AR(2) processes have same parameters + # and yield a spectral peak at 40Hz + alpha1, alpha2 = 0.55, -0.8 + coupling = 0.25 + + CSDav = np.zeros((nSamples // 2 + 1, 2, 2), dtype=np.complex64) + for _ in range(nTrials): + + # -- simulate 2 AR(2) processes -- + + sol = np.zeros((nSamples, 2)) + # pick the 1st values at random + xs_ini = np.random.randn(2, 2) + sol[:2, :] = xs_ini + for i in range(1, nSamples): + sol[i, 1] = alpha1 * sol[i - 1, 1] + alpha2 * sol[i - 2, 1] + sol[i, 1] += np.random.randn() + # X2 drives X1 + sol[i, 0] = alpha1 * sol[i - 1, 0] + alpha2 * sol[i - 2, 0] + sol[i, 0] += sol[i - 1, 1] * coupling + sol[i, 0] += np.random.randn() + + # --- get CSD --- + bw = 5 + NW = bw * nSamples / (2 * 1000) + Kmax = int(2 * NW - 1) # optimal number of tapers + CS2, freqs = stCR.cross_spectra_cF(sol, fs, + taper='dpss', + taper_opt={'Kmax' : Kmax, 'NW' : NW}, + fullOutput=True) + + CSD = CS2[0, ...] + CSDav += CSD + + CSDav /= nTrials + # with only 2 channels this CSD is well conditioned + assert np.linalg.cond(CSDav).max() < 1e2 + H, Sigma, conv = wilson_sf(CSDav) + + G = granger(CSDav, H, Sigma) + assert G.shape == CSDav.shape + + # check for directional causality at 40Hz + freq_idx = np.argmin(freqs < 40) + assert 39 < freqs[freq_idx] < 41 + + # check low to no causality for 1->2 + assert G[freq_idx, 0, 1] < 0.1 + # check high causality for 2->1 + assert G[freq_idx, 1, 0] > 0.8 + + fig, ax = ppl.subplots(figsize=(6, 4)) + ax.set_xlabel('frequency (Hz)') + ax.set_ylabel(r'Granger causality(f)') + assert ax.plot(freqs, G[:, 0, 1], label=r'Granger $1\rightarrow2$') + assert ax.plot(freqs, G[:, 1, 0], label=r'Granger $2\rightarrow1$') + ax.legend() + # --- Helper routines --- - + + # noisy phase evolution -> phase diffusion def phase_evo(omega0, eps, fs=1000, N=1000): wn = np.random.randn(N) From 7ed1431707d44f7d326978aeafaab66bdace42a4 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 23 Nov 2021 16:43:46 +0100 Subject: [PATCH 070/109] NEW : Granger backend test ..cleaning up - two unidirectionally coupled AR(2) processes akin to Dhamala 2008 - There are some artifacts at the boundary of the frequency axis (0Hz, f_Nyquis t), should not be a problem in practice (only if the foi is close to one of the boundaries..) but one should inquire if this is 'normal' also for other implementations Changes to be committed: deleted: ../../../dev_granger.py --- dev_granger.py | 128 ------------------------------------------------- 1 file changed, 128 deletions(-) delete mode 100644 dev_granger.py diff --git a/dev_granger.py b/dev_granger.py deleted file mode 100644 index 65b467564..000000000 --- a/dev_granger.py +++ /dev/null @@ -1,128 +0,0 @@ -import numpy as np -from scipy.signal import csd as sci_csd -import scipy.signal as sci -from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd -from syncopy.connectivity.granger import granger -from syncopy.connectivity.ST_compRoutines import cross_spectra_cF -#from syncopy.connectivity import wilson_sf -import matplotlib.pyplot as ppl - - -# noisy phase evolution -def phase_evo(omega0, eps, fs=1000, N=1000): - wn = np.random.randn(N) - delta_ts = np.ones(N) * 1 / fs - phase = np.cumsum(omega0 * delta_ts + eps * wn) - return phase - - -def brown_noise(N): - wn = np.random.randn(N) - xs = np.cumsum(wn) - - return xs - - -def plot_wilson_errs(errs, label='', c='k'): - - fig, ax = ppl.subplots(figsize=(6,4), num=2) - ax.set_xlabel('Iteration Step') - ax.set_ylabel(r'rel. Error $\frac{|CSD - \Psi\Psi^*|}{|CSD|}$') - ax.semilogy() - # ax.plot(errs, '-o', label=label, c=c) - - fig.subplots_adjust(left=0.15, bottom=0.2) - return ax - - -def make_test_data(nChannels=10, nSamples=1000, bw=5): - - # white noise ensemble - fs = 1000 - tvec = np.arange(nSamples) / fs - - data = np.zeros((nSamples, nChannels)) - for i in range(nChannels): - p1 = phase_evo(30 * 2 * np.pi, 0.1, N=nSamples) - p2 = phase_evo(60 * 2 * np.pi, 0.25, N=nSamples) - - data[:, i] = np.cos(p1) + np.sin(p2) + .5 * np.random.randn(nSamples) - # data[:, i] = brown_noise(nSamples) - - NW = bw * nSamples / (2 * fs) - Kmax = int(2 * NW - 1) # optimal number of tapers - - CS2, freqs = cross_spectra_cF(data, fs, taper='dpss', taper_opt={'Kmax' : Kmax, 'NW' : NW}, norm=False, fullOutput=True) - - CSD = CS2[0, ...] - - return CSD - - -def cond_samples(Ns, nChannels=2): - - ''' - Screens condition number for CSDs - with different channel Numbers - ''' - - cns = [] - - for N in Ns: - cn = np.linalg.cond(make_test_data(bw=5, nSamples=N, nChannels=nChannels)).max() - cns.append(cn) - - ax = ppl.gca() - ax.set_xlabel('nSamples') - ax.set_ylabel('Condition Number') - ax.plot(Ns, cns, '-o', label=f'nChannels={nChannels}') - ax.set_ylim((-1, 5000)) - - -def make_AR2_csd(nSamples=1000, coupling=0.2, fs=200, nTrials=10): - - # both processes have same parameters - alpha1, alpha2 = 0.55, -0.8 - - CSDav = np.zeros((nSamples // 2 + 1, 2, 2), dtype=np.complex64) - for _ in range(nTrials): - sol = np.zeros((nSamples, 2)) - - # pick the 1st values at random - xs_ini = np.random.randn(2,2) - - sol[:2,:] = xs_ini - - for i in range(1, nSamples): - sol[i, 1] = alpha1 * sol[i - 1, 1] + alpha2 * sol[i - 2, 1] - sol[i, 1] += np.random.randn() - - # X2 drives X1 - sol[i, 0] = alpha1 * sol[i - 1, 0] + alpha2 * sol[i - 2, 0] - sol[i, 0] += sol[i - 1, 1] * coupling - sol[i, 0] += np.random.randn() - - # --- get CSD --- - bw = 5 - NW = bw * nSamples / (2 * 1000) - Kmax = int(2 * NW - 1) # optimal number of tapers - CS2, freqs = cross_spectra_cF(sol, fs, - taper='dpss', - taper_opt={'Kmax' : Kmax, 'NW' : NW}, - norm=False, - fullOutput=True) - - CSD = CS2[0, ...] - CSDav += CSD - - print(Kmax) - CSDav /= nTrials - return CSDav, freqs, sol - - -# test run -CSDav, freqs, data = make_AR2_csd(nSamples=2500, nTrials=500, coupling=0.25) -H, Sigma, conv = wilson_sf(CSDav, nIter=20) -G = granger(CSDav, H, Sigma) - - From 910ff67ec5b290bc5df6acbf242702e394e96ee4 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 19:34:24 +0100 Subject: [PATCH 071/109] NEW/CHG: Added additional safeguard for accidental type casts - add verification of real/complex dtype to scalar operands as well; supplemented corresponding `BaseData` checks as well - new helper function `_check_complex_operand` in arithmetic.py parses scalar and array operands for complexity On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/methods/arithmetic.py | 32 ++++++++++++++++++-------- syncopy/tests/test_basedata.py | 8 ++++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index 14e04413d..e8cdf6e4d 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -67,7 +67,10 @@ def _parse_input(obj1, obj2, operator): if operator == "/" and operand == 0: raise SPYValueError("non-zero scalar for division", varname="operand", actual=str(operand)) - # Determine numeric type of operation's result + # Ensure complex and real values are not mashed up + _check_complex_operand(baseTrials, operand, "scalar") + + # Determine exact numeric type of operation's result opres_type = np.result_type(*(trl.dtype for trl in baseTrials), operand) # That's it set output vars @@ -81,15 +84,9 @@ def _parse_input(obj1, obj2, operator): operand = np.array(operand) # Ensure complex and real values are not mashed up - if np.iscomplexobj(operand): - sameType = lambda dt : "complex" in dt.name - else: - sameType = lambda dt : "complex" not in dt.name - if not all(sameType(trl.dtype) for trl in baseTrials): - lgl = "array of same numerical type (real/complex)" - raise SPYTypeError(operand, varname="operand", expected=lgl) + _check_complex_operand(baseTrials, operand, "array") - # Determine the numeric type of the operation's result + # Determine exact numeric type of the operation's result opres_type = np.result_type(*(trl.dtype for trl in baseTrials), operand.dtype) # Ensure shapes match up @@ -182,6 +179,23 @@ def _parse_input(obj1, obj2, operator): return baseObj, operand, operand_dat, opres_type, operand_idxs +def _check_complex_operand(baseTrials, operand, opDimType): + """ + Coming soon... + """ + + # Ensure complex and real values are not mashed up + if np.iscomplexobj(operand): + sameType = lambda dt : "complex" in dt.name + else: + sameType = lambda dt : "complex" not in dt.name + if not all(sameType(trl.dtype) for trl in baseTrials): + lgl = "{} of same mathematical type (real/complex)" + raise SPYTypeError(operand, varname="operand", expected=lgl.format(opDimType)) + + return + + def _perform_computation(baseObj, operand, operand_dat, diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index e864b14d8..22da7f526 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -382,6 +382,7 @@ def test_arithmetic(self): trialdefinition=self.trl[otherClass], samplerate=self.samplerate) complexArr = np.complex64(dummy.trials[0]) + complexNum = 3+4j # Start w/the one operator that does not handle zeros well... with pytest.raises(SPYValueError) as spyval: @@ -406,8 +407,13 @@ def test_arithmetic(self): operation(dummy, np.inf) assert "'inf'; expected finite scalar" in str(spyval.value) + # Complex scalar (all test data are real) + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, complexNum) + assert "expected scalar of same mathematical type (real/complex)" in str(spytyp.value) + # Array w/wrong numeric type - with pytest.raises(SPYTypeError) as spyval: + with pytest.raises(SPYTypeError) as spytyp: operation(dummy, complexArr) assert "array of same numerical type (real/complex) found ndarray" in str(spytyp.value) From b36f69ce8b599fca26177832de74b452cea67b1a Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 20:01:54 +0100 Subject: [PATCH 072/109] FIX: And another testing bugfix - do not allow Syncopy objects containing selections requiring advanced (aka fancy) indexing (i.e., dataset indices containing multiple lists and slices) as operands for arithmetics On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py --- syncopy/datatype/methods/arithmetic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index e8cdf6e4d..fb9f2bf1b 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -11,7 +11,7 @@ # Local imports from syncopy import __acme__ from syncopy.shared.parsers import data_parser -from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning, SPYParallelError from syncopy.shared.computational_routine import ComputationalRoutine from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.computational_routine import ComputationalRoutine @@ -156,14 +156,20 @@ def _parse_input(obj1, obj2, operator): raise SPYValueError(lgl.format(baseShapes), varname="operand", actual=act.format(opndShapes)) - # Avoid things becoming too nasty: if operand contains wild selections - # (unordered lists or index repetitions), abort + # Avoid things becoming too nasty: if `operand`` contains wild selections + # (unordered lists or index repetitions) or selections requiring advanced + # (aka fancy) indexing (multiple slices mixed with lists), abort for trl in opndTrials: if any(np.diff(sel).min() <= 0 if isinstance(sel, list) and len(sel) > 1 \ else False for sel in trl.idx): lgl = "Syncopy object with ordered unreverberated subset selection" act = "Syncopy object with selection {}" raise SPYValueError(lgl, varname="operand", actual=act.format(operand._selection)) + if sum(isinstance(sel, slice) for sel in trl.idx) > 1 and \ + sum(isinstance(sel, list) for sel in trl.idx) > 1: + lgl = "Syncopy object without selections requiring advanced indexing" + act = "Syncopy object with selection {}" + raise SPYValueError(lgl, varname="operand", actual=act.format(operand._selection)) # Propagate indices for fetching data from operand operand_idxs = [trl.idx for trl in opndTrials] From 1896c31556e2da39aaf18766731368648e95b6f6 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 23 Nov 2021 20:02:33 +0100 Subject: [PATCH 073/109] WIP: Started writing SpectralData arithmetic tests - copy-pasted AnalogData tests too see what works; this testing scheme might live best within its own encapsulated function... - included safeguard to prevent running parallel mtmfft tests on low-memory machines as well On branch spy_arithmetic Changes to be committed: modified: syncopy/tests/test_continuousdata.py modified: syncopy/tests/test_specest.py --- syncopy/tests/test_continuousdata.py | 162 ++++++++++++++++++++++++++- syncopy/tests/test_specest.py | 1 + 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 9535a4379..a4a62bf15 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -501,9 +501,16 @@ def test_ang_arithmetic(self): for operation in arithmetics: # First, ensure `dimord` is respected - with pytest.raises(SPYValueError) as sypval: + with pytest.raises(SPYValueError) as spyval: operation(dummy, ymmud) - assert "expected Syncopy 'time' x 'channel' data object" + assert "expected Syncopy 'time' x 'channel' data object" in str (spyval.value) + + # Next, ensure trial counts are properly vetted + dummy2.selectdata(trials=[0], inplace=True) + with pytest.raises(SPYValueError) as spyval: + operation(dummy, dummy2) + assert "Syncopy object with same number of trials (selected)" in str (spyval.value) + dummy2._selection = None # Scalar algebra must be commutative (except for pow) for operand in scalarOperands: @@ -778,6 +785,157 @@ def test_sd_dataselection(self): assert np.array_equal(cfg.out.data, selected.data) time.sleep(0.05) + # test arithmetic operations + def test_sd_arithmetic(self): + + # Create testing objects and corresponding arrays to perform arithmetics with + dummy = SpectralData(data=self.data, + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)]) + dummyC = SpectralData(data=np.complex64(self.data), + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)]) + ymmud = SpectralData(data=np.transpose(self.data, [3, 2, 1, 0]), + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)], + dimord=SpectralData._defaultDimord[::-1]) + dummy2 = SpectralData(data=self.data, + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)]) + ymmud2 = SpectralData(data=np.transpose(self.data, [3, 2, 1, 0]), + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)], + dimord=SpectralData._defaultDimord[::-1]) + dummyArr = 2 * np.ones((dummy.trials[0].shape)) + ymmudArr = 2 * np.ones((ymmud.trials[0].shape)) + scalarOperands = [2, np.pi] + dummyOperands = [dummyArr, dummyArr.tolist()] + ymmudOperands = [ymmudArr, ymmudArr.tolist()] + + # Perform basic arithmetic with +, -, *, / and ** (pow) + for operation in arithmetics: + + # First, ensure `dimord` is respected + with pytest.raises(SPYValueError) as spyval: + operation(dummy, ymmud) + assert "expected Syncopy 'time' x 'channel' data object" in str(spyval.value) + + # Next, ensure trial counts are properly vetted + dummy2.selectdata(trials=[0], inplace=True) + with pytest.raises(SPYValueError) as spyval: + operation(dummy, dummy2) + assert "Syncopy object with same number of trials (selected)" in str (spyval.value) + dummy2._selection = None + + # Scalar algebra must be commutative (except for pow) + for operand in scalarOperands: + result = operation(dummy, operand) # perform operation from right + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + # Don't try to compute `2 ** data`` + if operation(2,3) != 8: + result2 = operation(operand, dummy) # perform operation from left + assert np.array_equal(result2.data, result.data) + + # Same as above, but swapped `dimord` + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + if operation(2,3) != 8: + result2 = operation(operand, ymmud) + assert np.array_equal(result2.data, result.data) + + # Careful: NumPy tries to avoid failure by broadcasting; instead of relying + # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., + # ``np.ones((3,3)) + data`` performs ``1 + data`` nine times, so don't + # test for left/right arithmetics... + for operand in dummyOperands: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + for operand in ymmudOperands: + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + + # Ensure erroneous object type-casting is prevented + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, dummyC) + assert "Syncopy data object of same numerical type (real/complex)" in str(spytyp.value) + + # Most severe safety hazard: throw two objects at each other (with regular and + # swapped dimord) + result = operation(dummy, dummy2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) + + # Now the most complicated case: user-defined subset selections are present + for trialSel in trialSelections: + for chanSel in chanSelections: + for timeSel in timeSelections: + for freqSel in freqSelections: + for taperSel in taperSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["channels"] = chanSel + kwdict[timeSel[0]] = timeSel[1] + kwdict[freqSel[0]] = freqSel[1] + kwdict["tapers"] = taperSel + + # Perform in-place selection and construct array based on new subset + selected = dummy.selectdata(**kwdict) + dummy.selectdata(inplace=True, **kwdict) + arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) + for operand in [np.pi, arr]: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], operand)) + + # Most most complicated: subset selection present in base object + # and operand thrown at it: only attempt to do this if the selection + # is "well-behaved", i.e., is ordered and does not contain repetitions + # The operator code checks for this, so catch the corresponding + # `SpyValueError` and only attempt to test if coast is clear + dummy2.selectdata(inplace=True, **kwdict) + try: + result = operation(dummy, dummy2) + cleanSelection = True + except SPYValueError: + cleanSelection = False + # except: + # import pdb; pdb.set_trace() + if cleanSelection: + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) + selected = ymmud.selectdata(**kwdict) + ymmud.selectdata(inplace=True, **kwdict) + ymmud2.selectdata(inplace=True, **kwdict) + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) + + # Very important: clear manually set selections for next iteration + dummy._selection = None + dummy2._selection = None + ymmud._selection = None + ymmud2._selection = None + + # Finally, perform a representative chained operation to ensure chaining works + result = (dummy + dummy2) / dummy ** 3 + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, + (dummy.trials[tk] + dummy2.trials[tk]) / dummy.trials[tk] ** 3) + @skip_without_acme def test_sd_parallel(self, testcluster): # repeat selected test w/parallel processing engine diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index f6d2a75bc..99c9fb322 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -403,6 +403,7 @@ def test_vdata(self): gc.collect() # force-garbage-collect object so that tempdir can be closed @skip_without_acme + @skip_low_mem def test_parallel(self, testcluster): # collect all tests of current class and repeat them using dask # (skip VirtualData tests since ``wrapper_io`` expects valid headers) From 33161df8e8d0ea9ae0fadffab008fabe20bd0f23 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 24 Nov 2021 11:56:10 +0100 Subject: [PATCH 074/109] NEW/CHG: Re-vamped testing pipeline + extended data-selection tests - changed default dtype of `SpectralData` testing array to "float" (to avoid round-off errors when comparing actual/expected results) - wrapped arithmetic testing machinery in two helper functions: `_base_op_tests` and `_selection_op_tests` - extended data-selection tests to incorporate objects w/non-default `dimord` (closes #156) On branch spy_arithmetic Changes to be committed: modified: syncopy/tests/test_continuousdata.py modified: syncopy/tests/test_discretedata.py --- syncopy/tests/test_continuousdata.py | 456 +++++++++++---------------- syncopy/tests/test_discretedata.py | 136 ++++---- 2 files changed, 263 insertions(+), 329 deletions(-) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index a4a62bf15..b59b0bb95 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -77,6 +77,109 @@ + list(zip(["foilim"] * len(foilimSelections), foilimSelections)) +# Local helper function for performing basic arithmetic tests +def _base_op_tests(dummy, ymmud, dummy2, ymmud2, dummyC, operation): + + dummyArr = 2 * np.ones((dummy.trials[0].shape)) + ymmudArr = 2 * np.ones((ymmud.trials[0].shape)) + scalarOperands = [2, np.pi] + dummyOperands = [dummyArr, dummyArr.tolist()] + ymmudOperands = [ymmudArr, ymmudArr.tolist()] + + # Ensure trial counts are properly vetted + dummy2.selectdata(trials=[0], inplace=True) + with pytest.raises(SPYValueError) as spyval: + operation(dummy, dummy2) + assert "Syncopy object with same number of trials (selected)" in str (spyval.value) + dummy2._selection = None + + # Scalar algebra must be commutative (except for pow) + for operand in scalarOperands: + result = operation(dummy, operand) # perform operation from right + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + # Don't try to compute `2 ** data`` + if operation(2,3) != 8: + result2 = operation(operand, dummy) # perform operation from left + assert np.array_equal(result2.data, result.data) + + # Same as above, but swapped `dimord` + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + if operation(2,3) != 8: + result2 = operation(operand, ymmud) + assert np.array_equal(result2.data, result.data) + + # Careful: NumPy tries to avoid failure by broadcasting; instead of relying + # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., + # ``np.ones((3,3)) + data`` performs ``1 + data`` nine times, so don't + # test for left/right arithmetics... + for operand in dummyOperands: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], operand)) + for operand in ymmudOperands: + result = operation(ymmud, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) + + # Ensure erroneous object type-casting is prevented + if dummyC is not None: + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, dummyC) + assert "Syncopy data object of same numerical type (real/complex)" in str(spytyp.value) + + # Most severe safety hazard: throw two objects at each other (with regular and + # swapped dimord) + result = operation(dummy, dummy2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) + +def _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation): + + # Perform in-place selection and construct array based on new subset + selected = dummy.selectdata(**kwdict) + dummy.selectdata(inplace=True, **kwdict) + arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) + for operand in [np.pi, arr]: + result = operation(dummy, operand) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], operand)) + + # Most most complicated: subset selection present in base object + # and operand thrown at it: only attempt to do this if the selection + # is "well-behaved", i.e., is ordered and does not contain repetitions + # The operator code checks for this, so catch the corresponding + # `SpyValueError` and only attempt to test if coast is clear + dummy2.selectdata(inplace=True, **kwdict) + try: + result = operation(dummy, dummy2) + cleanSelection = True + except SPYValueError: + cleanSelection = False + if cleanSelection: + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) + selected = ymmud.selectdata(**kwdict) + ymmud.selectdata(inplace=True, **kwdict) + ymmud2.selectdata(inplace=True, **kwdict) + result = operation(ymmud, ymmud2) + for tk, trl in enumerate(result.trials): + assert np.array_equal(trl, operation(selected.trials[tk], + selected.trials[tk])) + + # Very important: clear manually set selections for next iteration + dummy._selection = None + dummy2._selection = None + ymmud._selection = None + ymmud2._selection = None + + class TestAnalogData(): # Allocate test-dataset @@ -440,38 +543,43 @@ def test_object_padding(self): # test data-selection via class method def test_dataselection(self): - # Create testing object and prepare multi-index + # Create testing objects (regular and swapped dimords) dummy = AnalogData(data=self.data, trialdefinition=self.trl, samplerate=self.samplerate) - idx = [slice(None)] * len(dummy.dimord) - timeIdx = dummy.dimord.index("time") - chanIdx = dummy.dimord.index("channel") - - for trialSel in trialSelections: - for chanSel in chanSelections: - for timeSel in timeSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["channels"] = chanSel - kwdict[timeSel[0]] = timeSel[1] - cfg = StructDict(kwdict) - # data selection via class-method + `Selector` instance for indexing - selected = dummy.selectdata(**kwdict) - time.sleep(0.05) - selector = Selector(dummy, kwdict) - idx[chanIdx] = selector.channel - for tk, trialno in enumerate(selector.trials): - idx[timeIdx] = selector.time[tk] - assert np.array_equal(selected.trials[tk].squeeze(), - dummy.trials[trialno][idx[0], :][:, idx[1]].squeeze()) - cfg.data = dummy - cfg.out = AnalogData(dimord=AnalogData._defaultDimord) - # data selection via package function and `cfg`: ensure equality - selectdata(cfg) - assert np.array_equal(cfg.out.channel, selected.channel) - assert np.array_equal(cfg.out.data, selected.data) - time.sleep(0.05) + ymmud = AnalogData(data=self.data.T, + trialdefinition=self.trl, + samplerate=self.samplerate, + dimord=AnalogData._defaultDimord[::-1]) + + for obj in [dummy, ymmud]: + idx = [slice(None)] * len(obj.dimord) + timeIdx = obj.dimord.index("time") + chanIdx = obj.dimord.index("channel") + for trialSel in trialSelections: + for chanSel in chanSelections: + for timeSel in timeSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["channels"] = chanSel + kwdict[timeSel[0]] = timeSel[1] + cfg = StructDict(kwdict) + # data selection via class-method + `Selector` instance for indexing + selected = obj.selectdata(**kwdict) + time.sleep(0.05) + selector = Selector(obj, kwdict) + idx[chanIdx] = selector.channel + for tk, trialno in enumerate(selector.trials): + idx[timeIdx] = selector.time[tk] + assert np.array_equal(selected.trials[tk].squeeze(), + obj.trials[trialno][idx[0], :][:, idx[1]].squeeze()) + cfg.data = obj + cfg.out = AnalogData(dimord=obj.dimord) + # data selection via package function and `cfg`: ensure equality + selectdata(cfg) + assert np.array_equal(cfg.out.channel, selected.channel) + assert np.array_equal(cfg.out.data, selected.data) + time.sleep(0.05) # test arithmetic operations def test_ang_arithmetic(self): @@ -491,11 +599,6 @@ def test_ang_arithmetic(self): trialdefinition=self.trl, samplerate=self.samplerate, dimord=AnalogData._defaultDimord[::-1]) - dummyArr = 2 * np.ones((dummy.trials[0].shape)) - ymmudArr = 2 * np.ones((ymmud.trials[0].shape)) - scalarOperands = [2, np.pi] - dummyOperands = [dummyArr, dummyArr.tolist()] - ymmudOperands = [ymmudArr, ymmudArr.tolist()] # Perform basic arithmetic with +, -, *, / and ** (pow) for operation in arithmetics: @@ -505,52 +608,7 @@ def test_ang_arithmetic(self): operation(dummy, ymmud) assert "expected Syncopy 'time' x 'channel' data object" in str (spyval.value) - # Next, ensure trial counts are properly vetted - dummy2.selectdata(trials=[0], inplace=True) - with pytest.raises(SPYValueError) as spyval: - operation(dummy, dummy2) - assert "Syncopy object with same number of trials (selected)" in str (spyval.value) - dummy2._selection = None - - # Scalar algebra must be commutative (except for pow) - for operand in scalarOperands: - result = operation(dummy, operand) # perform operation from right - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], operand)) - # Don't try to compute `2 ** data`` - if operation(2,3) != 8: - result2 = operation(operand, dummy) # perform operation from left - assert np.array_equal(result2.data, result.data) - - # Same as above, but swapped `dimord` - result = operation(ymmud, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) - if operation(2,3) != 8: - result2 = operation(operand, ymmud) - assert np.array_equal(result2.data, result.data) - - # Careful: NumPy tries to avoid failure by broadcasting; instead of relying - # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., - # ``np.ones((3,3)) + data`` performs ``1 + data`` nine times, so don't - # test for left/right arithmetics... - for operand in dummyOperands: - result = operation(dummy, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], operand)) - for operand in ymmudOperands: - result = operation(ymmud, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) - - # Most severe safety hazard: throw two objects at each other (with regular and - # swapped dimord) - result = operation(dummy, dummy2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) - result = operation(ymmud, ymmud2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) + _base_op_tests(dummy, ymmud, dummy2, ymmud2, None, operation) # Now the most complicated case: user-defined subset selections are present for trialSel in trialSelections: @@ -561,43 +619,7 @@ def test_ang_arithmetic(self): kwdict["channels"] = chanSel kwdict[timeSel[0]] = timeSel[1] - # Perform in-place selection and construct array based on new subset - selected = dummy.selectdata(**kwdict) - dummy.selectdata(inplace=True, **kwdict) - arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) - for operand in [np.pi, arr]: - result = operation(dummy, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], operand)) - - # Most most complicated: subset selection present in base object - # and operand thrown at it: only attempt to do this if the selection - # is "well-behaved", i.e., is ordered and does not contain repetitions - # The operator code checks for this, so catch the corresponding - # `SpyValueError` and only attempt to test if coast is clear - dummy2.selectdata(inplace=True, **kwdict) - try: - result = operation(dummy, dummy2) - cleanSelection = True - except SPYValueError: - cleanSelection = False - if cleanSelection: - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], - selected.trials[tk])) - selected = ymmud.selectdata(**kwdict) - ymmud.selectdata(inplace=True, **kwdict) - ymmud2.selectdata(inplace=True, **kwdict) - result = operation(ymmud, ymmud2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], - selected.trials[tk])) - - # Very important: clear manually set selections for next iteration - dummy._selection = None - dummy2._selection = None - ymmud._selection = None - ymmud2._selection = None + _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) # Finally, perform a representative chained operation to ensure chaining works result = (dummy + dummy2) / dummy ** 3 @@ -628,7 +650,7 @@ class TestSpectralData(): ns = 30 nt = 5 nf = 15 - data = np.arange(1, nc * ns * nt * nf + 1).reshape(ns, nt, nf, nc) + data = np.arange(1, nc * ns * nt * nf + 1, dtype="float").reshape(ns, nt, nf, nc) trl = np.vstack([np.arange(0, ns, 5), np.arange(5, ns + 5, 5), np.ones((int(ns / 5), )), @@ -673,21 +695,6 @@ def test_sd_trialretrieval(self): trl_ref = self.data2[..., start:start + 5] assert np.array_equal(dummy._get_trial(trlno), trl_ref) - # # test ``_copy_trial`` with memmap'ed data - # with tempfile.TemporaryDirectory() as tdir: - # fname = os.path.join(tdir, "dummy.npy") - # np.save(fname, self.data) - # mm = open_memmap(fname, mode="r") - # dummy = SpectralData(mm, trialdefinition=self.trl) - # for trlno, start in enumerate(range(0, self.ns, 5)): - # trl_ref = self.data[start:start + 5, ...] - # trl_tmp = dummy._copy_trial(trlno, - # dummy.filename, - # dummy.dimord, - # dummy.sampleinfo, - # None) - # assert np.array_equal(trl_tmp, trl_ref) - # del mm, dummy del dummy def test_sd_saveload(self): @@ -740,50 +747,56 @@ def test_sd_saveload(self): # test data-selection via class method def test_sd_dataselection(self): - # Create testing object and prepare multi-index + # Create testing objects (regular and swapped dimords) dummy = SpectralData(data=self.data, trialdefinition=self.trl, samplerate=self.samplerate, taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)]) - idx = [slice(None)] * len(dummy.dimord) - timeIdx = dummy.dimord.index("time") - chanIdx = dummy.dimord.index("channel") - freqIdx = dummy.dimord.index("freq") - taperIdx = dummy.dimord.index("taper") - - for trialSel in trialSelections: - for chanSel in chanSelections: - for timeSel in timeSelections: - for freqSel in freqSelections: - for taperSel in taperSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["channels"] = chanSel - kwdict[timeSel[0]] = timeSel[1] - kwdict[freqSel[0]] = freqSel[1] - kwdict["tapers"] = taperSel - cfg = StructDict(kwdict) - # data selection via class-method + `Selector` instance for indexing - selected = dummy.selectdata(**kwdict) - time.sleep(0.05) - selector = Selector(dummy, kwdict) - idx[chanIdx] = selector.channel - idx[freqIdx] = selector.freq - idx[taperIdx] = selector.taper - for tk, trialno in enumerate(selector.trials): - idx[timeIdx] = selector.time[tk] - indexed = dummy.trials[trialno][idx[0], ...][:, idx[1], ...][:, :, idx[2], :][..., idx[3]] - assert np.array_equal(selected.trials[tk].squeeze(), - indexed.squeeze()) - cfg.data = dummy - cfg.out = SpectralData(dimord=SpectralData._defaultDimord) - # data selection via package function and `cfg`: ensure equality - selectdata(cfg) - assert np.array_equal(cfg.out.channel, selected.channel) - assert np.array_equal(cfg.out.freq, selected.freq) - assert np.array_equal(cfg.out.taper, selected.taper) - assert np.array_equal(cfg.out.data, selected.data) - time.sleep(0.05) + ymmud = SpectralData(data=np.transpose(self.data, [3, 2, 1, 0]), + trialdefinition=self.trl, + samplerate=self.samplerate, + taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)], + dimord=SpectralData._defaultDimord[::-1]) + + for obj in [dummy, ymmud]: + idx = [slice(None)] * len(obj.dimord) + timeIdx = obj.dimord.index("time") + chanIdx = obj.dimord.index("channel") + freqIdx = obj.dimord.index("freq") + taperIdx = obj.dimord.index("taper") + for trialSel in trialSelections: + for chanSel in chanSelections: + for timeSel in timeSelections: + for freqSel in freqSelections: + for taperSel in taperSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["channels"] = chanSel + kwdict[timeSel[0]] = timeSel[1] + kwdict[freqSel[0]] = freqSel[1] + kwdict["tapers"] = taperSel + cfg = StructDict(kwdict) + # data selection via class-method + `Selector` instance for indexing + selected = obj.selectdata(**kwdict) + time.sleep(0.05) + selector = Selector(obj, kwdict) + idx[chanIdx] = selector.channel + idx[freqIdx] = selector.freq + idx[taperIdx] = selector.taper + for tk, trialno in enumerate(selector.trials): + idx[timeIdx] = selector.time[tk] + indexed = obj.trials[trialno][idx[0], ...][:, idx[1], ...][:, :, idx[2], :][..., idx[3]] + assert np.array_equal(selected.trials[tk].squeeze(), + indexed.squeeze()) + cfg.data = obj + cfg.out = SpectralData(dimord=obj.dimord) + # data selection via package function and `cfg`: ensure equality + selectdata(cfg) + assert np.array_equal(cfg.out.channel, selected.channel) + assert np.array_equal(cfg.out.freq, selected.freq) + assert np.array_equal(cfg.out.taper, selected.taper) + assert np.array_equal(cfg.out.data, selected.data) + time.sleep(0.05) # test arithmetic operations def test_sd_arithmetic(self): @@ -811,11 +824,6 @@ def test_sd_arithmetic(self): samplerate=self.samplerate, taper=["TestTaper_0{}".format(k) for k in range(1, self.nt + 1)], dimord=SpectralData._defaultDimord[::-1]) - dummyArr = 2 * np.ones((dummy.trials[0].shape)) - ymmudArr = 2 * np.ones((ymmud.trials[0].shape)) - scalarOperands = [2, np.pi] - dummyOperands = [dummyArr, dummyArr.tolist()] - ymmudOperands = [ymmudArr, ymmudArr.tolist()] # Perform basic arithmetic with +, -, *, / and ** (pow) for operation in arithmetics: @@ -825,57 +833,7 @@ def test_sd_arithmetic(self): operation(dummy, ymmud) assert "expected Syncopy 'time' x 'channel' data object" in str(spyval.value) - # Next, ensure trial counts are properly vetted - dummy2.selectdata(trials=[0], inplace=True) - with pytest.raises(SPYValueError) as spyval: - operation(dummy, dummy2) - assert "Syncopy object with same number of trials (selected)" in str (spyval.value) - dummy2._selection = None - - # Scalar algebra must be commutative (except for pow) - for operand in scalarOperands: - result = operation(dummy, operand) # perform operation from right - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], operand)) - # Don't try to compute `2 ** data`` - if operation(2,3) != 8: - result2 = operation(operand, dummy) # perform operation from left - assert np.array_equal(result2.data, result.data) - - # Same as above, but swapped `dimord` - result = operation(ymmud, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) - if operation(2,3) != 8: - result2 = operation(operand, ymmud) - assert np.array_equal(result2.data, result.data) - - # Careful: NumPy tries to avoid failure by broadcasting; instead of relying - # on an existing `__radd__` method, it performs arithmetic component-wise, i.e., - # ``np.ones((3,3)) + data`` performs ``1 + data`` nine times, so don't - # test for left/right arithmetics... - for operand in dummyOperands: - result = operation(dummy, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], operand)) - for operand in ymmudOperands: - result = operation(ymmud, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], operand)) - - # Ensure erroneous object type-casting is prevented - with pytest.raises(SPYTypeError) as spytyp: - operation(dummy, dummyC) - assert "Syncopy data object of same numerical type (real/complex)" in str(spytyp.value) - - # Most severe safety hazard: throw two objects at each other (with regular and - # swapped dimord) - result = operation(dummy, dummy2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(dummy.trials[tk], dummy2.trials[tk])) - result = operation(ymmud, ymmud2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(ymmud.trials[tk], ymmud2.trials[tk])) + _base_op_tests(dummy, ymmud, dummy2, ymmud2, dummyC, operation) # Now the most complicated case: user-defined subset selections are present for trialSel in trialSelections: @@ -890,45 +848,7 @@ def test_sd_arithmetic(self): kwdict[freqSel[0]] = freqSel[1] kwdict["tapers"] = taperSel - # Perform in-place selection and construct array based on new subset - selected = dummy.selectdata(**kwdict) - dummy.selectdata(inplace=True, **kwdict) - arr = 2 * np.ones((selected.trials[0].shape), dtype=np.intp) - for operand in [np.pi, arr]: - result = operation(dummy, operand) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], operand)) - - # Most most complicated: subset selection present in base object - # and operand thrown at it: only attempt to do this if the selection - # is "well-behaved", i.e., is ordered and does not contain repetitions - # The operator code checks for this, so catch the corresponding - # `SpyValueError` and only attempt to test if coast is clear - dummy2.selectdata(inplace=True, **kwdict) - try: - result = operation(dummy, dummy2) - cleanSelection = True - except SPYValueError: - cleanSelection = False - # except: - # import pdb; pdb.set_trace() - if cleanSelection: - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], - selected.trials[tk])) - selected = ymmud.selectdata(**kwdict) - ymmud.selectdata(inplace=True, **kwdict) - ymmud2.selectdata(inplace=True, **kwdict) - result = operation(ymmud, ymmud2) - for tk, trl in enumerate(result.trials): - assert np.array_equal(trl, operation(selected.trials[tk], - selected.trials[tk])) - - # Very important: clear manually set selections for next iteration - dummy._selection = None - dummy2._selection = None - ymmud._selection = None - ymmud2._selection = None + _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) # Finally, perform a representative chained operation to ensure chaining works result = (dummy + dummy2) / dummy ** 3 diff --git a/syncopy/tests/test_discretedata.py b/syncopy/tests/test_discretedata.py index eb37714b2..6d745a8f6 100644 --- a/syncopy/tests/test_discretedata.py +++ b/syncopy/tests/test_discretedata.py @@ -149,9 +149,16 @@ def test_saveload(self): # test data-selection via class method def test_dataselection(self): + + # Create testing objects (regular and swapped dimords) dummy = SpikeData(data=self.data, trialdefinition=self.trl, samplerate=2.0) + ymmud = SpikeData(data=self.data[:, ::-1], + trialdefinition=self.trl, + samplerate=2.0, + dimord=dummy.dimord[::-1]) + # selections are chosen so that result is not empty trialSelections = [ "all", # enforce below selections in all trials of `dummy` @@ -180,40 +187,40 @@ def test_dataselection(self): timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) - chanIdx = dummy.dimord.index("channel") - unitIdx = dummy.dimord.index("unit") - chanArr = np.arange(dummy.channel.size) - - for trialSel in trialSelections: - for chanSel in chanSelections: - for unitSel in unitSelections: - for timeSel in timeSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["channels"] = chanSel - kwdict["units"] = unitSel - kwdict[timeSel[0]] = timeSel[1] - cfg = StructDict(kwdict) - # data selection via class-method + `Selector` instance for indexing - selected = dummy.selectdata(**kwdict) - selector = Selector(dummy, kwdict) - tk = 0 - for trialno in selector.trials: - if selector.time[tk]: - assert np.array_equal(dummy.trials[trialno][selector.time[tk], :], - selected.trials[tk]) - tk += 1 - assert set(selected.data[:, chanIdx]).issubset(chanArr[selector.channel]) - assert set(selected.channel) == set(dummy.channel[selector.channel]) - assert np.array_equal(selected.unit, - dummy.unit[np.unique(selected.data[:, unitIdx])]) - cfg.data = dummy - cfg.out = SpikeData(dimord=SpikeData._defaultDimord) - # data selection via package function and `cfg`: ensure equality - selectdata(cfg) - assert np.array_equal(cfg.out.channel, selected.channel) - assert np.array_equal(cfg.out.unit, selected.unit) - assert np.array_equal(cfg.out.data, selected.data) + for obj in [dummy, ymmud]: + chanIdx = obj.dimord.index("channel") + unitIdx = obj.dimord.index("unit") + chanArr = np.arange(obj.channel.size) + for trialSel in trialSelections: + for chanSel in chanSelections: + for unitSel in unitSelections: + for timeSel in timeSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["channels"] = chanSel + kwdict["units"] = unitSel + kwdict[timeSel[0]] = timeSel[1] + cfg = StructDict(kwdict) + # data selection via class-method + `Selector` instance for indexing + selected = obj.selectdata(**kwdict) + selector = Selector(obj, kwdict) + tk = 0 + for trialno in selector.trials: + if selector.time[tk]: + assert np.array_equal(obj.trials[trialno][selector.time[tk], :], + selected.trials[tk]) + tk += 1 + assert set(selected.data[:, chanIdx]).issubset(chanArr[selector.channel]) + assert set(selected.channel) == set(obj.channel[selector.channel]) + assert np.array_equal(selected.unit, + obj.unit[np.unique(selected.data[:, unitIdx])]) + cfg.data = obj + cfg.out = SpikeData(dimord=obj.dimord) + # data selection via package function and `cfg`: ensure equality + selectdata(cfg) + assert np.array_equal(cfg.out.channel, selected.channel) + assert np.array_equal(cfg.out.unit, selected.unit) + assert np.array_equal(cfg.out.data, selected.data) @skip_without_acme def test_parallel(self, testcluster): @@ -473,9 +480,16 @@ def test_ed_trialsetting(self): # test data-selection via class method def test_ed_dataselection(self): + + # Create testing objects (regular and swapped dimords) dummy = EventData(data=self.data, trialdefinition=self.trl, samplerate=2.0) + ymmud = EventData(data=self.data[:, ::-1], + trialdefinition=self.trl, + samplerate=2.0, + dimord=dummy.dimord[::-1]) + # selections are chosen so that result is not empty trialSelections = [ "all", # enforce below selections in all trials of `dummy` @@ -497,33 +511,33 @@ def test_ed_dataselection(self): timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) - eventidIdx = dummy.dimord.index("eventid") - - for trialSel in trialSelections: - for eventidSel in eventidSelections: - for timeSel in timeSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["eventids"] = eventidSel - kwdict[timeSel[0]] = timeSel[1] - cfg = StructDict(kwdict) - # data selection via class-method + `Selector` instance for indexing - selected = dummy.selectdata(**kwdict) - selector = Selector(dummy, kwdict) - tk = 0 - for trialno in selector.trials: - if selector.time[tk]: - assert np.array_equal(dummy.trials[trialno][selector.time[tk], :], - selected.trials[tk]) - tk += 1 - assert np.array_equal(selected.eventid, - dummy.eventid[np.unique(selected.data[:, eventidIdx]).astype(np.intp)]) - cfg.data = dummy - cfg.out = EventData(dimord=EventData._defaultDimord) - # data selection via package function and `cfg`: ensure equality - selectdata(cfg) - assert np.array_equal(cfg.out.eventid, selected.eventid) - assert np.array_equal(cfg.out.data, selected.data) + for obj in [dummy, ymmud]: + eventidIdx = obj.dimord.index("eventid") + for trialSel in trialSelections: + for eventidSel in eventidSelections: + for timeSel in timeSelections: + kwdict = {} + kwdict["trials"] = trialSel + kwdict["eventids"] = eventidSel + kwdict[timeSel[0]] = timeSel[1] + cfg = StructDict(kwdict) + # data selection via class-method + `Selector` instance for indexing + selected = obj.selectdata(**kwdict) + selector = Selector(obj, kwdict) + tk = 0 + for trialno in selector.trials: + if selector.time[tk]: + assert np.array_equal(obj.trials[trialno][selector.time[tk], :], + selected.trials[tk]) + tk += 1 + assert np.array_equal(selected.eventid, + obj.eventid[np.unique(selected.data[:, eventidIdx]).astype(np.intp)]) + cfg.data = obj + cfg.out = EventData(dimord=obj.dimord) + # data selection via package function and `cfg`: ensure equality + selectdata(cfg) + assert np.array_equal(cfg.out.eventid, selected.eventid) + assert np.array_equal(cfg.out.data, selected.data) @skip_without_acme def test_ed_parallel(self, testcluster): From bd7129ecf5d2e79dc2be5dcedb0c5a5f0f6890c7 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 24 Nov 2021 15:01:28 +0100 Subject: [PATCH 075/109] FIX: Do not allow DiscreteData arithmetics + wrapped up tests - performing arithmetic operations on `SpikeData` or `EventData` objects works mechanically but does not make sense (e.g., multiplying sample indices or task-ids...). Attempting to do so, now raises a `SPYTypeError` - basal functionality tests have been modified to account for this - finally closes #153 - wrote docstrings for all introduced functions On branch spy_arithmetic Changes to be committed: modified: syncopy/datatype/methods/arithmetic.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/methods/arithmetic.py | 188 ++++++++++++++++++++++++- syncopy/tests/test_basedata.py | 21 ++- 2 files changed, 201 insertions(+), 8 deletions(-) diff --git a/syncopy/datatype/methods/arithmetic.py b/syncopy/datatype/methods/arithmetic.py index fb9f2bf1b..f05178094 100644 --- a/syncopy/datatype/methods/arithmetic.py +++ b/syncopy/datatype/methods/arithmetic.py @@ -11,7 +11,7 @@ # Local imports from syncopy import __acme__ from syncopy.shared.parsers import data_parser -from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning, SPYParallelError +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYWarning from syncopy.shared.computational_routine import ComputationalRoutine from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.computational_routine import ComputationalRoutine @@ -20,15 +20,99 @@ __all__ = [] + +# Main entry point for overloaded operators def _process_operator(obj1, obj2, operator): """ - Coming soon... + Perform binary arithmetic operation on Syncopy data object + + Parameters + ---------- + obj1 : Syncopy data class or Python object + Depending on left/right application of arithmetic operator, `obj1` may be + either a Syncopy class or any Python object + obj2 : Syncopy data class or Python object + Depending on left/right application of arithmetic operator, `obj2` may be + either a Syncopy class or any Python object + operator : str + Operation to be performed encoded as string. Currently supported operators + are `'+'`, `'-'`, `'*'`, `'/'` and `'**'` (i.e., `'pow'`). + + Returns + ------- + res : Syncopy object + Result of arithmetic operation + + Notes + ----- + All arithmetic operations are performed on a per-trial basis. This means, + any data not covered by a Syncopy object's `trialdefinition` will not be + affected by the arithmetic operation. + Note further, that error checking is only performed on a very basic level, i.e., + the code ensures that instances of different classes are not mashed together + (e.g., ``AnalogData + SpectralData``) and that objects have compatible trial + counts and dtypes (no mixing of complex/real data). However, as long as trial + shapes align, it is possible to process objects w/diverging `samplerate`, + `channels`, `freqs` etc. The reason for this object parsing leniency is that + it might be interesting/necessary to manipulate objects arising from different + configurations (e.g., subtract channel `x` in `obj1` from channel `y` in `obj2`). + + See also + -------- + _parse_input : prepare objects for arithmetic operations + _perform_computation : execute arithmetic operation """ baseObj, operand, operand_dat, opres_type, operand_idxs = _parse_input(obj1, obj2, operator) return _perform_computation(baseObj, operand, operand_dat, operand_idxs, opres_type, operator) +# Error checking and input preparation def _parse_input(obj1, obj2, operator): + """ + Prepare objects for performing binary arithmetics + + Parameters + ---------- + obj1 : Syncopy data class or Python object + See :func:`_process_operator` for details. + obj2 : Syncopy data class or Python object + See :func:`_process_operator` for details. + operator : str + See :func:`_process_operator` for details. + + Returns + ------- + baseObj : Syncopy data object + The "base" object to perform arithmetics on. By default, the left object + is considered as base (if possible), i.e., in the expression ``data1 + data2``, + `data1` is defined as base object + operand : Syncopy data object, scalar or array-like + Term to perform arithmetic operation with. + operand_dat : dict or scalar or array-like + If `operand` is a scalar, list or NumPy ndarray then ``operand_dat == operand``. + If `operand` is a Syncopy object, then `operand_dat` is a dictionary with + keys `"filename"` (pointing to the HDF5 backing device of `operand`) and + `"dsetname"`(name of the corresponding dataset(s) of `operand`). + opres_type : dtype + Numerical type of the Syncopy object resulting from applying the arithmetic + operation. + operand_idxs : None or list + If `operand` is a scalar, list or NumPy ndarray then `operand_idxs` is + `None`. If `operand` is a Syncopy object, then `operand_idxs` is a list + containing the array indices of `operands` data(subset) for each (selected) + trial. + + Note + ---- + The distinction between `baseObj` and `operand` is not only syntactic sugar + but has consequences if both `baseObj` and `operand` are Syncopy objects: + the `baseObj` is allowed to come with any valid subset selection (may require + advanced indexing involving multiple slice/list combinations, might include + repetitions and be unordered). Conversely, the `operand` object can only + contain `simple` selections (no fancy indexing allowed, no repetitions or + unordered selections). This restriction simplifies the required HDF dataset + indexing considerably. + """ # Determine which input is a Syncopy object (depending on lef/right application of # operator, i.e., `data + 1` or `1 + data`). Can be both as well, but we just need @@ -40,6 +124,11 @@ def _parse_input(obj1, obj2, operator): baseObj = obj2 operand = obj1 + # Ensure base object is not discrete + if "DiscreteData" in str(baseObj.__class__.__mro__): + lgl = "`AnalogData`, `SpectralData` or `CrossSpectralData`" + raise SPYTypeError(baseObj, varname="base", expected=lgl) + # Ensure our base object is not empty try: data_parser(baseObj, varname="base", empty=False) @@ -185,9 +274,10 @@ def _parse_input(obj1, obj2, operator): return baseObj, operand, operand_dat, opres_type, operand_idxs +# Check for complexity in `operand` vs. `baseObj` def _check_complex_operand(baseTrials, operand, opDimType): """ - Coming soon... + Local helper to determine if provided scalar/array and `baseObj` are both real/complex """ # Ensure complex and real values are not mashed up @@ -202,6 +292,7 @@ def _check_complex_operand(baseTrials, operand, opDimType): return +# Invoke `ComputationalRoutine` to compute arithmetic operation def _perform_computation(baseObj, operand, operand_dat, @@ -209,7 +300,41 @@ def _perform_computation(baseObj, opres_type, operator): """ - Coming soon... + Leverage `ComputationalRoutine` to process arithmetic operation + + Parameters + ---------- + baseObj : Syncopy data object + See :func:`_parse_input` for details. + operand : Syncopy data object, scalar or array-like + See :func:`_parse_input` for details. + operand_dat : dict or scalar or array-like + See :func:`_parse_input` for details. + opres_type : dtype + See :func:`_parse_input` for details. + operator : str + See :func:`_process_operator` for details. + + Returns + ------- + out : Syncopy data object + Result of performing arithmetic operation on `baseObj` and `operand` + + Note + ---- + This method instantiates a subclass of + :class:`~syncopy.shared.computational_routine.ComputationalRoutine` + to perform arithmetic operations on Syncopy objects either sequentially or + in parallel. Note that due to this code being only invoked via operator + overloading the `@detect_parallel_client` decorator is *not* invoked, since + the user cannot supply any keyword arguments. Instead, the code scans for + running dask distributed computing clients (if ACME is available) and uses + concurrent processing if a client is found. + + See also + -------- + arithmetic_cF : `computeFunction` performing arithmetics + SpyArithmetic : :class:`~syncopy.shared.computational_routine.ComputationalRoutine` subclass """ # Prepare logging info in dictionary: we know that `baseObj` is definitely @@ -291,7 +416,49 @@ def _perform_computation(baseObj, def arithmetic_cF(base_dat, operand_dat, operand_idx, operation=None, opres_type=None, noCompute=False, chunkShape=None): """ - Coming soon... + Perform arithmetic operation + + Parameters + ---------- + base_dat : :class:`numpy.ndarray` + Trial data + operand_dat : dict or scalar or array-like + If two Syncopy objects are processed, then `operand_dat` is a dictionary + containing information about the operand's HDF5 backing device (see + :func:`_parse_input` for details). Otherwise, `operand_dat` is either a + scalar or array-like quantity. + operand_idx : tuple + If `operand` is a scalar, list or NumPy ndarray then `operand_idx` is + `None`. If `operand` is a Syncopy object, then `operand_idx` is an indexing + tuple. + operation : lambda object + A lambda expression encapsulating the requested arithmetic operation. + opres_type : dtype + Numerical type of applying ``operation(base_dat, operand)`` + noCompute : bool + Preprocessing flag. If `True`, do not perform actual calculation but + instead return expected shape and :class:`numpy.dtype` of output + array. + chunkShape : None or tuple + If not `None`, represents shape of output + + Returns + ------- + res : :class:`numpy.ndarray` + Result of ``operation(base_dat, operand)`` + + Notes + ----- + This method is intended to be used as :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + See also + -------- + _perform_computation : execute arithmetic operation + SpyArithmetic : :class:`~syncopy.shared.computational_routine.ComputationalRoutine` subclass """ if noCompute: @@ -306,6 +473,17 @@ def arithmetic_cF(base_dat, operand_dat, operand_idx, operation=None, opres_type return operation(base_dat, operand) class SpyArithmetic(ComputationalRoutine): + """ + Compute class for performing arithmetic operations with Syncopy objects + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + _perform_computation : execute arithmetic operation + """ computeFunction = staticmethod(arithmetic_cF) diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index 22da7f526..f67524a83 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -371,9 +371,24 @@ def test_copy(self): # Test basic error handling of arithmetic ops def test_arithmetic(self): - # test shallow copy of data arrays (hashes must match up, since - # shallow copies are views in memory) - for dclass in self.classes: + # Define list of classes arithmetic ops should and should not work with + # FIXME: include `CrossSpectralData` here! + # continuousClasses = ["AnalogData", "SpectralData", "CrossSpectralData"] + continuousClasses = ["AnalogData", "SpectralData"] + discreteClasses = ["SpikeData", "EventData"] + + # Illegal classes for arithmetics + for dclass in discreteClasses: + dummy = getattr(spd, dclass)(self.data[dclass], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + for operation in arithmetics: + with pytest.raises(SPYTypeError) as spytyp: + operation(dummy, 2) + assert "Wrong type of base: expected `AnalogData`, `SpectralData`" in str(spytyp.value) + + # Now, test basic error handling for allowed classes + for dclass in continuousClasses: dummy = getattr(spd, dclass)(self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) From 347dd65a155c1c1e649484923f8488c3acb2a71d Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 24 Nov 2021 19:34:22 +0100 Subject: [PATCH 076/109] WIP: Started implementing object padding - laid out skeleton of padding `computeFunction` and `ComputationalRoutine` subclass - to not make things overly complicated, any existing selections are ignored (a corresponding warning message is shown) On branch padding Changes to be committed: modified: syncopy/datatype/methods/padding.py modified: syncopy/tests/test_continuousdata.py --- syncopy/datatype/methods/padding.py | 53 +++++++++++++++++++++++++--- syncopy/tests/test_continuousdata.py | 13 ++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/syncopy/datatype/methods/padding.py b/syncopy/datatype/methods/padding.py index d8cf97480..a570172ad 100644 --- a/syncopy/datatype/methods/padding.py +++ b/syncopy/datatype/methods/padding.py @@ -7,6 +7,9 @@ import numpy as np # Local imports +from syncopy.datatype.continuous_data import AnalogData +from syncopy.shared.computational_routine import ComputationalRoutine +from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.parsers import data_parser, array_parser, scalar_parser from syncopy.shared.errors import SPYTypeError, SPYValueError, SPYWarning @@ -285,11 +288,14 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, timeAxis = 0 spydata = False - # FIXME: Creation of new spy-object currently not supported + # Any existing in-place selections will be ignored + if data._selection is not None: + wrng = "Existing in-place selection{} will be ignored for padding." + SPYWarning(wrng.format(data._selection.__str__().partition("with")[-1])) + + # Ensure `create_new` is not weird if not isinstance(create_new, bool): raise SPYTypeError(create_new, varname="create_new", expected="bool") - if spydata and create_new: - raise NotImplementedError("Creation of padded spy objects currently not supported. ") # Use FT-compatible options (sans FT option 'remove') if not isinstance(padtype, str): @@ -441,7 +447,7 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, "edge": {"mode": "edge"}, "mirror": {"mode": "reflect"}} - # If in put was syncopy data object, padding is done on a per-trial basis + # If input was syncopy data object, padding is done on a per-trial basis if spydata: # A list of input keywords for ``np.pad`` is constructed, no matter if @@ -464,6 +470,7 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, pad_opts[-1]["stat_length"] = pw[timeAxis, :] if create_new: + out = AnalogData(dimord=data.dimord) pass else: return pad_opts @@ -516,3 +523,41 @@ def _nextpow2(number): while n < number: n *= 2 return n + + +@unwrap_io +def padding_cF(trl_dat, timeAxis=0, pad_opt, noCompute=False, chunkShape=None): + """ + Coming Soon + """ + + # Re-arrange array if necessary and get dimensional information + if timeAxis != 0: + dat = trl_dat.T # does not copy but creates view of `trl_dat` + else: + dat = trl_dat + + if noCompute: + return base_dat.shape, opres_type + + # Symmetric Padding (updates no. of samples) + return np.pad(dat, **pad_opt) + +class PaddingRoutine(ComputationalRoutine): + + computeFunction = staticmethod(padding_cF) + + def process_metadata(self, baseObj, out): + + # Get/set timing-related selection modifiers + out.trialdefinition = baseObj._selection.trialdefinition + # if baseObj._selection._timeShuffle: # FIXME: should be implemented done the road + # out.time = baseObj._selection.timepoints + if baseObj._selection._samplerate: + out.samplerate = baseObj.samplerate + + # Get/set dimensional attributes changed by selection + for prop in baseObj._selection._dimProps: + selection = getattr(baseObj._selection, prop) + if selection is not None: + setattr(out, prop, getattr(baseObj, prop)[selection]) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index b59b0bb95..f61342167 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -631,12 +631,11 @@ def test_ang_arithmetic(self): def test_parallel(self, testcluster): # repeat selected test w/parallel processing engine client = dd.Client(testcluster) - par_tests = ["test_ang_arithmetic"] - # par_tests = ["test_relative_array_padding", - # "test_absolute_nextpow2_array_padding", - # "test_object_padding", - # "test_dataselection", - # "test_ang_arithmetic"] + par_tests = ["test_relative_array_padding", + "test_absolute_nextpow2_array_padding", + "test_object_padding", + "test_dataselection", + "test_ang_arithmetic"] for test in par_tests: getattr(self, test)() flush_local_cluster(testcluster) @@ -860,7 +859,7 @@ def test_sd_arithmetic(self): def test_sd_parallel(self, testcluster): # repeat selected test w/parallel processing engine client = dd.Client(testcluster) - par_tests = ["test_sd_dataselection"] + par_tests = ["test_sd_dataselection", "test_sd_arithmetic"] for test in par_tests: getattr(self, test)() flush_local_cluster(testcluster) From ec6ee20d57db2be2ea4739a49e35bef5441a4692 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 25 Nov 2021 12:47:57 +0100 Subject: [PATCH 077/109] NEW : Capture frontend parameters which have no effect (**kwargs) - new fontend input validation with 'check_passed_kwargs', if the user submitts sth like 'freqanalysis(data, ..., foo=5)` warns that 'foo' has no effect - added CR effective paramter checks for connectivity --- dev_frontend.py | 36 +++++------ syncopy/connectivity/ST_compRoutines.py | 13 ++-- syncopy/connectivity/__init__.py | 2 +- syncopy/connectivity/connectivity_analysis.py | 37 ++++++------ syncopy/shared/const_def.py | 2 +- syncopy/shared/input_validators.py | 59 ++++++++++++++++++- syncopy/specest/compRoutines.py | 15 ++--- syncopy/specest/freqanalysis.py | 52 ++++------------ 8 files changed, 121 insertions(+), 95 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 2561f42b9..dfa1118c6 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -4,7 +4,7 @@ from syncopy.datatype import CrossSpectralData, padding, SpectralData, AnalogData from syncopy.connectivity.ST_compRoutines import cross_spectra_cF, ST_CrossSpectra from syncopy.connectivity.ST_compRoutines import cross_covariance_cF -from syncopy.connectivity import connectivityanalysis +from syncopy.connectivity import connectivity from syncopy.specest import freqanalysis import matplotlib.pyplot as ppl @@ -20,29 +20,29 @@ sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} # no problems here.. -coherence = connectivityanalysis(data=tdat, - foilim=None, - output='pow', - taper='dpss', - tapsmofrq=5, - keeptrials=False) - -granger = connectivityanalysis(data=tdat, - method='granger', - foilim=[0, 50], - output='pow', - taper='dpss', - tapsmofrq=5, - keeptrials=False) +coherence = connectivity(data=tdat, + foilim=None, + output='pow', + taper='dpss', + tapsmofrq=5, + foo = 3, # non-sensical + keeptrials=False) + +granger = connectivity(data=tdat, + method='granger', + foilim=[0, 50], + output='pow', + taper='dpss', + tapsmofrq=5, + keeptrials=False) # D = SpectralData(dimord=['freq','test1','test2','taper']) # D2 = AnalogData(dimord=['freq','test1']) # a lot of problems here.. -# correlation = connectivityanalysis(data=tdat, method='corr', keeptrials=False) +# correlation = connectivity(data=tdat, method='corr', keeptrials=False, taper='df') # the hard wired dimord of the cF -dimord = ['None', 'freq', 'channel_i', 'channel_j'] res = freqanalysis(data=tdat, method='mtmfft', @@ -55,7 +55,7 @@ t_ftimwin=0.5, keeptrials=True, taper='dpss', - nTaper = 11, + nTaper = 19, tapsmofrq=5, keeptapers=True, parallel=False, # try this!!!!!! diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 0b467bfda..be89813d5 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -212,11 +212,13 @@ class ST_CrossSpectra(ComputationalRoutine): computeFunction = staticmethod(cross_spectra_cF) - method = "cross_spectra" - # 1st argument,the data, gets omitted - method_keys = list(signature(cross_spectra_cF).parameters.keys())[1:] - cF_keys = list(signature(cross_spectra_cF).parameters.keys())[1:] - + backends = [mtmfft] + # 1st argument,the data, gets omitted + valid_kws = list(signature(mtmfft).parameters.keys())[1:] + valid_kws += list(signature(cross_spectra_cF).parameters.keys())[1:] + # hardcode some parameter names which got digested from the frontend + valid_kws += ['tapsmofrq', 'nTaper'] + def process_metadata(self, data, out): # Some index gymnastics to get trial begin/end "samples" @@ -395,7 +397,6 @@ class ST_CrossCovariance(ComputationalRoutine): computeFunction = staticmethod(cross_covariance_cF) - method = "" # there is no backend # 1st argument,the data, gets omitted valid_kws = list(signature(cross_covariance_cF).parameters.keys())[1:] diff --git a/syncopy/connectivity/__init__.py b/syncopy/connectivity/__init__.py index d5521f3de..b77eb5f05 100644 --- a/syncopy/connectivity/__init__.py +++ b/syncopy/connectivity/__init__.py @@ -4,7 +4,7 @@ # connectivity methods # -from .connectivity_analysis import connectivityanalysis +from .connectivity_analysis import connectivity from .connectivity_analysis import __all__ as _all_ # Populate local __all__ namespace diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 50c2edeb8..d173878a7 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -20,28 +20,28 @@ from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) from syncopy.shared.tools import best_match -from syncopy.shared.input_validators import validate_taper, validate_foi -from syncopy.shared.const_def import ( - spectralConversions, - availableTapers, - generalParameters +from syncopy.shared.input_validators import ( + validate_taper, + validate_foi, + check_effective_parameters, + check_passed_kwargs ) from .ST_compRoutines import ST_CrossSpectra, ST_CrossCovariance from .AV_compRoutines import NormalizeCrossSpectra, NormalizeCrossCov, GrangerCausality -__all__ = ["connectivityanalysis"] +__all__ = ["connectivity"] availableMethods = ("coh", "corr", "granger") @unwrap_cfg @unwrap_select @detect_parallel_client -def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", - foi=None, foilim=None, pad_to_length=None, - polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", rtol=1e-7, nIter=100, cond_max=1e6, - out=None, **kwargs): +def connectivity(data, method="coh", keeptrials=False, output="abs", + foi=None, foilim=None, pad_to_length=None, + polyremoval=None, taper="hann", tapsmofrq=None, + nTaper=None, toi="all", + out=None, **kwargs): """ coming soon.. @@ -56,9 +56,10 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", timeAxis = data.dimord.index("time") # Get everything of interest in local namespace - defaults = get_defaults(connectivityanalysis) + defaults = get_defaults(connectivity) lcls = locals() - + check_passed_kwargs(lcls, defaults, "connectivity") + # Ensure a valid computational method was selected if method not in availableMethods: lgl = "'" + "or '".join(opt + "' " for opt in availableMethods) @@ -163,6 +164,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", nSamples=nSamples, output="pow") # ST_CSD's always have this unit/norm + check_effective_parameters(ST_CrossSpectra, defaults, lcls) # parallel computation over trials st_compRoutine = ST_CrossSpectra(samplerate=data.samplerate, padding_opt=padding_opt, @@ -180,13 +182,14 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", if method == 'granger': # after trial averaging - av_compRoutine = GrangerCausality(rtol=rtol, - nIter=nIter, - cond_max=cond_max + # hardcoded numerical parameters + av_compRoutine = GrangerCausality(rtol=1e-8, + nIter=100, + cond_max=1e5 ) if method == 'corr': - + check_effective_parameters(ST_CrossCovariance, defaults, lcls) # parallel computation over trials st_compRoutine = ST_CrossCovariance(samplerate=data.samplerate, padding_opt=padding_opt, diff --git a/syncopy/shared/const_def.py b/syncopy/shared/const_def.py index 2033be140..dfecad7de 100644 --- a/syncopy/shared/const_def.py +++ b/syncopy/shared/const_def.py @@ -25,5 +25,5 @@ availableTapers = all_windows #: general, method agnostic, parameters of :func:`~syncopy.freqanalysis` -generalParameters = ("method", "output", "keeptrials", +generalParameters = ("method", "output", "keeptrials","samplerate", "foi", "foilim", "polyremoval", "out") diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index c96b01538..a511e1f43 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -10,7 +10,7 @@ from syncopy.shared.errors import SPYValueError, SPYWarning, SPYInfo from syncopy.shared.parsers import data_parser, scalar_parser, array_parser -from syncopy.shared.const_def import availableTapers +from syncopy.shared.const_def import availableTapers, generalParameters def validate_foi(foi, foilim, samplerate): @@ -156,9 +156,10 @@ def validate_taper(taper, # Set/get `tapsmofrq` if we're working w/Slepian tapers elif taper == "dpss": - # minimal smoothing bandwidth in Hz - # if sampling rate is given in Hz + # --- minimal smoothing bandwidth --- + # --- such that Kmax/nTaper is at least 1 minBw = 2 * samplerate / nSamples + # ----------------------------------- # Try to derive "sane" settings by using 3/4 octave # smoothing of highest `foi` @@ -218,3 +219,55 @@ def validate_taper(taper, dpss_opt = {'NW' : NW, 'Kmax' : nTaper} return dpss_opt + + +def check_effective_parameters(CR, defaults, lcls): + + ''' + For a given ComputationalRoutine, compare set parameters + (*lcls*) with the accepted parameters and the *defaults* + to warn if any ineffective parameters are set. + + Parameters + ---------- + + CR : :class:`~syncopy.shared.computational_routine.ComputationalRoutine + Needs to have a `valid_kws` attribute + defaults : dict + Result of :func:`~syncopy.shared.tools.get_defaults`, the frontend + parameter names plus values with default values + lcls : dict + Result of `locals()`, all names and values of the local name space + ''' + # list of possible parameter names of the CR + expected = CR.valid_kws + ["parallel", "select"] + relevant = [name for name in defaults if name not in generalParameters] + for name in relevant: + if name not in expected and (lcls[name] != defaults[name]): + msg = f"option `{name}` has no effect in method `{CR.__name__}`!" + SPYWarning(msg, caller=__name__.split('.')[-1]) + + +def check_passed_kwargs(lcls, defaults, frontend_name): + + ''' + Catch additional kwargs passed to the frontends + which have no effect + ''' + + # unpack **kwargs of frontend call which + # might contain arbitrary kws passed from the user + kw_dict = lcls.get("kwargs") + + # nothing to do.. + if not kw_dict: + return + + relevant = list(kw_dict.keys()) + expected = [name for name in defaults] + + for name in relevant: + if name not in expected: + msg = f"option `{name}` has no effect in `{frontend_name}`!" + SPYWarning(msg, caller=__name__.split('.')[-1]) + diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index ab7a4d3e3..19794b50c 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -183,10 +183,9 @@ class MultiTaperFFT(ComputationalRoutine): computeFunction = staticmethod(mtmfft_cF) - method = "mtmfft" # 1st argument,the data, gets omitted - valid_kws = list(signature(mtmfft_cF).parameters.keys())[1:] - valid_kws += list(signature(mtmfft).parameters.keys())[1:] + valid_kws = list(signature(mtmfft).parameters.keys())[1:] + valid_kws += list(signature(mtmfft_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend valid_kws += ['tapsmofrq', 'nTaper'] @@ -416,9 +415,10 @@ class MultiTaperFFTConvol(ComputationalRoutine): """ computeFunction = staticmethod(mtmconvol_cF) + # 1st argument,the data, gets omitted - valid_kws = list(signature(mtmconvol_cF).parameters.keys())[1:] - valid_kws += list(signature(mtmconvol).parameters.keys())[1:] + valid_kws = list(signature(mtmconvol).parameters.keys())[1:] + valid_kws += list(signature(mtmconvol_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend valid_kws += ['tapsmofrq', 't_ftimwin', 'nTaper'] @@ -586,7 +586,6 @@ class WaveletTransform(ComputationalRoutine): computeFunction = staticmethod(wavelet_cF) - method = "wavelet" # 1st argument,the data, gets omitted valid_kws = list(signature(wavelet).parameters.keys())[1:] # here also last argument, the method_kwargs, are omitted @@ -752,9 +751,7 @@ class SuperletTransform(ComputationalRoutine): computeFunction = staticmethod(superlet_cF) - method = "superlet" - # 1st argument,the data, gets omitted - + # 1st argument,the data, gets omitted valid_kws = list(signature(superlet).parameters.keys())[1:] valid_kws += list(signature(superlet_cF).parameters.keys())[1:-1] diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 1b9fcccb4..e42f59e97 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -16,13 +16,14 @@ from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) from syncopy.shared.tools import best_match -from syncopy.shared.const_def import ( - spectralConversions, - availableTapers, - generalParameters -) +from syncopy.shared.const_def import spectralConversions -from syncopy.shared.input_validators import validate_taper, validate_foi +from syncopy.shared.input_validators import ( + validate_taper, + validate_foi, + check_effective_parameters, + check_passed_kwargs +) # method specific imports - they should go! import syncopy.specest.wavelets as spywave @@ -325,6 +326,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Get everything of interest in local namespace defaults = get_defaults(freqanalysis) lcls = locals() + check_passed_kwargs(lcls, defaults, "freqanalysis") # Ensure a valid computational method was selected if method not in availableMethods: @@ -574,7 +576,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', if method == "mtmfft": - _check_effective_parameters(MultiTaperFFT, defaults, lcls) + check_effective_parameters(MultiTaperFFT, defaults, lcls) # method specific parameters method_kwargs = { @@ -597,7 +599,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', elif method == "mtmconvol": - _check_effective_parameters(MultiTaperFFTConvol, defaults, lcls) + check_effective_parameters(MultiTaperFFTConvol, defaults, lcls) # Process `toi` for sliding window multi taper fft, # we have to account for three scenarios: (1) center sliding @@ -749,7 +751,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', elif method == "wavelet": - _check_effective_parameters(WaveletTransform, defaults, lcls) + check_effective_parameters(WaveletTransform, defaults, lcls) # Check wavelet selection if wavelet not in availableWavelets: @@ -834,7 +836,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', elif method == "superlet": - _check_effective_parameters(SuperletTransform, defaults, lcls) + check_effective_parameters(SuperletTransform, defaults, lcls) # check and parse superlet specific arguments if order_max is None: @@ -936,33 +938,3 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Either return newly created output object or simply quit return out if new_out else None - - -def _check_effective_parameters(CR, defaults, lcls): - - ''' - For a given ComputationalRoutine, compare set parameters - (*lcls*) with the accepted parameters and the *defaults* - to warn if any ineffective parameters are set. - - #FIXME: If general structure of this function proofs - useful for all CRs/syncopy in general, - probably best to move this to syncopy.shared.tools - - Parameters - ---------- - - CR : :class:`~syncopy.shared.computational_routine.ComputationalRoutine - defaults : dict - Result of :func:`~syncopy.shared.tools.get_defaults`, the function - parameter names plus values with default values - lcls : dict - Result of `locals()`, all names and values of the local name space - ''' - # list of possible parameter names of the CR - expected = CR.valid_kws + ["parallel", "select"] - relevant = [name for name in defaults if name not in generalParameters] - for name in relevant: - if name not in expected and (lcls[name] != defaults[name]): - msg = f"option `{name}` has no effect in method `{CR.method}`!" - SPYWarning(msg, caller=__name__.split('.')[-1]) From 6a2220bff909f8f0bbecc76660d02a8bb48a2288 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 25 Nov 2021 16:19:48 +0100 Subject: [PATCH 078/109] WIP: Non-functional code; more work on object padding - change of heart: allow data selection since the required extra work is actually minimal - getting the output object's trial definition right is the actual challenge, the rest is already done (leveraging NumPy's `pad` function) On branch padding Changes to be committed: modified: syncopy/datatype/methods/padding.py modified: syncopy/tests/spy_setup.py modified: syncopy/tests/test_continuousdata.py --- syncopy/datatype/methods/padding.py | 96 +++++++++++++++++++--------- syncopy/tests/spy_setup.py | 2 + syncopy/tests/test_continuousdata.py | 5 ++ 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/syncopy/datatype/methods/padding.py b/syncopy/datatype/methods/padding.py index a570172ad..c030570e0 100644 --- a/syncopy/datatype/methods/padding.py +++ b/syncopy/datatype/methods/padding.py @@ -12,12 +12,16 @@ from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.parsers import data_parser, array_parser, scalar_parser from syncopy.shared.errors import SPYTypeError, SPYValueError, SPYWarning +from syncopy.shared.kwarg_decorators import unwrap_cfg, unwrap_select, detect_parallel_client __all__ = ["padding"] +@unwrap_cfg +@unwrap_select +@detect_parallel_client def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, - postpadlength=None, unit="samples", create_new=True): + postpadlength=None, unit="samples", create_new=True, **kwargs): """ Perform data padding on Syncopy object or :class:`numpy.ndarray` @@ -288,10 +292,20 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, timeAxis = 0 spydata = False - # Any existing in-place selections will be ignored - if data._selection is not None: - wrng = "Existing in-place selection{} will be ignored for padding." - SPYWarning(wrng.format(data._selection.__str__().partition("with")[-1])) + if spydata: + if data._selection is not None: + trialList = data._selection.trials + data._pad_sinfo = np.zeros((len(trialList), 2)) + for tk, trlno in enumerate(trialList): + trl = data._preview_trial(trlno) + tsel = trl.idx[timeAxis] + if isinstance(tsel, list): + data._pad_sinfo[tk, :] = [0, len(tsel)] + else: + data._pad_sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] + else: + trialList = list(range(len(data.trials))) + data._pad_sinfo = data.sampleinfo # Ensure `create_new` is not weird if not isinstance(create_new, bool): @@ -335,7 +349,7 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, # trials, compute lower bound for padding (in samples or seconds) if pad in ["absolute", "maxlen"]: if spydata: - maxTrialLen = np.diff(data.sampleinfo).max() + maxTrialLen = np.diff(data._pad_sinfo).max() else: maxTrialLen = data.shape[timeAxis] # if `pad="absolute" and data is array else: @@ -453,8 +467,8 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, # A list of input keywords for ``np.pad`` is constructed, no matter if # we actually want to build a new object or not pad_opts = [] - for trl in data.trials: - nSamples = trl.shape[timeAxis] + for tk in trialList: + nSamples = data._preview_trial(tk).shape[timeAxis] if pad == "absolute": padding = (padlength - nSamples)/(prepadlength + postpadlength) elif pad == "relative": @@ -471,7 +485,21 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, if create_new: out = AnalogData(dimord=data.dimord) - pass + log_dct = {"padtype": padtype, + "pad": pad, + "padlength": padlength, + "prepadlength": prepadlength, + "postpadlength": postpadlength, + "unit": unit} + + chanAxis = list(set([0, 1]).difference([timeAxis]))[0] + padMethod = PaddingRoutine(timeAxis, chanAxis, pad_opts) + padMethod.initialize(data, + out._stackingDim, + chan_per_worker=kwargs.get("chan_per_worker"), + keeptrials=True) + padMethod.compute(data, out, parallel=kwargs.get("parallel"), log_dict=log_dct) + return out else: return pad_opts @@ -526,38 +554,48 @@ def _nextpow2(number): @unwrap_io -def padding_cF(trl_dat, timeAxis=0, pad_opt, noCompute=False, chunkShape=None): +def padding_cF(trl_dat, timeAxis, chanAxis, pad_opt, noCompute=False, chunkShape=None): """ Coming Soon """ - # Re-arrange array if necessary and get dimensional information - if timeAxis != 0: - dat = trl_dat.T # does not copy but creates view of `trl_dat` - else: - dat = trl_dat + nSamples = trl_dat.shape[timeAxis] + nChannels = trl_dat.shape[chanAxis] if noCompute: - return base_dat.shape, opres_type + outShape = [None] * 2 + outShape[timeAxis] = pad_opt['pad_width'].sum() + nSamples + outShape[chanAxis] = nChannels + return outShape, trl_dat.dtype # Symmetric Padding (updates no. of samples) - return np.pad(dat, **pad_opt) + return np.pad(trl_dat, **pad_opt) class PaddingRoutine(ComputationalRoutine): computeFunction = staticmethod(padding_cF) - def process_metadata(self, baseObj, out): + def process_metadata(self, data, out): + + + pad_opts = self.argv[2] + import pdb; pdb.set_trace() + + ss = [pad_opt["pad_width"].sum() for pad_opt in pad_opts] + np.diff(sinfo).squeeze() + ss + t0 = ss - t0 + + delattr(data, "_pad_sinfo") - # Get/set timing-related selection modifiers - out.trialdefinition = baseObj._selection.trialdefinition - # if baseObj._selection._timeShuffle: # FIXME: should be implemented done the road - # out.time = baseObj._selection.timepoints - if baseObj._selection._samplerate: - out.samplerate = baseObj.samplerate + # # Get/set timing-related selection modifiers + # out.trialdefinition = baseObj._selection.trialdefinition + # # if baseObj._selection._timeShuffle: # FIXME: should be implemented done the road + # # out.time = baseObj._selection.timepoints + # if baseObj._selection._samplerate: + # out.samplerate = baseObj.samplerate - # Get/set dimensional attributes changed by selection - for prop in baseObj._selection._dimProps: - selection = getattr(baseObj._selection, prop) - if selection is not None: - setattr(out, prop, getattr(baseObj, prop)[selection]) + # # Get/set dimensional attributes changed by selection + # for prop in baseObj._selection._dimProps: + # selection = getattr(baseObj._selection, prop) + # if selection is not None: + # setattr(out, prop, getattr(baseObj, prop)[selection]) diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 774beea15..2830ffe86 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -25,6 +25,8 @@ # Test stuff within here... data1 = generate_artificial_data(nTrials=5, nChannels=16, equidistant=False, inmemory=False) data2 = generate_artificial_data(nTrials=5, nChannels=16, equidistant=True, inmemory=False) + + # client = spy.esi_cluster_setup(interactive=False) # data1 + data2 diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index f61342167..2f2cc960f 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -489,6 +489,11 @@ def test_object_padding(self): total_time = 30 pad_list = padding(adata, "zero", pad="absolute", padlength=total_time, unit="time", create_new=False) + + res = padding(adata, "zero", pad="absolute", padlength=total_time,unit="time", create_new=True) + + import pdb; pdb.set_trace() + for tk, trl in enumerate(adata.trials): assert "pad_width" in pad_list[tk].keys() assert "constant_values" in pad_list[tk].keys() From 6f07356e5e1a9a6cbc62369fcc99ed4bd0a03e86 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 26 Nov 2021 12:35:18 +0100 Subject: [PATCH 079/109] NEW: Padding of AnalogData objects - included docstrings for padding computeFunction and corresponding ComputationalRoutine subclass - incorporated data-selection support: allow basic selections (no time-point subset, since that defeats the purpose of padding - toilim has been left intact for the time being) - extended (already existing) tests to cover actual padded object generation functionality - closes #76 On branch padding Changes to be committed: modified: syncopy/datatype/methods/padding.py modified: syncopy/tests/test_continuousdata.py --- syncopy/datatype/methods/padding.py | 105 +++++++++++++++++++++------ syncopy/tests/test_continuousdata.py | 79 +++++++++++++++++--- 2 files changed, 149 insertions(+), 35 deletions(-) diff --git a/syncopy/datatype/methods/padding.py b/syncopy/datatype/methods/padding.py index c030570e0..0984a33c3 100644 --- a/syncopy/datatype/methods/padding.py +++ b/syncopy/datatype/methods/padding.py @@ -292,20 +292,29 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, timeAxis = 0 spydata = False + # If input is a syncopy object, fetch trial list and `sampleinfo` (thereby + # accounting for in-place selections); to not repeat this later, save relevant + # quantities in tmp attributes (all prefixed by `'_pad'`) if spydata: if data._selection is not None: trialList = data._selection.trials data._pad_sinfo = np.zeros((len(trialList), 2)) + data._pad_t0 = np.zeros((len(trialList),)) for tk, trlno in enumerate(trialList): trl = data._preview_trial(trlno) tsel = trl.idx[timeAxis] if isinstance(tsel, list): - data._pad_sinfo[tk, :] = [0, len(tsel)] + lgl = "Syncopy AnalogData object with no or channe/trial selection" + raise SPYValueError(lgl, varname="data", actual=data._selection) else: data._pad_sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] + data._pad_t0[tk] = data._t0[trlno] + data._pad_channel = data.channel[data._selection.channel] else: trialList = list(range(len(data.trials))) data._pad_sinfo = data.sampleinfo + data._pad_t0 = data._t0 + data._pad_channel = data.channel # Ensure `create_new` is not weird if not isinstance(create_new, bool): @@ -483,6 +492,8 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, if padtype == "localmean": pad_opts[-1]["stat_length"] = pw[timeAxis, :] + # If a new object is requested, use the legwork performed above to fire + # up the corresponding ComputationalRoutine if create_new: out = AnalogData(dimord=data.dimord) log_dct = {"padtype": padtype, @@ -531,7 +542,7 @@ def padding(data, padtype, pad="absolute", padlength=None, prepadlength=None, if create_new: if isinstance(data, np.ndarray): return np.pad(data, **pad_opts) - else: # FIXME: currently only supports FauxTrial + else: shp = list(data.shape) shp[timeAxis] += pw[timeAxis, :].sum() idx = list(data.idx) @@ -556,7 +567,43 @@ def _nextpow2(number): @unwrap_io def padding_cF(trl_dat, timeAxis, chanAxis, pad_opt, noCompute=False, chunkShape=None): """ - Coming Soon + Perform trial data padding + + Parameters + ---------- + trl_dat : :class:`numpy.ndarray` + Trial data + timeAxis : int + Index of running time axis in `trl_dat` (0 or 1) + chanAxis : int + Index of channel axis in `trl_dat` (0 or 1) + pad_opt : dict + Dictionary of options for :func:`numpy.pad` + noCompute : bool + Preprocessing flag. If `True`, do not perform actual padding but + instead return expected shape and :class:`numpy.dtype` of output + array. + chunkShape : None or tuple + If not `None`, represents shape of output + + Returns + ------- + res : :class:`numpy.ndarray` + Padded array + + Notes + ----- + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. + Thus, input parameters are presumed to be forwarded from a parent metafunction. + Consequently, this function does **not** perform any error checking and operates + under the assumption that all inputs have been externally validated and cross-checked. + + See also + -------- + syncopy.padding : pad :class:`syncopy.AnalogData` objects + PaddingRoutine : :class:`~syncopy.shared.computational_routine.ComputationalRoutine` subclass """ nSamples = trl_dat.shape[timeAxis] @@ -572,30 +619,42 @@ def padding_cF(trl_dat, timeAxis, chanAxis, pad_opt, noCompute=False, chunkShape return np.pad(trl_dat, **pad_opt) class PaddingRoutine(ComputationalRoutine): + """ + Compute class for performing data padding on Syncopy AnalogData objects + + Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, + see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute + classes and metafunctions. + + See also + -------- + syncopy.padding : pad :class:`syncopy.AnalogData` objects + """ computeFunction = staticmethod(padding_cF) def process_metadata(self, data, out): - + # Fetch index of running time and used padding options from provided + # positional args and use them to compute new start/stop/trigger onset samples + timeAxis = self.argv[0] pad_opts = self.argv[2] - import pdb; pdb.set_trace() - - ss = [pad_opt["pad_width"].sum() for pad_opt in pad_opts] - np.diff(sinfo).squeeze() + ss - t0 = ss - t0 - + prePadded = [pad_opt["pad_width"][timeAxis, 0] for pad_opt in pad_opts] + totalPadded = [pad_opt["pad_width"].sum() for pad_opt in pad_opts] + accumSamples = np.cumsum(np.diff(data._pad_sinfo).squeeze() + totalPadded) + + # Construct trialdefinition array (columns: start/stop/t0/etc) + trialdefinition = np.zeros((len(totalPadded), data.trialdefinition.shape[1])) + trialdefinition[1:, 0] = accumSamples[:-1] + trialdefinition[:, 1] = accumSamples + trialdefinition[:, 2] = data._pad_t0 - prePadded + + # Set relevant properties in output object + out.samplerate = data.samplerate + out.trialdefinition = trialdefinition + out.channel = data._pad_channel + + # Remove inpromptu attributes generated above delattr(data, "_pad_sinfo") - - # # Get/set timing-related selection modifiers - # out.trialdefinition = baseObj._selection.trialdefinition - # # if baseObj._selection._timeShuffle: # FIXME: should be implemented done the road - # # out.time = baseObj._selection.timepoints - # if baseObj._selection._samplerate: - # out.samplerate = baseObj.samplerate - - # # Get/set dimensional attributes changed by selection - # for prop in baseObj._selection._dimProps: - # selection = getattr(baseObj._selection, prop) - # if selection is not None: - # setattr(out, prop, getattr(baseObj, prop)[selection]) + delattr(data, "_pad_t0") + delattr(data, "_pad_channel") diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 2f2cc960f..427653b3f 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -483,32 +483,76 @@ def test_object_padding(self): adata = generate_artificial_data(nTrials=7, nChannels=16, equidistant=False, inmemory=False) timeAxis = adata.dimord.index("time") + chanAxis = adata.dimord.index("channel") + + # Define trial/channel selections for tests + trialSel = [0, 2, 1] + chanSel = range(4) # test dictionary generation for `create_new = False`: ensure all trials # have padded length of `total_time` seconds (1 sample tolerance) total_time = 30 pad_list = padding(adata, "zero", pad="absolute", padlength=total_time, unit="time", create_new=False) - - res = padding(adata, "zero", pad="absolute", padlength=total_time,unit="time", create_new=True) - - import pdb; pdb.set_trace() - for tk, trl in enumerate(adata.trials): assert "pad_width" in pad_list[tk].keys() assert "constant_values" in pad_list[tk].keys() trl_time = (pad_list[tk]["pad_width"][timeAxis, :].sum() + trl.shape[timeAxis]) / adata.samplerate - assert trl_time - total_time < 1/adata.samplerate + assert trl_time - total_time < 1 / adata.samplerate + + # real thing: pad object with standing channel selection + res = padding(adata, "zero", pad="absolute", padlength=total_time,unit="time", + create_new=True, select={"trials": trialSel, "channels": chanSel}) + for tk, trl in enumerate(res.trials): + adataTrl = adata.trials[trialSel[tk]] + nSamples = pad_list[trialSel[tk]]["pad_width"][timeAxis, :].sum() + adataTrl.shape[timeAxis] + assert trl.shape[timeAxis] == nSamples + assert trl.shape[chanAxis] == len(list(chanSel)) + + # test correct update of trigger onset w/pre-padding + adataTimes = adata.time + prepadTime = 5 + res = padding(adata, "zero", pad="relative", prepadlength=prepadTime, + unit="time", create_new=True) + resTimes = res.time + adataTimes = adata.time + for tk, timeArr in enumerate(resTimes): + assert timeArr[0] == adataTimes[tk][0] - prepadTime + assert np.array_equal(timeArr[timeArr >= 0], adataTimes[tk][adataTimes[tk] >= 0]) + + # postpadding must not change trigger onset timing + postpadTime = 5 + res = padding(adata, "zero", pad="relative", postpadlength=postpadTime, + unit="time", create_new=True) + resTimes = res.time + for tk, timeArr in enumerate(resTimes): + assert timeArr[0] == adataTimes[tk][0] + assert np.array_equal(timeArr[timeArr <= 0], adataTimes[tk][adataTimes[tk] <= 0]) # jumble axes of `AnalogData` object and compute max. trial length adata2 = generate_artificial_data(nTrials=7, nChannels=16, - equidistant=False, inmemory=False, - dimord=adata.dimord[::-1]) + equidistant=False, inmemory=False, + dimord=adata.dimord[::-1]) timeAxis2 = adata2.dimord.index("time") + chanAxis2 = adata2.dimord.index("channel") maxtrllen = 0 for trl in adata2.trials: maxtrllen = max(maxtrllen, trl.shape[timeAxis2]) + # same as above, but this time w/swapped dimensions + res2 = padding(adata2, "zero", pad="absolute", padlength=total_time, unit="time", + create_new=True, select={"trials": trialSel, "channels": chanSel}) + pad_list2 = padding(adata2, "zero", pad="absolute", padlength=total_time, + unit="time", create_new=False) + for tk, trl in enumerate(res2.trials): + adataTrl = adata2.trials[trialSel[tk]] + nSamples = pad_list2[trialSel[tk]]["pad_width"][timeAxis2, :].sum() + adataTrl.shape[timeAxis2] + try: + assert trl.shape[timeAxis2] == nSamples + except: + import pdb; pdb.set_trace() + assert trl.shape[chanAxis2] == len(list(chanSel)) + # symmetric `maxlen` padding: 1 sample tolerance pad_list2 = padding(adata2, "zero", pad="maxlen", create_new=False) for tk, trl in enumerate(adata2.trials): @@ -532,6 +576,21 @@ def test_object_padding(self): trl_len = pad_list2[tk]["pad_width"][timeAxis2, :].sum() + trl.shape[timeAxis2] assert trl_len == maxtrllen + # make things maximally intersting: relative + time + non-equidistant + + # overlapping + selection + nonstandard dimord + adata3 = generate_artificial_data(nTrials=7, nChannels=16, + equidistant=False, overlapping=True, + inmemory=False, dimord=adata2.dimord) + res3 = padding(adata3, "zero", pad="absolute", padlength=total_time, unit="time", + create_new=True, select={"trials": trialSel, "channels": chanSel}) + pad_list3 = padding(adata3, "zero", pad="absolute", padlength=total_time, + unit="time", create_new=False) + for tk, trl in enumerate(res3.trials): + adataTrl = adata3.trials[trialSel[tk]] + nSamples = pad_list3[trialSel[tk]]["pad_width"][timeAxis2, :].sum() + adataTrl.shape[timeAxis2] + assert trl.shape[timeAxis2] == nSamples + assert trl.shape[chanAxis2] == len(list(chanSel)) + # `maxlen'-specific errors: `padlength` wrong type, wrong combo with `prepadlength` with pytest.raises(SPYTypeError): padding(adata, "zero", pad="maxlen", padlength=self.ns, create_new=False) @@ -541,10 +600,6 @@ def test_object_padding(self): padding(adata, "zero", pad="maxlen", padlength=self.ns, prepadlength=True, create_new=False) - # FIXME: implement as soon as object padding is supported: - # test absolute + time + non-equidistant! - # test relative + time + non-equidistant + overlapping! - # test data-selection via class method def test_dataselection(self): From 6b7981ba97e34b9d4ba9290065a2acddf2660d3a Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 26 Nov 2021 13:44:12 +0100 Subject: [PATCH 080/109] CHG: Streamlined testing pipeline - don't run through all possible data selections for arithmetic tests; this increases testing runtime considerably without providing much added value On branch padding Changes to be committed: modified: syncopy/tests/test_continuousdata.py --- syncopy/tests/test_continuousdata.py | 68 +++++++++++++++++----------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 427653b3f..037bf3bfd 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -395,7 +395,10 @@ def test_relative_array_padding(self): for ptype in ["zero", "mean", "localmean", "edge", "mirror"]: arr = padding(self.data, ptype, pad="relative", **kws) for k, idx in enumerate(expected_idx[loc]): - assert np.all(arr[idx, :] == expected_vals[loc][ptype][k]) + try: + assert np.all(arr[idx, :] == expected_vals[loc][ptype][k]) + except: + import pdb; pdb.set_trace() assert arr.shape[0] == expected_shape[loc] arr = padding(self.data, "nan", pad="relative", **kws) for idx in expected_idx[loc]: @@ -547,10 +550,7 @@ def test_object_padding(self): for tk, trl in enumerate(res2.trials): adataTrl = adata2.trials[trialSel[tk]] nSamples = pad_list2[trialSel[tk]]["pad_width"][timeAxis2, :].sum() + adataTrl.shape[timeAxis2] - try: - assert trl.shape[timeAxis2] == nSamples - except: - import pdb; pdb.set_trace() + assert trl.shape[timeAxis2] == nSamples assert trl.shape[chanAxis2] == len(list(chanSel)) # symmetric `maxlen` padding: 1 sample tolerance @@ -671,15 +671,21 @@ def test_ang_arithmetic(self): _base_op_tests(dummy, ymmud, dummy2, ymmud2, None, operation) # Now the most complicated case: user-defined subset selections are present - for trialSel in trialSelections: - for chanSel in chanSelections: - for timeSel in timeSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["channels"] = chanSel - kwdict[timeSel[0]] = timeSel[1] - - _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) + kwdict = {} + kwdict["trials"] = trialSelections[1] + kwdict["channels"] = chanSelections[3] + kwdict[timeSelections[4][0]] = timeSelections[4][1] + _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) + + # # Go through full selection stack - WARNING: this takes > 15 minutes + # for trialSel in trialSelections: + # for chanSel in chanSelections: + # for timeSel in timeSelections: + # kwdict = {} + # kwdict["trials"] = trialSel + # kwdict["channels"] = chanSel + # kwdict[timeSel[0]] = timeSel[1] + # _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) # Finally, perform a representative chained operation to ensure chaining works result = (dummy + dummy2) / dummy ** 3 @@ -895,19 +901,27 @@ def test_sd_arithmetic(self): _base_op_tests(dummy, ymmud, dummy2, ymmud2, dummyC, operation) # Now the most complicated case: user-defined subset selections are present - for trialSel in trialSelections: - for chanSel in chanSelections: - for timeSel in timeSelections: - for freqSel in freqSelections: - for taperSel in taperSelections: - kwdict = {} - kwdict["trials"] = trialSel - kwdict["channels"] = chanSel - kwdict[timeSel[0]] = timeSel[1] - kwdict[freqSel[0]] = freqSel[1] - kwdict["tapers"] = taperSel - - _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) + kwdict = {} + kwdict["trials"] = trialSelections[1] + kwdict["channels"] = chanSelections[3] + kwdict[timeSelections[4][0]] = timeSelections[4][1] + kwdict[freqSelections[4][0]] = freqSelections[4][1] + kwdict["tapers"] = taperSelections[2] + _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) + + # # Go through full selection stack - WARNING: this takes > 1 hour + # for trialSel in trialSelections: + # for chanSel in chanSelections: + # for timeSel in timeSelections: + # for freqSel in freqSelections: + # for taperSel in taperSelections: + # kwdict = {} + # kwdict["trials"] = trialSel + # kwdict["channels"] = chanSel + # kwdict[timeSel[0]] = timeSel[1] + # kwdict[freqSel[0]] = freqSel[1] + # kwdict["tapers"] = taperSel + # _selection_op_tests(dummy, ymmud, dummy2, ymmud2, kwdict, operation) # Finally, perform a representative chained operation to ensure chaining works result = (dummy + dummy2) / dummy ** 3 From b6178fdfa6332fff3ea94e2b4375bac74cb81086 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 26 Nov 2021 15:55:24 +0100 Subject: [PATCH 081/109] FIX: Allow meta-functions to process only NumPy arrays - the cfg-unwrapping code assumed that all Syncopy meta-functions must be called with at least one Syncopy object. Loosen this restriction to allow meta functions to solely work on, e.g., NumPy arrays - adapted decorator tests accordingly On branch padding Changes to be committed: modified: syncopy/shared/kwarg_decorators.py modified: syncopy/tests/test_decorators.py --- syncopy/shared/kwarg_decorators.py | 8 +--- syncopy/tests/test_decorators.py | 70 ++++++++++++++---------------- 2 files changed, 33 insertions(+), 45 deletions(-) diff --git a/syncopy/shared/kwarg_decorators.py b/syncopy/shared/kwarg_decorators.py index 316ba9bec..a4f588399 100644 --- a/syncopy/shared/kwarg_decorators.py +++ b/syncopy/shared/kwarg_decorators.py @@ -241,12 +241,6 @@ def wrapper_cfg(*args, **kwargs): else: posargs.append(arg) - # At this point, `data` is a list: if it's empty, not a single Syncopy data object - # was provided (neither via `cfg`, `kwargs`, or `args`) and the call is invalid - if len(data) == 0: - err = "{0} missing mandatory argument: `{1}`" - raise SPYError(err.format(func.__name__, arg0)) - # Call function with unfolded `data` + modified positional/keyword args return func(*data, *posargs, **cfg) @@ -481,7 +475,7 @@ def parallel_client_detector(*args, **kwargs): kwargs["parallel"] = parallel # Process provided object(s) - if nObs == 1: + if nObs <= 1: results = func(*args, **kwargs) else: results = [] diff --git a/syncopy/tests/test_decorators.py b/syncopy/tests/test_decorators.py index fd2d3be79..6bf81cba9 100644 --- a/syncopy/tests/test_decorators.py +++ b/syncopy/tests/test_decorators.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Test proper functionality of Syncopy's decorator mechanics -# +# # Builtin/3rd party package imports import string @@ -17,7 +17,7 @@ def group_objects(*data, groupbychan=None, select=None): """ Dummy function that collects the `filename` property of all - input objects that contain a specific channel given by + input objects that contain a specific channel given by `groupbychan` """ group = [] @@ -31,13 +31,13 @@ def group_objects(*data, groupbychan=None, select=None): class TestSpyCalls(): - + nChan = 13 nObjs = nChan - # Generate `nChan` objects whose channel-labeling scheme obeys: + # Generate `nChan` objects whose channel-labeling scheme obeys: # ob1.channel = ["A", "B", "C", ..., "M"] - # ob2.channel = [ "B", "C", ..., "M", "N"] + # ob2.channel = [ "B", "C", ..., "M", "N"] # ob3.channel = [ "C", ..., "M", "N", "O"] # ... # ob13.channel = [ "M", "N", "O", ..., "Z"] @@ -48,23 +48,23 @@ class TestSpyCalls(): obj.channel = list(string.ascii_uppercase[n : nChan + n]) dataObjs.append(obj) data = dataObjs[0] - + def test_validcallstyles(self): - + # data positional fname, = group_objects(self.data) assert fname == self.data.filename - + # data as keyword fname, = group_objects(data=self.data) assert fname == self.data.filename - + # data in cfg cfg = StructDict() cfg.data = self.data fname, = group_objects(cfg) assert fname == self.data.filename - + # 1. data positional, 2. cfg positional cfg = StructDict() cfg.groupbychan = None @@ -74,38 +74,38 @@ def test_validcallstyles(self): # 1. cfg positional, 2. data positional fname, = group_objects(cfg, self.data) assert fname == self.data.filename - + # data positional, cfg as keyword fname, = group_objects(self.data, cfg=cfg) assert fname == self.data.filename - + # cfg positional, data as keyword fname, = group_objects(cfg, data=self.data) assert fname == self.data.filename - + # both keywords fname, = group_objects(cfg=cfg, data=self.data) assert fname == self.data.filename - + def test_invalidcallstyles(self): - + # expected error messages errmsg1 = "expected Syncopy data object(s) provided either via " +\ "`cfg`/keyword or positional arguments, not both" errmsg2 = "expected Syncopy data object(s) provided either via `cfg` " +\ "or as keyword argument, not both" errmsg3 = "expected either 'data' or 'dataset' in `cfg`/keywords, not both" - + # ensure things break reliably for 'data' as well as 'dataset' for key in ["data", "dataset"]: - + # data + cfg w/data cfg = StructDict() cfg[key] = self.data with pytest.raises(SPYValueError) as exc: group_objects(self.data, cfg) assert errmsg1 in str(exc.value) - + # data as positional + kwarg with pytest.raises(SPYValueError) as exc: group_objects(self.data, data=self.data) @@ -121,7 +121,7 @@ def test_invalidcallstyles(self): with pytest.raises(SPYValueError) as exc: group_objects(self.data, cfg, dataset=self.data) assert errmsg1 in str(exc.value) - + # cfg w/data + kwarg with pytest.raises(SPYValueError) as exc: group_objects(cfg, data=self.data) @@ -136,12 +136,12 @@ def test_invalidcallstyles(self): with pytest.raises(SPYValueError)as exc: group_objects(self.data, cfg, cfg=cfg) assert "expected `cfg` either as positional or keyword argument, not both" in str(exc.value) - + # keyword set via cfg and kwarg with pytest.raises(SPYValueError) as exc: group_objects(self.data, cfg, groupbychan="invalid") assert "'non-default value for groupbychan'; expected no keyword arguments" in str(exc.value) - + # both data and dataset in cfg/keywords cfg = StructDict() cfg.data = self.data @@ -157,19 +157,14 @@ def test_invalidcallstyles(self): with pytest.raises(SPYError)as exc: group_objects(data="invalid") assert "`data` must be Syncopy data object(s)!" in str(exc.value) - + # cfg is not dict/StructDict with pytest.raises(SPYTypeError)as exc: group_objects(cfg="invalid") assert "Wrong type of cfg: expected dictionary-like" in str(exc.value) - # no data input whatsoever - with pytest.raises(SPYError)as exc: - group_objects("invalid") - assert "missing mandatory argument: `data`" in str(exc.value) - def test_varargin(self): - + # data positional allFnames = group_objects(*self.dataObjs) assert allFnames == [obj.filename for obj in self.dataObjs] @@ -179,7 +174,7 @@ def test_varargin(self): cfg.data = self.dataObjs fnameList = group_objects(cfg) assert allFnames == fnameList - + # group objects by single-letter "channels" in various ways for letter in ["L", "E", "I", "A"]: letterIdx = string.ascii_uppercase.index(letter) @@ -188,7 +183,7 @@ def test_varargin(self): # data positional + keyword to get "reference" groupList = group_objects(*self.dataObjs, groupbychan=letter) assert len(groupList) == nOccurences - + # 1. data positional, 2. cfg positional cfg = StructDict() cfg.groupbychan = letter @@ -198,11 +193,11 @@ def test_varargin(self): # 1. cfg positional, 2. data positional fnameList = group_objects(cfg, *self.dataObjs) assert groupList == fnameList - + # data positional, cfg as keyword fnameList = group_objects(*self.dataObjs, cfg=cfg) assert groupList == fnameList - + # cfg w/data + keyword cfg = StructDict() cfg.dataset = self.dataObjs @@ -211,7 +206,7 @@ def test_varargin(self): assert groupList == fnameList # data positional + select keyword - fnameList = group_objects(*self.dataObjs[:letterIdx + 1], + fnameList = group_objects(*self.dataObjs[:letterIdx + 1], select={"channels": [letter]}) assert groupList == fnameList @@ -227,17 +222,16 @@ def test_varargin(self): cfg.select = {"channels": [letter]} fnameList = group_objects(cfg) assert groupList == fnameList - + # invalid selection with pytest.raises(SPYValueError) as exc: group_objects(*self.dataObjs, select={"channels": ["Z"]}) assert "expected list/array of channel existing names or indices" in str(exc.value) - # data does not only contain Syncopy objects + # data does not only contain Syncopy objects cfg = StructDict() cfg.data = self.dataObjs + ["invalid"] with pytest.raises(SPYError)as exc: group_objects(cfg) assert "`data` must be Syncopy data object(s)!" in str(exc.value) - - \ No newline at end of file + From a5d0e247b0a14f18ebc9e4ea81b80970bfc2b53a Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 26 Nov 2021 16:49:57 +0100 Subject: [PATCH 082/109] NEW: Added support for clearing in-place selections - a new keyword `clear` in `selectdata` allows to unset previously defined in-place selections (without the need to mess around with the `_selection` property); closes #159 - in the process the logging dictionary setup in `selectdata` has been cleared up - a few simple tests addressing this new feature have been added On branch patches Changes to be committed: modified: syncopy/datatype/methods/selectdata.py modified: syncopy/tests/test_selectdata.py --- syncopy/datatype/methods/selectdata.py | 49 +++++++++++++++++--------- syncopy/tests/test_selectdata.py | 10 ++++++ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 8aa6c0e88..83432b2c7 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -17,7 +17,7 @@ @detect_parallel_client def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None, foilim=None, tapers=None, units=None, eventids=None, - out=None, inplace=False, **kwargs): + out=None, inplace=False, clear=False, **kwargs): """ Create a new Syncopy object from a selection @@ -245,9 +245,11 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None except Exception as exc: raise exc - # Vet the only input not checked by `Selector` + # Vet the only inputs not checked by `Selector` if not isinstance(inplace, bool): raise SPYTypeError(inplace, varname="inplace", expected="Boolean") + if not isinstance(inplace, bool): + raise SPYTypeError(clear, varname="clear", expected="Boolean") # If provided, make sure output object is appropriate if not inplace: @@ -267,16 +269,31 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None lgl = "no output object for in-place selection" raise SPYValueError(lgl, varname="out", actual=out.__class__.__name__) + # Collect provided keywords in dict + selectDict = {"trials": trials, + "channels": channels, + "toi": toi, + "toilim": toilim, + "foi": foi, + "foilim": foilim, + "tapers": tapers, + "units": units, + "eventids": eventids} + + # First simplest case: determine whether we just need to clear an existing selection + if clear: + if any(value is not None for value in selectDict.values()): + lgl = "no data selectors if `clear = True`" + raise SPYValueError(lgl, varname="select", actual=selectDict) + if data._selection is None: + SPYInfo("No in-place selection found. ") + else: + data._selection = None + SPYInfo("In-place selection cleared") + return + # Pass provided selections on to `Selector` class which performs error checking - data._selection = {"trials": trials, - "channels": channels, - "toi": toi, - "toilim": toilim, - "foi": foi, - "foilim": foilim, - "tapers": tapers, - "units": units, - "eventids": eventids} + data._selection = selectDict # If an in-place selection was requested we're done if inplace: @@ -285,17 +302,15 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None # Create inventory of all available selectors and actually provided values # to create a bookkeeping dict for logging - provided = {**locals(), **kwargs} - available = get_defaults(data.selectdata) - actualSelection = {} - for key in available: - actualSelection[key] = provided[key] + log_dct = {"inplace": inplace, "clear": clear} + log_dct.update(selectDict) + log_dct.update(**kwargs) # Fire up `ComputationalRoutine`-subclass to do the actual selecting/copying selectMethod = DataSelection() selectMethod.initialize(data, out._stackingDim, chan_per_worker=kwargs.get("chan_per_worker")) selectMethod.compute(data, out, parallel=kwargs.get("parallel"), - log_dict=actualSelection) + log_dict=log_dct) # Wipe data-selection slot to not alter input object data._selection = None diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index 365ea61f1..5571c845c 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -387,6 +387,16 @@ def test_general(self): with pytest.raises(SPYValueError): Selector(ang, {"wrongkey": [1]}) + # set/clear in-place data selection (both setting and clearing are idempotent, + # i.e., repeated execution must work, hence the double whammy) + ang.selectdata(trials=[3, 1]) + ang.selectdata(trials=[3, 1]) + ang.selectdata(clear=True) + ang.selectdata(clear=True) + with pytest.raises(SPYValueError) as spyval: + ang.selectdata(trials=[3, 1], clear=True) + assert "no data selectors if `clear = True`" in str(spyval.value) + # go through all data-classes defined above for dclass in self.classes: dummy = getattr(spd, dclass)(data=self.data[dclass], From dc6634d49bbe711439736fadb459c4a762e84f74 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 26 Nov 2021 17:44:00 +0100 Subject: [PATCH 083/109] NEW: Included code-coverage in testing pipeline - included code-coverage functionality in testing pipeline (requires pytest-cov) - prepared codecov webhook integration On branch patches Changes to be committed: new file: .coveragerc modified: .travis.yml new file: codecov.yml modified: syncopy.yml modified: syncopy/tests/run_tests.sh modified: tox.ini --- .coveragerc | 16 ++++++++++++++++ .travis.yml | 8 +++++--- codecov.yml | 14 ++++++++++++++ syncopy.yml | 4 ++-- syncopy/tests/run_tests.sh | 4 +++- tox.ini | 5 ++--- 6 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 .coveragerc create mode 100644 codecov.yml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..5bc000434 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,16 @@ +[run] +branch = True +source = syncopy + +[report] +exclude_lines = + if self.debug: + if debug: + raise NotImplementedError + if __name__ == .__main__.: +ignore_errors = True +omit = + syncopy/tests/* + *conda2pip.py + *setup.py + test_* diff --git a/.travis.yml b/.travis.yml index 1e52b6f42..89addbbc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: cache: pip -# safelist branches: only: - master @@ -16,6 +15,9 @@ install: - pip install -r requirements.txt - pip install -r requirements-test.txt - python setup.py -q install -# command to run tests + script: - - pytest -v + - pytest -v --cov=./ + +after_success: + - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..ca4291b1f --- /dev/null +++ b/codecov.yml @@ -0,0 +1,14 @@ +coverage: + status: + project: + default: + # Allow coverage to drop `threshold` percent in PRs to master/dev + target: auto + threshold: 5% + base: auto + branches: + - master + - dev + if_ci_failed: error #success, failure, error, ignore + informational: false + only_pulls: true diff --git a/syncopy.yml b/syncopy.yml index ae1fec83c..f6d05ea95 100644 --- a/syncopy.yml +++ b/syncopy.yml @@ -7,7 +7,7 @@ dependencies: - python >= 3.8, < 3.9 - pip - numpy >= 1.10, < 2.0 - - scipy >= 1.5, < 1.6 + - scipy >= 1.5 - h5py >= 2.9, < 3 - matplotlib >= 3.3, < 3.5 - tqdm >= 4.31 @@ -18,7 +18,7 @@ dependencies: - memory_profiler - numpydoc - sphinx_bootstrap_theme - - pytest + - pytest-cov - pylint - ipdb - tox diff --git a/syncopy/tests/run_tests.sh b/syncopy/tests/run_tests.sh index f561ed01c..d041c1506 100755 --- a/syncopy/tests/run_tests.sh +++ b/syncopy/tests/run_tests.sh @@ -34,7 +34,7 @@ if [ "$1" == "" ]; then usage fi -# Set up "global" pytest options for running test-suite +# Set up "global" pytest options for running test-suite (coverage is only done in local pytest runs) export PYTEST_ADDOPTS="--color=yes --tb=short --verbose" # The while construction allows parsing of multiple positional/optional args (future-proofing...) @@ -46,6 +46,8 @@ while [ "$1" != "" ]; do if [ $_useSLURM ]; then srun -p DEV --mem=8000m -c 4 pytest else + PYTEST_ADDOPTS="$PYTEST_ADDOPTS --cov=../../syncopy --cov-config=../../.coveragerc" + export PYTEST_ADDOPTS pytest fi ;; diff --git a/tox.ini b/tox.ini index 065399184..86277a0b5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38-scipy{14,15}-{noacme, acme} +envlist = py38-scipy15-{noacme, acme} requires = tox-conda isolated_build = True @@ -13,8 +13,7 @@ deps = tqdm >= 4.31 memory_profiler conda_deps= - scipy14: scipy >= 1.4, < 1.5 - scipy15: scipy >= 1.5, < 1.6 + scipy15: scipy >= 1.5 acme: esi-acme conda_channels= defaults From 0a97757c87ec43b2123f4a935737a0f7e0bcb6f8 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 29 Nov 2021 10:16:44 +0100 Subject: [PATCH 084/109] FIX: Patched `CrossSpectralData` to be compliant w/spy.save/load - updated `_hdfFileAttributeProperties` and `_infoFileProperties` of `CrossSpectralData` to work with Syncopy's I/O routines - added `CrossSpectralData` to supported data-types for performing I/O ops On branch tests Changes to be committed: modified: syncopy/datatype/continuous_data.py modified: syncopy/shared/filetypes.py --- syncopy/datatype/continuous_data.py | 3 ++- syncopy/shared/filetypes.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index a59f8f449..4309dd393 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -660,7 +660,8 @@ class CrossSpectralData(ContinuousData): frequency and optionally time or lag. The datatype can be complex or float. """ - _infoFileProperties = ContinuousData._infoFileProperties + ("freq",) + _infoFileProperties = BaseData._infoFileProperties + ("samplerate", "channel_i", "channel_j",) + _hdfFileAttributeProperties = BaseData._hdfFileAttributeProperties + ("samplerate", "channel_i", "channel_j",) _defaultDimord = ["time", "freq", "channel_i", "channel_j"] _stackingDimLabel = "time" _channel_i = None diff --git a/syncopy/shared/filetypes.py b/syncopy/shared/filetypes.py index f3717ff76..874a2d8f4 100644 --- a/syncopy/shared/filetypes.py +++ b/syncopy/shared/filetypes.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- -# +# # Supported Syncopy classes and file extensions -# +# def _data_classname_to_extension(name): return "." + name.split('Data')[0].lower() # data file extensions are first word of data class name in lower-case -supportedClasses = ('AnalogData', 'SpectralData', # ContinousData +supportedClasses = ('AnalogData', 'SpectralData', 'CrossSpectralData', # ContinousData 'SpikeData', 'EventData', # DiscreteData 'TimelockData', ) # StatisticalData From dd68c743957f6dca700b2eebc070e54a564a7a84 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 29 Nov 2021 13:29:49 +0100 Subject: [PATCH 085/109] NEW: Added support for CrossSpectralData in selectdata - included new selector-dict keywords `channels_i` and `channels_j` in `selectdata` - added corresponding channel-selection props in `Selector` class - modified existing `channel` property of `Selector` to adequately complain if a selection is performed on a CrossSpectralData object using `channels` as selection keyword - simple channel selections work as expected, more complex cases have to be hammered out via testing (cf. #165) - closes #164 On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/methods/selectdata.py modified: syncopy/tests/test_continuousdata.py --- syncopy/datatype/base_data.py | 33 ++++++++- syncopy/datatype/continuous_data.py | 93 ++------------------------ syncopy/datatype/methods/selectdata.py | 15 +++-- syncopy/tests/test_continuousdata.py | 5 +- 4 files changed, 46 insertions(+), 100 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 1ed2e5fcd..10c623a26 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -1314,8 +1314,8 @@ def __init__(self, data, select): varname="select", actual=select) if not isinstance(select, dict): raise SPYTypeError(select, "select", expected="dict") - supported = ["trials", "channels", "toi", "toilim", "foi", "foilim", - "tapers", "units", "eventids"] + supported = ["trials", "channels", "channels_i", "channels_j", "toi", + "toilim", "foi", "foilim", "tapers", "units", "eventids"] if not set(select.keys()).issubset(supported): lgl = "dict with one or all of the following keys: '" +\ "'".join(opt + "', " for opt in supported)[:-2] @@ -1328,7 +1328,7 @@ def __init__(self, data, select): # Set up lists of (a) all selectable properties (b) trial-dependent ones # and (c) selectors independent from trials - self._allProps = ["channel", "time", "freq", "taper", "unit", "eventid"] + self._allProps = ["channel", "channel_i", "channel_j", "time", "freq", "taper", "unit", "eventid"] self._byTrialProps = ["time", "unit", "eventid"] self._dimProps = list(self._allProps) for prop in self._byTrialProps: @@ -1399,8 +1399,35 @@ def channel(self): @channel.setter def channel(self, dataselect): data, select = dataselect + chanSpec = select.get("channels") + if self._dataClass == "CrossSpectralData": + if chanSpec is not None: + lgl = "`channel_i` and/or `channel_j` selectors for `CrossSpectralData`" + raise SPYValueError(legal=lgl, varname="select: channels", actual=data.__class__.__name__) + else: + return self._selection_setter(data, select, "channel", "channels") + @property + def channel_i(self): + """List or slice encoding principal channel-pair selection""" + return self._channel_i + + @channel_i.setter + def channel_i(self, dataselect): + data, select = dataselect + self._selection_setter(data, select, "channel_i", "channels_i") + + @property + def channel_j(self): + """List or slice encoding principal channel-pair selection""" + return self._channel_j + + @channel_j.setter + def channel_j(self, dataselect): + data, select = dataselect + self._selection_setter(data, select, "channel_j", "channels_j") + @property def time(self): """len(self.trials) list of lists/slices of by-trial time-selections""" diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 4309dd393..71b58868c 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -473,41 +473,6 @@ def __init__(self, channel=channel, dimord=dimord) - # # Overload ``copy`` method to account for `VirtualData` memmaps - # def copy(self, deep=False): - # """Create a copy of the data object in memory. - - # Parameters - # ---------- - # deep : bool - # If `True`, a copy of the underlying data file is created in the temporary Syncopy folder - - - # Returns - # ------- - # AnalogData - # in-memory copy of AnalogData object - - # See also - # -------- - # save_spy - - # """ - - # cpy = copy(self) - - # if deep: - # if isinstance(self.data, VirtualData): - # print("SyNCoPy core - copy: Deep copy not possible for " + - # "VirtualData objects. Please use `save_spy` instead. ") - # return - # elif isinstance(self.data, (np.memmap, h5py.Dataset)): - # self.data.flush() - # filename = self._gen_filename() - # shutil.copyfile(self._filename, filename) - # cpy.data = filename - # return cpy - class SpectralData(ContinuousData): """ @@ -660,6 +625,7 @@ class CrossSpectralData(ContinuousData): frequency and optionally time or lag. The datatype can be complex or float. """ + # Adapt `infoFileProperties` and `hdfFileAttributeProperties` from `ContinuousData` _infoFileProperties = BaseData._infoFileProperties + ("samplerate", "channel_i", "channel_j",) _hdfFileAttributeProperties = BaseData._hdfFileAttributeProperties + ("samplerate", "channel_i", "channel_j",) _defaultDimord = ["time", "freq", "channel_i", "channel_j"] @@ -669,6 +635,10 @@ class CrossSpectralData(ContinuousData): _samplerate = None _data = None + # Steal frequency-related stuff from `SpectralData` + _get_freq = SpectralData._get_freq + freq = SpectralData.freq + # override channel property to avoid accidental access @property def channel(self): @@ -742,59 +712,6 @@ def channel_j(self, channel_j): self._channel_j = np.array(channel_j) - @property - def freq(self): - """:class:`numpy.ndarray`: frequency axis in Hz """ - # if data exists but no user-defined frequency axis, - # create a dummy one on the fly - - if self._freq is None and self._data is not None: - return np.arange(self.data.shape[self.dimord.index("freq")]) - return self._freq - - @freq.setter - def freq(self, freq): - - if freq is None: - self._freq = None - return - - if self.data is None: - print("Syncopy core - freq: Cannot assign `freq` without data. "+\ - "Please assing data first") - return - try: - - array_parser(freq, varname="freq", hasnan=False, hasinf=False, - dims=(self.data.shape[self.dimord.index("freq")],)) - except Exception as exc: - raise exc - - self._freq = np.array(freq) - - # Override selector method - def selectdata(self, trials=None, channels1=None, channels2=None, toi=None, toilim=None, - foi=None, foilim=None): - """ - Create new `CrossSpectralData` object from selection - - Please refer to :func:`syncopy.selectdata` for detailed usage information. - - Examples - -------- - >>> Coming soon - - See also - -------- - syncopy.selectdata : create new objects via deep-copy selections - """ - - if channels1 is not None or channels2 is not None: - raise NotImplementedError("Channel selection not yet supported for CrossSpectralData") - - return selectdata(self, trials=trials, toi=toi, - toilim=toilim, foi=foi, foilim=foilim) - # # Local 2d -> 1d channel index converter # def _ind2sub(self, channel1, channel2): # """Convert 2d channel tuple to linear 1d index""" diff --git a/syncopy/datatype/methods/selectdata.py b/syncopy/datatype/methods/selectdata.py index 83432b2c7..f81f7d1d6 100644 --- a/syncopy/datatype/methods/selectdata.py +++ b/syncopy/datatype/methods/selectdata.py @@ -5,8 +5,7 @@ # Local imports from syncopy.shared.parsers import data_parser -from syncopy.shared.tools import get_defaults -from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYInfo +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYInfo, SPYWarning from syncopy.shared.kwarg_decorators import unwrap_cfg, unwrap_io, detect_parallel_client from syncopy.shared.computational_routine import ComputationalRoutine @@ -15,9 +14,9 @@ @unwrap_cfg @detect_parallel_client -def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None, - foilim=None, tapers=None, units=None, eventids=None, - out=None, inplace=False, clear=False, **kwargs): +def selectdata(data, trials=None, channels=None, channels_i=None, channels_j=None, + toi=None, toilim=None, foi=None, foilim=None, tapers=None, units=None, + eventids=None, out=None, inplace=False, clear=False, **kwargs): """ Create a new Syncopy object from a selection @@ -269,9 +268,15 @@ def selectdata(data, trials=None, channels=None, toi=None, toilim=None, foi=None lgl = "no output object for in-place selection" raise SPYValueError(lgl, varname="out", actual=out.__class__.__name__) + # FIXME: remove once tests are in place (cf #165) + if channels_i is not None or channels_j is not None: + SPYWarning("CrossSpectralData channel selection currently untested and experimental!") + # Collect provided keywords in dict selectDict = {"trials": trials, "channels": channels, + "channels_i": channels_i, + "channels_j": channels_j, "toi": toi, "toilim": toilim, "foi": foi, diff --git a/syncopy/tests/test_continuousdata.py b/syncopy/tests/test_continuousdata.py index 037bf3bfd..1b594c1d9 100644 --- a/syncopy/tests/test_continuousdata.py +++ b/syncopy/tests/test_continuousdata.py @@ -395,10 +395,7 @@ def test_relative_array_padding(self): for ptype in ["zero", "mean", "localmean", "edge", "mirror"]: arr = padding(self.data, ptype, pad="relative", **kws) for k, idx in enumerate(expected_idx[loc]): - try: - assert np.all(arr[idx, :] == expected_vals[loc][ptype][k]) - except: - import pdb; pdb.set_trace() + assert np.all(arr[idx, :] == expected_vals[loc][ptype][k]) assert arr.shape[0] == expected_shape[loc] arr = padding(self.data, "nan", pad="relative", **kws) for idx in expected_idx[loc]: From 2be9f4d757fd2778c20a12c4c8e83d7faf6c0533 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 29 Nov 2021 14:58:53 +0100 Subject: [PATCH 086/109] WIP: First version __eq__ operator - included new `__eq__` method in `BaseData` to overload the "==" operator (code not functional yet) On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py --- syncopy/datatype/base_data.py | 78 +++++++++++++++++++++++++++++ syncopy/datatype/continuous_data.py | 9 ++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 10c623a26..af5fce30c 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -766,6 +766,84 @@ def __rtruediv__(self, other): def __pow__(self, other): return _process_operator(self, other, "**") + def __eq__(self, other): + + # If other object is not a Syncopy data-class, get out + if not "BaseData" in str(other.__class__.__mro__): + SPYInfo("Not a Syncopy object") + return False + + # Check if two Syncopy objects of same type/dimord are present + try: + data_parser(other, dimord=self.dimord, dataclass=self.__class__.__name__) + except Exception as exc: + SPYInfo("Syncopy object of different type/dimord") + return False + + # First, ensure we have something to compare here + if self._is_empty() and not other._is_empty(): + SPYInfo("Empty and non-empty Syncopy object") + return False + + # Start cheap: check if samplerates are identical (if present) + baseSr = getattr(self, "samplerate") + opndSr = getattr(other, "samplerate") + if baseSr != opndSr: + SPYInfo("Mismatch in samplerates") + return False + + # If in-place selections are present, abort + if self._selection is not None or other._selection is not None: + err = "Cannot perform object comparison with existing in-place selection" + raise SPYError(err) + + # Use in-place selections to query class-specific dimensional properties + # (i.e., channels, freq, taper etc.) + # FIXME: don't do this; loop over `_infoFileProperties` instead and kick + # out `dimord`, `cfg` and all underscore attrs + try: + self.selectdata(inplace=True) + except: + import ipdb; ipdb.set_trace() + other.selectdata(inplace=True) + isEqual = True + for prop in self._selection._dimProps: + if getattr(self._selection, prop) != getattr(other._selection, prop): + SPYInfo("Mismatch in {}".format(prop)) + isEqual = False + self.selectdata(clear=True) + other.selectdata(clear=True) + if not isEqual: + return False + + # Check if trial setup is identical + if not np.array_equal(self.trialdefinition, other.trialdefinition): + SPYInfo("Mismatch in trial layouts") + return False + + # If an object is compared to itself (or its shallow copy), don't bother + # juggling NumPy arrays but simply perform a quick dataset/filename comparison + if self.filename == other.filename: + for dsetName in self._hdfFileDatasetProperties: + if not getattr(self, dsetName) == getattr(other, dsetName): + isEqual = False + if not isEqual: + SPYInfo("HDF dataset mismatch") + return False + return True + + # The other object really is a standalone Syncopy class instance and + # everything but the data itself aligns; now the most expensive part: + # trial by trial data comparison + for tk in range(len(self.trials)): + if not np.allclose(self.trials[tk], other.trials[tk]): + SPYInfo("Mismatch in trial #{}".format(tk)) + return False + + # If we made it this far, `self` and `other` really seem to be identical + return True + + # Class "constructor" def __init__(self, filename=None, dimord=None, mode="r+", **kwargs): """ diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 71b58868c..acc4a1ed7 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -9,8 +9,6 @@ """ # Builtin/3rd party package imports -import h5py -import os import inspect import numpy as np from abc import ABC @@ -19,7 +17,6 @@ # Local imports from .base_data import BaseData, FauxTrial from .methods.definetrial import definetrial -from .methods.selectdata import selectdata from syncopy.shared.parsers import scalar_parser, array_parser from syncopy.shared.errors import SPYValueError from syncopy.shared.tools import best_match @@ -626,8 +623,10 @@ class CrossSpectralData(ContinuousData): """ # Adapt `infoFileProperties` and `hdfFileAttributeProperties` from `ContinuousData` - _infoFileProperties = BaseData._infoFileProperties + ("samplerate", "channel_i", "channel_j",) - _hdfFileAttributeProperties = BaseData._hdfFileAttributeProperties + ("samplerate", "channel_i", "channel_j",) + _infoFileProperties = BaseData._infoFileProperties +\ + ("samplerate", "channel_i", "channel_j", "freq", ) + _hdfFileAttributeProperties = BaseData._hdfFileAttributeProperties +\ + ("samplerate", "channel_i", "channel_j", "freq", ) _defaultDimord = ["time", "freq", "channel_i", "channel_j"] _stackingDimLabel = "time" _channel_i = None From dc56196ee88f4efe4ecedbb6ebf4db1560d9fcfa Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 29 Nov 2021 17:59:26 +0100 Subject: [PATCH 087/109] NEW: First version of __eq__ ready - wrapped up initial implementation of `__eq__` operator for Syncopy objects; formal tests still need to be implemented (closes # 14) On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py --- syncopy/datatype/base_data.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index af5fce30c..d8b7e2f07 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -788,7 +788,7 @@ def __eq__(self, other): # Start cheap: check if samplerates are identical (if present) baseSr = getattr(self, "samplerate") opndSr = getattr(other, "samplerate") - if baseSr != opndSr: + if baseSr != opndSr: SPYInfo("Mismatch in samplerates") return False @@ -797,22 +797,18 @@ def __eq__(self, other): err = "Cannot perform object comparison with existing in-place selection" raise SPYError(err) - # Use in-place selections to query class-specific dimensional properties - # (i.e., channels, freq, taper etc.) - # FIXME: don't do this; loop over `_infoFileProperties` instead and kick - # out `dimord`, `cfg` and all underscore attrs - try: - self.selectdata(inplace=True) - except: - import ipdb; ipdb.set_trace() - other.selectdata(inplace=True) + # Use `_infoFileProperties` to fetch dimensional object props: remove `dimord` + # (has already been checked by `data_parser` above) and remove `cfg` (two + # objects might be identical even if their history deviates) isEqual = True - for prop in self._selection._dimProps: - if getattr(self._selection, prop) != getattr(other._selection, prop): - SPYInfo("Mismatch in {}".format(prop)) - isEqual = False - self.selectdata(clear=True) - other.selectdata(clear=True) + dimProps = [prop for prop in self._infoFileProperties if not prop.startswith("_")] + dimProps = list(set(dimProps).difference(["dimord", "cfg"])) + for prop in dimProps: + val = getattr(self, prop) + if isinstance(val, np.ndarray): + isEqual = val.tolist() == getattr(other, prop).tolist() + else: + isEqual = val == getattr(other, prop) if not isEqual: return False From 26a03c22fb34fda4cca622856126d41ca4f04e53 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 30 Nov 2021 12:55:51 +0100 Subject: [PATCH 088/109] FIX: Testing induced bugfixes - repaired `dimProps` looping logic for property comparison; this makes an explicit samplerate comparison superfluous as well - fixed a bug in channel assignment of `SpikeData`: respect unordered channel specs - made `__eq__` compatible w/memory mapped data On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/discrete_data.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/base_data.py | 29 ++++----- syncopy/datatype/discrete_data.py | 2 +- syncopy/tests/test_basedata.py | 105 +++++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 17 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index d8b7e2f07..b0e28b865 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -781,16 +781,11 @@ def __eq__(self, other): return False # First, ensure we have something to compare here - if self._is_empty() and not other._is_empty(): - SPYInfo("Empty and non-empty Syncopy object") - return False - - # Start cheap: check if samplerates are identical (if present) - baseSr = getattr(self, "samplerate") - opndSr = getattr(other, "samplerate") - if baseSr != opndSr: - SPYInfo("Mismatch in samplerates") - return False + if self._is_empty(): + if not other._is_empty(): + SPYInfo("Empty and non-empty Syncopy object") + return False + return True # If in-place selections are present, abort if self._selection is not None or other._selection is not None: @@ -800,7 +795,6 @@ def __eq__(self, other): # Use `_infoFileProperties` to fetch dimensional object props: remove `dimord` # (has already been checked by `data_parser` above) and remove `cfg` (two # objects might be identical even if their history deviates) - isEqual = True dimProps = [prop for prop in self._infoFileProperties if not prop.startswith("_")] dimProps = list(set(dimProps).difference(["dimord", "cfg"])) for prop in dimProps: @@ -809,8 +803,9 @@ def __eq__(self, other): isEqual = val.tolist() == getattr(other, prop).tolist() else: isEqual = val == getattr(other, prop) - if not isEqual: - return False + if not isEqual: + SPYInfo("Mismatch in {}".format(prop)) + return False # Check if trial setup is identical if not np.array_equal(self.trialdefinition, other.trialdefinition): @@ -819,10 +814,14 @@ def __eq__(self, other): # If an object is compared to itself (or its shallow copy), don't bother # juggling NumPy arrays but simply perform a quick dataset/filename comparison + isEqual = True if self.filename == other.filename: for dsetName in self._hdfFileDatasetProperties: - if not getattr(self, dsetName) == getattr(other, dsetName): - isEqual = False + val = getattr(self, dsetName) + if isinstance(val, h5py.Dataset): + isEqual = val == getattr(other, dsetName) + else: + isEqual = np.allclose(val, getattr(other, dsetName)) if not isEqual: SPYInfo("HDF dataset mismatch") return False diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index 993036430..cc4f7b0f2 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -371,7 +371,7 @@ def channel(self, chan): # Remove duplicate entries from channel array but preserve original order # (e.g., `[2, 0, 0, 1]` -> `[2, 0, 1`); allows for complex subset-selections _, idx = np.unique(chan, return_index=True) - chan = np.array(chan)[idx] + chan = np.array(chan)[np.sort(idx)] nchan = np.unique(self.data[:, self.dimord.index("channel")]).size if chan.size != nchan: lgl = "channel label array of length {0:d}".format(nchan) diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index f67524a83..234a146f7 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -6,6 +6,7 @@ # Builtin/3rd party package imports import os import tempfile +from attr import has import h5py import time import pytest @@ -17,7 +18,7 @@ from syncopy.datatype import AnalogData import syncopy.datatype as spd from syncopy.datatype.base_data import VirtualData -from syncopy.shared.errors import SPYValueError, SPYTypeError +from syncopy.shared.errors import SPYValueError, SPYTypeError, SPYError from syncopy.tests.misc import is_win_vm, is_slurm_node # Construct decorators for skipping certain tests @@ -437,3 +438,105 @@ def test_arithmetic(self): operation(dummy, other) err = "expected Syncopy {} object found {}" assert err.format(dclass, otherClass) in str(spytyp.value) + + # Next, validate proper functionality of `==` operator for Syncopy objects + for dclass in self.classes: + + # Start simple compare obj to itself, to empty object and compare two empties + dummy = getattr(spd, dclass)(self.data[dclass], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + assert dummy == dummy + assert dummy != getattr(spd, dclass)() + assert getattr(spd, dclass)() == getattr(spd, dclass)() + + # Basic type mismatch + assert dummy != complexArr + assert dummy != complexNum + + # Two differing Syncopy object classes + otherClass = list(set(self.classes).difference([dclass]))[0] + other = getattr(spd, otherClass)(self.data[otherClass], + trialdefinition=self.trl[otherClass], + samplerate=self.samplerate) + assert dummy != other + + # Ensure shallow and deep copies are "==" to their origin + dummy2 = dummy.copy() + assert dummy2 == dummy + dummy3 = dummy.copy(deep=True) + assert dummy3 == dummy + + # Ensure differing samplerate evaluates to `False` + dummy3.samplerate = 2*dummy.samplerate + assert dummy3 != dummy + dummy3.samplerate = dummy.samplerate + + # In-place selections are invalid for `==` comparisons + dummy3.selectdata(inplace=True) + with pytest.raises(SPYError) as spe: + dummy3 == dummy + assert "Cannot perform object comparison" in str(spe.value) + + # Abuse existing in-place selection to alter dimensional props of dummy3 + # and ensure inequality + dimProps = dummy3._selector._dimProps + dummy3.selectdata(clear=True) + for prop in dimProps: + if hasattr(dummy3, prop): + setattr(dummy3, prop, getattr(dummy, prop)[::-1]) + assert dummy3 != dummy + setattr(dummy3, prop, getattr(dummy, prop)) + + # Different trials + dummy3 = dummy.selectdata(trials=list(range(len(dummy.trials) - 1))) + assert dummy3 != dummy + + # Different trial offsets + trl = self.trl[dclass] + trl[:, 1] -= 1 + dummy3 = getattr(spd, dclass)(self.data[dclass], + trialdefinition=trl, + samplerate=self.samplerate) + assert dummy3 != dummy + + # Different trial annotations + trl = self.trl[dclass] + trl[:, -1] = np.sqrt(2) + dummy3 = getattr(spd, dclass)(self.data[dclass], + trialdefinition=trl, + samplerate=self.samplerate) + assert dummy3 != dummy + + # Difference in actual numerical data + dummy3 = dummy.copy(deep=True) + for dsetName in dummy3._hdfFileDatasetProperties: + getattr(dummy3, dsetName)[0] = np.pi + assert dummy3 != dummy + + del dummy, dummy2, dummy3, other + + # Same objects but different dimords: `ContinuousData`` children + for dclass in continuousClasses: + dummy = getattr(spd, dclass)(self.data[dclass], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + ymmud = getattr(spd, dclass)(self.data[dclass].T, + dimord=dummy.dimord[::-1], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + assert dummy != ymmud + + # Same objects but different dimords: `DiscreteData`` children + for dclass in discreteClasses: + dummy = getattr(spd, dclass)(self.data[dclass], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + ymmud = getattr(spd, dclass)(self.data[dclass], + dimord=dummy.dimord[::-1], + trialdefinition=self.trl[dclass], + samplerate=self.samplerate) + assert dummy != ymmud + + + From ed298d7f761e45ec812a1415be124e2397887dc8 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 30 Nov 2021 14:30:45 +0100 Subject: [PATCH 089/109] FIX: Repaired trialdef for cross-covariances - since lags only take up `nSamples // 2` in time-axis, trim `trialdefinition` array correspondingly (cf #151) On branch tests Changes to be committed: modified: dev_frontend.py modified: syncopy/connectivity/AV_compRoutines.py modified: syncopy/connectivity/ST_compRoutines.py --- dev_frontend.py | 10 ++--- syncopy/connectivity/AV_compRoutines.py | 52 ++++++++++++------------- syncopy/connectivity/ST_compRoutines.py | 16 ++++---- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/dev_frontend.py b/dev_frontend.py index 20191ad88..17880ea1e 100644 --- a/dev_frontend.py +++ b/dev_frontend.py @@ -19,13 +19,13 @@ # this still gives type(tsel) = slice :) sdict1 = {"trials": [0], 'channels' : ['channel1'], 'toi': np.arange(-1, 1, 0.001)} -# no problems here.. -coherence = connectivityanalysis(data=tdat, - foilim=None, - output='pow') +# # no problems here.. +# coherence = connectivityanalysis(data=tdat, method= +# foilim=None, +# output='pow') # a lot of problems here.. -# correlation = connectivityanalysis(data=tdat, method='corr', keeptrials=False) +correlation = connectivityanalysis(data=tdat, method='corr', keeptrials=False) # the hard wired dimord of the cF diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 3e4fb9b5d..5d3a0f3aa 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -31,12 +31,12 @@ def normalize_csd_cF(trl_av_dat, output='abs', chunkShape=None, noCompute=False): - + """ Given the trial averaged cross spectral densities, - calculates the normalizations to arrive at the + calculates the normalizations to arrive at the channel x channel coherencies. If S_ij(f) is the - averaged cross-spectrum between channel i and j, the + averaged cross-spectrum between channel i and j, the coherency [1]_ is defined as: C_ij = S_ij(f) / (|S_ii| |S_jj|) @@ -51,11 +51,11 @@ def normalize_csd_cF(trl_av_dat, Cross-spectral densities for `N` x `N` channels and `nFreq` frequencies averaged over trials. output : {'abs', 'pow', 'fourier'}, default: 'abs' - Also after normalization the coherency is still complex (`'fourier'`), + Also after normalization the coherency is still complex (`'fourier'`), to get the real valued coherence 0 < C_ij(f) < 1 one can either take the absolute (`'abs'`) or the absolute squared (`'pow'`) values of the - coherencies. The definitions are not uniform in the literature, - hence multiple output types are supported. + coherencies. The definitions are not uniform in the literature, + hence multiple output types are supported. noCompute : bool Preprocessing flag. If `True`, do not perform actual calculation but instead return expected shape and :class:`numpy.dtype` of output @@ -77,8 +77,8 @@ def normalize_csd_cF(trl_av_dat, Consequently, this function does **not** perform any error checking and operates under the assumption that all inputs have been externally validated and cross-checked. - .. [1] Nolte, Guido, et al. "Identifying true brain interaction from EEG - data using the imaginary part of coherency." + .. [1] Nolte, Guido, et al. "Identifying true brain interaction from EEG + data using the imaginary part of coherency." Clinical neurophysiology 115.10 (2004): 2292-2307. @@ -100,10 +100,10 @@ def normalize_csd_cF(trl_av_dat, # re-shape to (nChannels x nChannels x nFreq) CS_ij = trl_av_dat.transpose(0, 2, 3, 1)[0, ...] - + # main diagonal has shape (nFreq x nChannels): the auto spectra diag = CS_ij.diagonal() - + # get the needed product pairs of the autospectra Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T CS_ij = CS_ij / Ciijj @@ -113,13 +113,13 @@ def normalize_csd_cF(trl_av_dat, # re-shape to original form and re-attach dummy time axis return CS_ij[None, ...].transpose(0, 3, 1, 2) - + class NormalizeCrossSpectra(ComputationalRoutine): """ Compute class that normalizes trial averaged csd's of :class:`~syncopy.CrossSpectralData` objects - to arrive at the respective coherencies. + to arrive at the respective coherencies. Sub-class of :class:`~syncopy.shared.computational_routine.ComputationalRoutine`, see :doc:`/developer/compute_kernels` for technical details on Syncopy's compute @@ -141,7 +141,7 @@ class NormalizeCrossSpectra(ComputationalRoutine): def pre_check(self): ''' - Make sure we have a trial average, + Make sure we have a trial average, so the input data only consists of `1 trial`. Can only be performed after initialization! ''' @@ -150,12 +150,12 @@ def pre_check(self): lgl = 'Initialize the computational Routine first!' act = 'ComputationalRoutine not initialized!' raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) - + if self.numTrials != 1: lgl = "1 trial: normalizations can only be done on averaged quantities!" act = f"DataSet contains {self.numTrials} trials" raise SPYValueError(legal=lgl, varname="data", actual=act) - + def process_metadata(self, data, out): # Some index gymnastics to get trial begin/end "samples" @@ -177,7 +177,7 @@ def process_metadata(self, data, out): out.trialdefinition = trl else: out.trialdefinition = np.array([[0, 1, 0]]) - + # Attach remaining meta-data out.samplerate = data.samplerate out.channel_i = np.array(data.channel_i[chanSec]) @@ -189,7 +189,7 @@ def process_metadata(self, data, out): def normalize_ccov_cF(trl_av_dat, chunkShape=None, noCompute=False): - + """ Given the trial averaged cross-covariances, we normalize with the 0-lag auto-covariances @@ -240,12 +240,12 @@ def normalize_ccov_cF(trl_av_dat, # re-shape to (nLag x nChannels x nChannels) CCov_ij = trl_av_dat[:, 0, ...] - + # main diagonal has shape (nChannels x nChannels): # the auto-covariances at 0-lag (~stds) diag = trl_av_dat[0, 0, ...].diagonal() - # get the needed product pairs + # get the needed product pairs Ciijj = np.sqrt(diag[:, None] * diag[None, :]).T CCov_ij = CCov_ij / Ciijj @@ -256,7 +256,7 @@ def normalize_ccov_cF(trl_av_dat, class NormalizeCrossCov(ComputationalRoutine): """ - Compute class that normalizes trial averaged + Compute class that normalizes trial averaged cross-covariances of :class:`~syncopy.CrossSpectralData` objects to arrive at the respective correlations @@ -278,9 +278,9 @@ class NormalizeCrossCov(ComputationalRoutine): # 1st argument,the data, gets omitted valid_kws = list(signature(normalize_ccov_cF).parameters.keys())[1:] - def pre_check(self): + def pre_check(self): ''' - Make sure we have a trial average, + Make sure we have a trial average, so the input data only consists of `1 trial`. Can only be performed after initialization! ''' @@ -289,14 +289,14 @@ def pre_check(self): lgl = 'Initialize the computational Routine first!' act = 'ComputationalRoutine not initialized!' raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) - + if self.numTrials != 1: lgl = "1 trial: normalizations can only be done on averaged quantities!" act = f"DataSet contains {self.numTrials} trials" raise SPYValueError(legal=lgl, varname="data", actual=act) - + def process_metadata(self, data, out): - + # Get trialdef array + channels from source if data._selection is not None: chanSec = data._selection.channel @@ -305,7 +305,7 @@ def process_metadata(self, data, out): chanSec = slice(None) trl = data.trialdefinition - out.trialdefinition = trl + out.trialdefinition = trl # Attach remaining meta-data out.samplerate = data.samplerate out.channel_i = np.array(data.channel_i[chanSec]) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index b635d4af6..fb672343b 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -189,7 +189,7 @@ def cross_spectra_cF(trl_dat, else: return CS_ij[None, ..., freq_idx].transpose(0, 3, 1, 2), freqs[freq_idx] - + class ST_CrossSpectra(ComputationalRoutine): """ @@ -236,7 +236,7 @@ def process_metadata(self, data, out): out.trialdefinition = trl else: out.trialdefinition = np.array([[0, 1, 0]]) - + # Attach remaining meta-data out.samplerate = data.samplerate out.channel_i = np.array(data.channel[chanSec]) @@ -399,13 +399,15 @@ class ST_CrossCovariance(ComputationalRoutine): def process_metadata(self, data, out): - # Get trialdef array + channels from source + # Get trialdef array + channels from source: note, since lags are encoded + # in time-axis, trial offsets etc. are bogus anyway: simply take max-sample + # counts / 2 to fit lags if data._selection is not None: chanSec = data._selection.channel - trl = data._selection.trialdefinition + trl = np.ceil(data._selection.trialdefinition / 2) else: chanSec = slice(None) - trl = data.trialdefinition + trl = np.ceil(data.trialdefinition / 2) # If trial-averaging was requested, use the first trial as reference # (all trials had to have identical lengths), and average onset timings @@ -414,10 +416,10 @@ def process_metadata(self, data, out): trl = trl[[0], :] trl[:, 2] = t0 - out.trialdefinition = trl + out.trialdefinition = trl # Attach remaining meta-data out.samplerate = data.samplerate out.channel_i = np.array(data.channel[chanSec]) out.channel_j = np.array(data.channel[chanSec]) - + From 0aa3fce8d1d500317f3fba2612ee34f21a39cfbe Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 30 Nov 2021 19:57:48 +0100 Subject: [PATCH 090/109] WIP: Started working on synthetic data generation - first foray into #147 On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py --- syncopy/datatype/base_data.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index b0e28b865..2f13ee626 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -162,6 +162,7 @@ def _set_dataset_property(self, dataIn, propertyName, ndim=None): ndim = len(self._defaultDimord) supportedSetters = { + list : self._set_dataset_property_with_list, str : self._set_dataset_property_with_str, np.ndarray : self._set_dataset_property_with_ndarray, np.core.memmap : self._set_dataset_property_with_memmap, @@ -335,6 +336,18 @@ def _set_dataset_property_with_dataset(self, inData, propertyName, ndim): setattr(self, "_" + propertyName, inData) + def _set_dataset_property_with_list(self, inData, propertyName, ndim): + """ + Coming soon... + """ + + if any(not isinstance(val, np.ndarray) for val in inData): + lgl = "list of NumPy arrays" + act = "mixed element list" + raise SPYValueError(legal=lgl, varname="data", actual=act) + + pass + def _is_empty(self): return all([getattr(self, attr) is None for attr in self._hdfFileDatasetProperties]) From 917c872b4726f4281ce04adbff2560d53b80c81f Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 1 Dec 2021 16:02:58 +0100 Subject: [PATCH 091/109] WIP: More prep work - laid out general structure of init routine On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py --- syncopy/datatype/base_data.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 2f13ee626..d606e0971 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -341,12 +341,34 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): Coming soon... """ - if any(not isinstance(val, np.ndarray) for val in inData): - lgl = "list of NumPy arrays" - act = "mixed element list" + # Check list entries: must be numeric, finite NumPy arrays + for val in inData: + try: + array_parser(val, varname="data", hasinf=False, dims=ndim) + except Exception as exc: + raise exc + + # Ensure we don't have a mix of real/complex arrays + if np.unique([np.iscomplexobj(val) for val in inData]).size > 1: + lgl = "list of numeric NumPy arrays of same numeric type (real/complex)" + act = "real and complex NumPy arrays" raise SPYValueError(legal=lgl, varname="data", actual=act) - pass + # Ensure shapes match up + if any(val.shape != inData[0].shape for val in inData): + lgl = "NumPy arrays of identical shape" + act = "NumPy arrays with differing shapes" + raise SPYValueError(legal=lgl, varname="data", actual=act) + + import ipdb; ipdb.set_trace() + + # use stackingdim to build data-array! + if self.__class__.__name__ == "AnalogData": + pass # use np.h/vstack here + else: + pass # allocate ndarray + + def _is_empty(self): return all([getattr(self, attr) is None From 57cdbc831546a9b15e895a4edf98441a7d3d4ca0 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 2 Dec 2021 12:47:57 +0100 Subject: [PATCH 092/109] NEW: First implementation of synthetic data initialization - new data setting method `_set_dataset_property_with_list` processes lists of NumPy arrays representing trials - since only time-series data is submitted to the setter, all timing information has to be made up from scratch (specifically samplerate and trigger offset); closes #147 On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/tests/spy_setup.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/base_data.py | 51 ++++++++++++++++++++++++++-------- syncopy/tests/spy_setup.py | 9 ++++++ syncopy/tests/test_basedata.py | 1 + 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index d606e0971..e62709ace 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -31,7 +31,7 @@ from syncopy.shared.tools import StructDict from syncopy.shared.parsers import (scalar_parser, array_parser, io_parser, filename_parser, data_parser) -from syncopy.shared.errors import SPYInfo, SPYTypeError, SPYValueError, SPYError +from syncopy.shared.errors import SPYInfo, SPYTypeError, SPYValueError, SPYError, SPYWarning from syncopy.datatype.methods.definetrial import definetrial as _definetrial from syncopy import __version__, __storage__, __acme__, __sessionid__, __storagelimit__ if __acme__: @@ -354,21 +354,48 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): act = "real and complex NumPy arrays" raise SPYValueError(legal=lgl, varname="data", actual=act) - # Ensure shapes match up - if any(val.shape != inData[0].shape for val in inData): - lgl = "NumPy arrays of identical shape" - act = "NumPy arrays with differing shapes" - raise SPYValueError(legal=lgl, varname="data", actual=act) + # Requirements for input arrays differ wrt data-class (`DiscreteData` always 2D) + if any(["ContinuousData" in str(base) for base in self.__class__.__mro__]): - import ipdb; ipdb.set_trace() + # Ensure shapes match up + if any(val.shape != inData[0].shape for val in inData): + lgl = "NumPy arrays of identical shape" + act = "NumPy arrays with differing shapes" + raise SPYValueError(legal=lgl, varname="data", actual=act) + trialLens = [val.shape[self.dimord.index("time")] for val in inData] - # use stackingdim to build data-array! - if self.__class__.__name__ == "AnalogData": - pass # use np.h/vstack here else: - pass # allocate ndarray - + # Ensure all arrays have shape `(N, 3)`` + if any(val.shape[1] != 3 for val in inData): + lgl = "NumPy 2d-arrays with 3 columns" + act = "NumPy arrays of different shape" + raise SPYValueError(legal=lgl, varname="data", actual=act) + trialLens = [np.nanmax(val[:, self.dimord.index("sample")]) for val in inData] + + # Now the shaky stuff: use determined trial lengths to cook up a (completely + # fictional) samplerate: we aim for `smax` Hz and round down to `sround` Hz + nTrials = len(trialLens) + sround = 50 + smax = 1000 + srate = min(max(min(smax, tlen / 2) // sround * sround, 1) for tlen in trialLens) + t0 = -srate + msg = "Artificially generated trial-layout: trigger offset = {t0} sec, " +\ + "samplerate = {srate} Hz (rounded to {sround} Hz with max of {smax} Hz)" + SPYWarning(msg.format(t0=t0/srate, srate=srate, sround=sround, smax=smax), caller="data") + + # Use constructed quantities to set up trial layout matrix + accumSamples = np.cumsum(trialLens) + trialdefinition = np.zeros((nTrials, 3)) + trialdefinition[1:, 0] = accumSamples[:-1] + trialdefinition[:, 1] = accumSamples + trialdefinition[:, 2] = t0 + + # Finally, concatenate provided arrays and let corresponding setting method + # perform the actual HDF magic + data = np.concatenate(inData, axis=self._stackingDim) + self._set_dataset_property_with_ndarray(data, propertyName, ndim) + self.trialdefinition = trialdefinition def _is_empty(self): return all([getattr(self, attr) is None diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 2830ffe86..188345f82 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -27,6 +27,15 @@ data2 = generate_artificial_data(nTrials=5, nChannels=16, equidistant=True, inmemory=False) + nSamples = 1000 + nChannels = 50 + my_noise = np.random.randn(nSamples, nChannels) + + trl_dat = [my_noise, 5 * my_noise + 10, np.random.randn(nSamples, nChannels)] + + spy.AnalogData(trl_dat) + + # client = spy.esi_cluster_setup(interactive=False) # data1 + data2 diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index 234a146f7..4d41b0425 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -383,6 +383,7 @@ def test_arithmetic(self): dummy = getattr(spd, dclass)(self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) + for operation in arithmetics: with pytest.raises(SPYTypeError) as spytyp: operation(dummy, 2) From 60954ae9c3fdcc9692949d915ddb2c77b2034c29 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 2 Dec 2021 12:58:19 +0100 Subject: [PATCH 093/109] FIX: Cleaned up hierarchical class attribute assignment - test-driving the new `_set_dataset_property_with_list` method revealed that data assignment is accidentally performed twice in abstract class tree: once in `BaseData` (via one of the `set_dataset_property_with*`) and then again in the constructor of `ContinuousData` and `DiscreteData`, respectively. The second invocation has been removed (only relevant if synthetic data is generated via list of NumPy arrays) - fixed display of `dimord` in `DiscreteData` (cf. #89) and removed unnecessary if-clause in `__str__` of `AnalogData` On branch tests Changes to be committed: modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/discrete_data.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/continuous_data.py | 3 --- syncopy/datatype/discrete_data.py | 16 ++++++++-------- syncopy/tests/test_basedata.py | 1 - 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index acc4a1ed7..7fe98c6a2 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -81,8 +81,6 @@ def __str__(self): printString = "{0:>" + str(maxKeyLength + 5) + "} : {1:}\n" for attr in ppattrs: value = getattr(self, attr) - if attr == "dimord" and value is not None: - valueString = dsep.join(dim for dim in self.dimord) if hasattr(value, 'shape') and attr == "data" and self.sampleinfo is not None: tlen = np.unique([sinfo[1] - sinfo[0] for sinfo in self.sampleinfo]) if tlen.size == 1: @@ -379,7 +377,6 @@ def __init__(self, data=None, channel=None, samplerate=None, **kwargs): self.channel = channel self.samplerate = samplerate # use setter for error-checking - self.data = data if self.data is not None: diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index cc4f7b0f2..da7e8984d 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -67,17 +67,15 @@ def __str__(self): ppattrs.sort() # Construct string for pretty-printing class attributes - dinfo = " '" + self._classname_to_extension()[1:] + "' x " - dsep = "'-'" - + dsep = " by " hdstr = "Syncopy {clname:s} object with fields\n\n" - ppstr = hdstr.format(diminfo=dinfo + "'" + \ - dsep.join(dim for dim in self.dimord) + "' " if self.dimord is not None else "Empty ", - clname=self.__class__.__name__) + ppstr = hdstr.format(clname=self.__class__.__name__) maxKeyLength = max([len(k) for k in ppattrs]) printString = "{0:>" + str(maxKeyLength + 5) + "} : {1:}\n" for attr in ppattrs: value = getattr(self, attr) + # if attr == "dimord" and value is not None: + # valueString = dsep.join(dim for dim in self.dimord) if hasattr(value, 'shape') and attr == "data" and self.sampleinfo is not None: tlen = np.unique([sinfo[1] - sinfo[0] for sinfo in self.sampleinfo]) if tlen.size == 1: @@ -101,7 +99,10 @@ def __str__(self): valueString = "[" + " x ".join([str(numel) for numel in value.shape]) \ + "] element " + str(type(value)) elif isinstance(value, list): - valueString = "{0} element list".format(len(value)) + if attr == "dimord" and value is not None: + valueString = dsep.join(dim for dim in self.dimord) + else: + valueString = "{0} element list".format(len(value)) elif isinstance(value, dict): msg = "dictionary with {nk:s}keys{ks:s}" keylist = value.keys() @@ -315,7 +316,6 @@ def __init__(self, data=None, samplerate=None, trialid=None, **kwargs): self.samplerate = samplerate self.trialid = trialid - self.data = data if self.data is not None: diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index 4d41b0425..234a146f7 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -383,7 +383,6 @@ def test_arithmetic(self): dummy = getattr(spd, dclass)(self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) - for operation in arithmetics: with pytest.raises(SPYTypeError) as spytyp: operation(dummy, 2) From 5cc4e288d9b1eb5e2b53c1c4128714bfe82f0bc7 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 2 Dec 2021 14:46:33 +0100 Subject: [PATCH 094/109] FIX: More class attribute fixes - new `_set_dataset_property_with_list` incorrectly assumed all `DiscreteData` children data arrays have 3 columns; this has been fixed - actually assign artificially computed samplerate to object in `_set_dataset_property_with_list` - to do this, don't override `samplerate` in constructors of `ContinuousData` and `DiscreteData` - don't check for empty `cfg` in constructors of `ContinuousData` and `DiscreteData` to decide whether to generate dummy trial definition. Look for `sampleinfo` instead On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/datatype/continuous_data.py modified: syncopy/datatype/discrete_data.py --- syncopy/datatype/base_data.py | 31 +++++++++++++++++++---------- syncopy/datatype/continuous_data.py | 5 +++-- syncopy/datatype/discrete_data.py | 10 ++++------ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index e62709ace..c7537bdc0 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -366,23 +366,32 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): else: - # Ensure all arrays have shape `(N, 3)`` - if any(val.shape[1] != 3 for val in inData): + # Ensure all arrays have shape `(N, nCol)`` + if self.__class__.__name__ == "SpikeData": + nCol = 3 + else: # EventData + nCol = 2 + if any(val.shape[1] != nCol for val in inData): lgl = "NumPy 2d-arrays with 3 columns" act = "NumPy arrays of different shape" raise SPYValueError(legal=lgl, varname="data", actual=act) trialLens = [np.nanmax(val[:, self.dimord.index("sample")]) for val in inData] - # Now the shaky stuff: use determined trial lengths to cook up a (completely - # fictional) samplerate: we aim for `smax` Hz and round down to `sround` Hz + # Now the shaky stuff: if not provided, use determined trial lengths to + # cook up a (completely fictional) samplerate: we aim for `smax` Hz and + # round down to `sround` Hz nTrials = len(trialLens) - sround = 50 - smax = 1000 - srate = min(max(min(smax, tlen / 2) // sround * sround, 1) for tlen in trialLens) - t0 = -srate - msg = "Artificially generated trial-layout: trigger offset = {t0} sec, " +\ - "samplerate = {srate} Hz (rounded to {sround} Hz with max of {smax} Hz)" - SPYWarning(msg.format(t0=t0/srate, srate=srate, sround=sround, smax=smax), caller="data") + msg2 = "" + if self.samplerate is None: + sround = 50 + smax = 1000 + srate = min(max(min(smax, tlen / 2) // sround * sround, 1) for tlen in trialLens) + self.samplerate = srate + msg2 = ", samplerate = {srate} Hz (rounded to {sround} Hz with max of {smax} Hz)" + msg2 = msg2.format(srate=srate, sround=sround, smax=smax) + t0 = -self.samplerate + msg = "Artificially generated trial-layout: trigger offset = {t0} sec" + msg2 + SPYWarning(msg.format(t0=t0/self.samplerate), caller="data") # Use constructed quantities to set up trial layout matrix accumSamples = np.cumsum(trialLens) diff --git a/syncopy/datatype/continuous_data.py b/syncopy/datatype/continuous_data.py index 7fe98c6a2..485db60c6 100644 --- a/syncopy/datatype/continuous_data.py +++ b/syncopy/datatype/continuous_data.py @@ -372,17 +372,18 @@ def __init__(self, data=None, channel=None, samplerate=None, **kwargs): self._samplerate = None self._data = None + self.samplerate = samplerate # use setter for error-checking + # Call initializer super().__init__(data=data, **kwargs) self.channel = channel - self.samplerate = samplerate # use setter for error-checking if self.data is not None: # In case of manual data allocation (reading routine would leave a # mark in `cfg`), fill in missing info - if len(self.cfg) == 0: + if self.sampleinfo is None: # First, fill in dimensional info definetrial(self, kwargs.get("trialdefinition")) diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index da7e8984d..c8ae89521 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -74,8 +74,6 @@ def __str__(self): printString = "{0:>" + str(maxKeyLength + 5) + "} : {1:}\n" for attr in ppattrs: value = getattr(self, attr) - # if attr == "dimord" and value is not None: - # valueString = dsep.join(dim for dim in self.dimord) if hasattr(value, 'shape') and attr == "data" and self.sampleinfo is not None: tlen = np.unique([sinfo[1] - sinfo[0] for sinfo in self.sampleinfo]) if tlen.size == 1: @@ -311,17 +309,17 @@ def __init__(self, data=None, samplerate=None, trialid=None, **kwargs): self._hdr = None self._data = None - # Call initializer - super().__init__(data=data, **kwargs) - self.samplerate = samplerate self.trialid = trialid + # Call initializer + super().__init__(data=data, **kwargs) + if self.data is not None: # In case of manual data allocation (reading routine would leave a # mark in `cfg`), fill in missing info - if len(self.cfg) == 0: + if self.sampleinfo is None: # Fill in dimensional info definetrial(self, kwargs.get("trialdefinition")) From f6c22ac52c3c77fa54a192e81444a189bc550efe Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Thu, 2 Dec 2021 18:09:28 +0100 Subject: [PATCH 095/109] NEW: Tests + docstring - included docstring for `_set_dataset_property_with_list` - wrote basal tests for new data allocation method On branch tests Changes to be committed: modified: syncopy/datatype/base_data.py modified: syncopy/tests/spy_setup.py modified: syncopy/tests/test_basedata.py --- syncopy/datatype/base_data.py | 13 +++++++++++-- syncopy/tests/spy_setup.py | 2 +- syncopy/tests/test_basedata.py | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index c7537bdc0..32a0c325d 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -337,8 +337,17 @@ def _set_dataset_property_with_dataset(self, inData, propertyName, ndim): setattr(self, "_" + propertyName, inData) def _set_dataset_property_with_list(self, inData, propertyName, ndim): - """ - Coming soon... + """Set a dataset property with list of NumPy arrays + + Parameters + ---------- + inData : list + list of :class:`numpy.ndarray`s. Each array corresponds to + a trial. Arrays are stacked together to fill dataset. + propertyName : str + Name of the property to be filled with the concatenated array + ndim : int + Number of expected array dimensions. """ # Check list entries: must be numeric, finite NumPy arrays diff --git a/syncopy/tests/spy_setup.py b/syncopy/tests/spy_setup.py index 188345f82..4ddd4ce0c 100644 --- a/syncopy/tests/spy_setup.py +++ b/syncopy/tests/spy_setup.py @@ -33,7 +33,7 @@ trl_dat = [my_noise, 5 * my_noise + 10, np.random.randn(nSamples, nChannels)] - spy.AnalogData(trl_dat) + aa = spy.AnalogData(trl_dat) # client = spy.esi_cluster_setup(interactive=False) diff --git a/syncopy/tests/test_basedata.py b/syncopy/tests/test_basedata.py index 234a146f7..909ed8795 100644 --- a/syncopy/tests/test_basedata.py +++ b/syncopy/tests/test_basedata.py @@ -244,6 +244,33 @@ def test_data_alloc(self): with pytest.raises(SPYValueError): getattr(spd, dclass)(data=open_memmap(fname)) + # ensure synthetic data allocation via list of arrays works + dummy = getattr(spd, dclass)(data=[self.data[dclass], self.data[dclass]]) + assert len(dummy.trials) == 2 + + dummy = getattr(spd, dclass)(data=[self.data[dclass], self.data[dclass]], + samplerate=10.0) + assert len(dummy.trials) == 2 + assert dummy.samplerate == 10 + + if any(["ContinuousData" in str(base) for base in self.__class__.__mro__]): + nChan = self.data[dclass].shape[dummy.dimord.index("channel")] + dummy = getattr(spd, dclass)(data=[self.data[dclass], self.data[dclass]], + channel=['label']*nChan) + assert len(dummy.trials) == 2 + assert np.array_equal(dummy.channel, np.array(['label']*nChan)) + + # the most egregious input errors are caught by `array_parser`; only + # test list-routine-specific stuff: complex/real mismatch + with pytest.raises(SPYValueError) as spyval: + getattr(spd, dclass)(data=[self.data[dclass], np.complex64(self.data[dclass])]) + assert "same numeric type (real/complex)" in str(spyval.value) + + # shape mismatch + with pytest.raises(SPYValueError): + getattr(spd, dclass)(data=[self.data[dclass], self.data[dclass].T]) + + time.sleep(0.01) del dummy @@ -373,8 +400,8 @@ def test_copy(self): def test_arithmetic(self): # Define list of classes arithmetic ops should and should not work with - # FIXME: include `CrossSpectralData` here! - # continuousClasses = ["AnalogData", "SpectralData", "CrossSpectralData"] + # FIXME: include `CrossSpectralData` here and use something like + # if any(["ContinuousData" in str(base) for base in self.__class__.__mro__]) continuousClasses = ["AnalogData", "SpectralData"] discreteClasses = ["SpikeData", "EventData"] @@ -527,7 +554,7 @@ def test_arithmetic(self): samplerate=self.samplerate) assert dummy != ymmud - # Same objects but different dimords: `DiscreteData`` children + # Same objects but different dimords: `DiscreteData` children for dclass in discreteClasses: dummy = getattr(spd, dclass)(self.data[dclass], trialdefinition=self.trl[dclass], From 7478257669adf862397d90c2a94a780ab02a1c23 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 13 Dec 2021 16:30:29 +0100 Subject: [PATCH 096/109] CHG : doc/comment cosmetics --- syncopy/shared/const_def.py | 4 ++-- syncopy/shared/input_validators.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/syncopy/shared/const_def.py b/syncopy/shared/const_def.py index dfecad7de..468d45e79 100644 --- a/syncopy/shared/const_def.py +++ b/syncopy/shared/const_def.py @@ -18,12 +18,12 @@ "abs": lambda x: (np.absolute(x)).real.astype(np.float32)} -#: available tapers of :func:`~syncopy.freqanalysis` +#: available tapers of :func:`~syncopy.freqanalysis` and :func:`~syncopy.connectivity` all_windows = windows.__all__ all_windows.remove("exponential") # not symmetric all_windows.remove("hanning") # deprecated availableTapers = all_windows -#: general, method agnostic, parameters of :func:`~syncopy.freqanalysis` +#: general, method agnostic, parameters for our CRs generalParameters = ("method", "output", "keeptrials","samplerate", "foi", "foilim", "polyremoval", "out") diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index a511e1f43..ed5e42507 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -225,8 +225,8 @@ def check_effective_parameters(CR, defaults, lcls): ''' For a given ComputationalRoutine, compare set parameters - (*lcls*) with the accepted parameters and the *defaults* - to warn if any ineffective parameters are set. + (*lcls*) with the accepted parameters and the frontend + meta function *defaults* to warn if any ineffective parameters are set. Parameters ---------- From 61f2ee3a14f7b054aa8f814109bc60544798ed1e Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 14 Dec 2021 11:44:46 +0100 Subject: [PATCH 097/109] FIX: Updated erroneous channel selection processing - do not use `data._selection.channel` in `process_metadata` of the `AV_*` routines, since `data` is a `CrossSpectralData` object On branch tests Changes to be committed: modified: syncopy/connectivity/AV_compRoutines.py --- syncopy/connectivity/AV_compRoutines.py | 68 ++++++++++++++----------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 49a399e02..37d91f503 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -103,7 +103,7 @@ def normalize_csd_cF(csd_av_dat, # re-shape to (nChannels x nChannels x nFreq) CS_ij = csd_av_dat.transpose(0, 2, 3, 1)[0, ...] - + # main diagonal has shape (nFreq x nChannels): the auto spectra diag = CS_ij.diagonal() @@ -163,12 +163,14 @@ def process_metadata(self, data, out): # Some index gymnastics to get trial begin/end "samples" if data._selection is not None: - chanSec = data._selection.channel + chanSec_i = data._selection.channel_i + chanSec_j = data._selection.channel_j trl = data._selection.trialdefinition for row in range(trl.shape[0]): trl[row, :2] = [row, row + 1] else: - chanSec = slice(None) + chanSec_i = slice(None) + chanSec_j = slice(None) time = np.arange(len(data.trials)) time = time.reshape((time.size, 1)) trl = np.hstack((time, time + 1, @@ -183,8 +185,8 @@ def process_metadata(self, data, out): # Attach remaining meta-data out.samplerate = data.samplerate - out.channel_i = np.array(data.channel_i[chanSec]) - out.channel_j = np.array(data.channel_j[chanSec]) + out.channel_i = np.array(data.channel_i[chanSec_i]) + out.channel_j = np.array(data.channel_j[chanSec_j]) out.freq = data.freq @@ -302,17 +304,19 @@ def process_metadata(self, data, out): # Get trialdef array + channels from source if data._selection is not None: - chanSec = data._selection.channel + chanSec_i = data._selection.channel_i + chanSec_j = data._selection.channel_j trl = data._selection.trialdefinition else: - chanSec = slice(None) + chanSec_i = slice(None) + chanSec_j = slice(None) trl = data.trialdefinition out.trialdefinition = trl # Attach remaining meta-data out.samplerate = data.samplerate - out.channel_i = np.array(data.channel_i[chanSec]) - out.channel_j = np.array(data.channel_j[chanSec]) + out.channel_i = np.array(data.channel_i[chanSec_i]) + out.channel_j = np.array(data.channel_j[chanSec_j]) @unwrap_io @@ -322,7 +326,7 @@ def granger_cF(csd_av_dat, cond_max=1e6, chunkShape=None, noCompute=False): - + """ Given the trial averaged cross spectral densities, calculates the pairwise Granger-Geweke causalities @@ -339,8 +343,8 @@ def granger_cF(csd_av_dat, Critical numerical parameters for Wilson's algorithm (`rtol`, `nIter`, `cond_max`) have sensitive defaults, - which were tested for datasets with up to - 5000 samples and 256 channels. Changing them is + which were tested for datasets with up to + 5000 samples and 256 channels. Changing them is recommended for expert users only. Parameters @@ -350,7 +354,7 @@ def granger_cF(csd_av_dat, and `nFreq` frequencies averaged over trials. rtol : float Relative error tolerance for Wilson's algorithm - for spectral matrix factorization. Default should + for spectral matrix factorization. Default should be fine for most cases, handle with care! nIter : int Maximum Number of iterations for CSD factorization. A result @@ -360,8 +364,8 @@ def granger_cF(csd_av_dat, The CSD matrix can be almost singular in cases of many channels and low sample number. In these cases Wilson's factorization fails to converge, as it relies on positive definiteness of the CSD matrix. - If the condition number is above `cond_max`, a brute force - regularization is performed until the regularized CSD matrix has a + If the condition number is above `cond_max`, a brute force + regularization is performed until the regularized CSD matrix has a condition number below `cond_max`. noCompute : bool Preprocessing flag. If `True`, do not perform actual calculation but @@ -386,8 +390,8 @@ def granger_cF(csd_av_dat, Consequently, this function does **not** perform any error checking and operates under the assumption that all inputs have been externally validated and cross-checked. - .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. - "Estimating Granger causality from Fourier and wavelet transforms + .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms of time series data." Physical review letters 100.1 (2008): 018701. See also @@ -397,16 +401,16 @@ def granger_cF(csd_av_dat, can be obtained by calling the respective computational routine with `keeptrials=False`. wilson_sf : :func:`~syncopy.connectivity.wilson_sf.wilson_sf - Spectral matrix factorization that yields the + Spectral matrix factorization that yields the transfer functions and noise covariances from a cross spectral density. regularize_csd : :func:`~syncopy.connectivity.wilson_sf.regularize_csd Brute force regularization scheme for the CSD matrix granger : :func:`~syncopy.connectivity.granger.granger - Given the results of the spectral matrix + Given the results of the spectral matrix factorization, calculates the granger causalities """ - + # it's the same as the input shape! outShape = csd_av_dat.shape @@ -425,10 +429,10 @@ def granger_cF(csd_av_dat, # if this is not enough! CSDreg, factor = regularize_csd(CSD, cond_max=cond_max, eps_max=1e-3) # call Wilson - + H, Sigma, conv = wilson_sf(CSDreg, nIter=nIter, rtol=rtol) - - # calculate G-causality + + # calculate G-causality Granger = granger(CSDreg, H, Sigma) # reattach dummy time axis @@ -461,7 +465,7 @@ class GrangerCausality(ComputationalRoutine): def pre_check(self): ''' - Make sure we have a trial average, + Make sure we have a trial average, so the input data only consists of `1 trial`. Can only be performed after initialization! ''' @@ -470,22 +474,24 @@ def pre_check(self): lgl = 'Initialize the computational Routine first!' act = 'ComputationalRoutine not initialized!' raise SPYValueError(legal=lgl, varname=self.__class__.__name__, actual=act) - + if self.numTrials != 1: lgl = "1 trial: Granger causality can only be computed on trial averages!" act = f"DataSet contains {self.numTrials} trials" raise SPYValueError(legal=lgl, varname="data", actual=act) - + def process_metadata(self, data, out): # Some index gymnastics to get trial begin/end "samples" if data._selection is not None: - chanSec = data._selection.channel + chanSec_i = data._selection.channel_i + chanSec_j = data._selection.channel_j trl = data._selection.trialdefinition for row in range(trl.shape[0]): trl[row, :2] = [row, row + 1] else: - chanSec = slice(None) + chanSec_i = slice(None) + chanSec_j = slice(None) time = np.arange(len(data.trials)) time = time.reshape((time.size, 1)) trl = np.hstack((time, time + 1, @@ -497,9 +503,9 @@ def process_metadata(self, data, out): out.trialdefinition = trl else: out.trialdefinition = np.array([[0, 1, 0]]) - + # Attach remaining meta-data out.samplerate = data.samplerate - out.channel_i = np.array(data.channel_i[chanSec]) - out.channel_j = np.array(data.channel_j[chanSec]) + out.channel_i = np.array(data.channel_i[chanSec_i]) + out.channel_j = np.array(data.channel_j[chanSec_j]) out.freq = data.freq From 71ed7fe50d36224f8574148eba3dd6df15f9e8d6 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 14 Dec 2021 15:41:45 +0100 Subject: [PATCH 098/109] WIP : Connectivity frontend log - added log_dict to connectivity_analysis.py - logging information propagatres from data creation to the ST-CRs and finally to the AV-CRs - extended logging the actually used foi also in freqanalysis.py Changes to be committed: modified: syncopy/connectivity/connectivity_analysis.py modified: syncopy/specest/freqanalysis.py --- syncopy/connectivity/connectivity_analysis.py | 38 +++++++++++-------- syncopy/specest/freqanalysis.py | 37 ++++++++++-------- 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index d173878a7..f3dada1a6 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -8,7 +8,7 @@ from numbers import Number # Syncopy imports -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.parsers import data_parser, scalar_parser from syncopy.shared.tools import get_defaults from syncopy.datatype import CrossSpectralData from syncopy.datatype.methods.padding import _nextpow2 @@ -19,7 +19,6 @@ SPYInfo) from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, detect_parallel_client) -from syncopy.shared.tools import best_match from syncopy.shared.input_validators import ( validate_taper, validate_foi, @@ -40,8 +39,7 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", foi=None, foilim=None, pad_to_length=None, polyremoval=None, taper="hann", tapsmofrq=None, - nTaper=None, toi="all", - out=None, **kwargs): + nTaper=None, out=None, **kwargs): """ coming soon.. @@ -58,13 +56,14 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", # Get everything of interest in local namespace defaults = get_defaults(connectivity) lcls = locals() - check_passed_kwargs(lcls, defaults, "connectivity") + # check for ineffective additional kwargs + check_passed_kwargs(lcls, defaults, frontend_name="connectivity") # Ensure a valid computational method was selected if method not in availableMethods: lgl = "'" + "or '".join(opt + "' " for opt in availableMethods) raise SPYValueError(legal=lgl, varname="method", actual=method) - + # If only a subset of `data` is to be processed, # make some necessary adjustments # and compute minimal sample-count across (selected) trials @@ -135,6 +134,14 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", if foilim: foi = np.arange(foilim[0], foilim[1] + 1) + # Prepare keyword dict for logging (use `lcls` to get actually provided + # keyword values, not defaults set above) + log_dict = {"method": method, + "output": output, + "keeptrials": keeptrials, + "polyremoval": polyremoval, + "pad_to_length": pad_to_length} + # --- Setting up specific Methods --- if method in ['coh', 'granger']: @@ -163,6 +170,13 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", samplerate=data.samplerate, nSamples=nSamples, output="pow") # ST_CSD's always have this unit/norm + + log_dict["foi"] = foi + log_dict["taper"] = taper + # only dpss returns non-empty taper_opt dict + if taper_opt: + log_dict["nTaper"] = taper_opt["Kmax"] + log_dict["tapsmofrq"] = tapsmofrq check_effective_parameters(ST_CrossSpectra, defaults, lcls) # parallel computation over trials @@ -212,15 +226,7 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", st_out._stackingDim, chan_per_worker=None, # no parallelisation over channel possible keeptrials=keeptrials) # we most likely need trial averaging! - st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict={}) - - # for debugging ccov - # print(5*'#',' after st_compRoutine call! ', 5*'#') - # print(st_out) - # print(st_out.trialdefinition) - # print(len(st_out.trials)) - # print(st_out.sampleinfo) - # return st_out + st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict=log_dict) # ---------------------------------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine on the averaged ST output @@ -242,7 +248,7 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", # now take the trial average from the single trial CR as input av_compRoutine.initialize(st_out, out._stackingDim, chan_per_worker=None) av_compRoutine.pre_check() # make sure we got a trial_average - av_compRoutine.compute(st_out, out, parallel=False) + av_compRoutine.compute(st_out, out, parallel=False, log_dict=log_dict) # Either return newly created output object or simply quit return out if new_out else None diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index e42f59e97..21153a499 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -326,7 +326,8 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Get everything of interest in local namespace defaults = get_defaults(freqanalysis) lcls = locals() - check_passed_kwargs(lcls, defaults, "freqanalysis") + # check for ineffective additional kwargs + check_passed_kwargs(lcls, defaults, frontend_name="freqanalysis") # Ensure a valid computational method was selected if method not in availableMethods: @@ -442,8 +443,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', "polyremoval": polyremoval, "pad": lcls["pad"], "padtype": lcls["padtype"], - "padlength": lcls["padlength"], - "foi": lcls["foi"]} + "padlength": lcls["padlength"]} # -------------------------------- # 1st: Check time-frequency inputs @@ -548,7 +548,8 @@ def freqanalysis(data, method='mtmfft', output='fourier', f"{freqs[-1]:.1f}Hz") SPYInfo(msg) foi = freqs - + log_dct["foi"] = foi + # Abort if desired frequency selection is empty if foi.size == 0: lgl = "non-empty frequency specification" @@ -557,18 +558,20 @@ def freqanalysis(data, method='mtmfft', output='fourier', # sanitize taper selection and retrieve dpss settings taper_opt = validate_taper(taper, - tapsmofrq, - nTaper, - keeptapers, - foimax=foi.max(), - samplerate=data.samplerate, - nSamples=minSampleNum, - output=output) - - # Update `log_dct` w/method-specific options (use `lcls` to get actually - # provided keyword values, not defaults set in here) - log_dct["taper"] = lcls["taper"] - log_dct["tapsmofrq"] = lcls["tapsmofrq"] + tapsmofrq, + nTaper, + keeptapers, + foimax=foi.max(), + samplerate=data.samplerate, + nSamples=minSampleNum, + output=output) + + # Update `log_dct` w/method-specific options + log_dct["taper"] = taper + # only dpss returns non-empty taper_opt dict + if taper_opt: + log_dct["nTaper"] = taper_opt["Kmax"] + log_dct["tapsmofrq"] = tapsmofrq # ------------------------------------------------------- # Now, prepare explicit compute-classes for chosen method @@ -813,6 +816,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', # Update `log_dct` w/method-specific options (use `lcls` to get actually # provided keyword values, not defaults set in here) + log_dct["foi"] = foi log_dct["wavelet"] = lcls["wavelet"] log_dct["width"] = lcls["width"] log_dct["order"] = lcls["order"] @@ -888,6 +892,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', SPYWarning(msg) scales = np.sort(scales)[::-1] + log_dct["foi"] = foi log_dct["c_1"] = lcls["c_1"] log_dct["order_max"] = lcls["order_max"] log_dct["order_min"] = lcls["order_min"] From 464b5e35e386a5c867a48fc1dff26ac7c877431f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 14 Dec 2021 16:04:12 +0100 Subject: [PATCH 099/109] FIX : Remove obsolete csd.py from /specest Changes to be committed: deleted: syncopy/specest/csd.py --- syncopy/specest/csd.py | 78 ------------------------------------------ 1 file changed, 78 deletions(-) delete mode 100644 syncopy/specest/csd.py diff --git a/syncopy/specest/csd.py b/syncopy/specest/csd.py deleted file mode 100644 index 791008dca..000000000 --- a/syncopy/specest/csd.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Cross Spectral Density -# - -# Builtin/3rd party package imports -import numpy as np - -# syncopy imports -from syncopy.specest.mtmfft import mtmfft - - -def cross_spectra(data_arr, samplerate, taper="hann", taperopt={}): - - """ - Single trial cross spectra estimates between all channels - of the input data. First all the individual Fourier transforms - are calculated via a (multi-)tapered FFT, then the pairwise - cross-spectra are calculated. Averaging over tapers is done implicitly. - Output consists of all (nChannels x nChannels+1)/2 different estimates - aranged in a symmetric fashion (CS_ij == CS_ji). The elements on the - main diagonal (CS_ii) are the auto-spectra. - - This is NOT the same as what is commonly referred to as - "cross spectral density" as there is no (time) averaging!! - Multi-tapering alone usually is not sufficient to get enough - statitstical power for a robust csd estimate. - - Parameters - ---------- - data_arr : (K,N) :class:`numpy.ndarray` - Uniformly sampled multi-channel time-series data - The 1st dimension is interpreted as the time axis, - columns represent individual channels. - - Returns - ------- - CS_ij : (N, N, M) :class:`numpy.ndarray` - Cross spectra for all channel combinations i,j. - `M = K // 2 + 1` is the number of Fourier frequency bins, - `N` corresponds to number of input channels. - - freqs : (M,) :class:`numpy.ndarray` - The Fourier frequencies - - See also - -------- - mtmfft : :func:`~syncopy.specest.mtmfft.mtmfft` - (Multi-)tapered Fourier analysis - - """ - nSamples, nChannels = data_arr.shape - - # has shape (nTapers x nFreq x nChannels) - specs, freqs = mtmfft(data_arr, samplerate, taper, taperopt) - - # outer product along channel axes - # has shape (nTapers x nFreq x nChannels x nChannels) - CSD_ij = specs[:, :, np.newaxis, :] * specs[:, :, :, np.newaxis].conj() - - # average tapers and transpose: - # now has shape (nChannels x nChannels x nFreq) - CSD_ij = CSD_ij.mean(axis=0).T - - if norm: - # main diagonal: the auto spectra - # has shape (nChannels x nFreq) - diag = CSD_ij.diagonal() - # get the needed product pairs of the autospectra - Ciijj = np.sqrt(diag[:, :, None] * diag[:, None, :]).T - CSD_ij = CSD_ij / Ciijj - - return freqs, CSD_ij, specs - - -# white noise ensemble - - From 7fc6f296491b6c2c148a505264ae4f75c0f31270 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 15 Dec 2021 14:25:19 +0100 Subject: [PATCH 100/109] FIX: Fixed typos - pedantic commit On branch coherence Changes to be committed: modified: syncopy/connectivity/AV_compRoutines.py modified: syncopy/connectivity/ST_compRoutines.py modified: syncopy/connectivity/granger.py modified: syncopy/connectivity/wilson_sf.py modified: syncopy/specest/compRoutines.py modified: syncopy/specest/mtmfft.py --- syncopy/connectivity/AV_compRoutines.py | 27 ++++---- syncopy/connectivity/ST_compRoutines.py | 30 ++++----- syncopy/connectivity/granger.py | 45 +++++++------- syncopy/connectivity/wilson_sf.py | 83 ++++++++++++------------- syncopy/specest/compRoutines.py | 36 +++++------ syncopy/specest/mtmfft.py | 35 +++++------ 6 files changed, 126 insertions(+), 130 deletions(-) diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 37d91f503..76abc45c7 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -4,7 +4,7 @@ # the parallel single trial computations to be found in ST_compRoutines.py # The standard use case involves computations on the # trial average, meaning that the SyNCoPy input to these routines -# consists of only '1 trial` and parallelising over channels +# consists of only '1 trial' and parallelising over channels # is non trivial and atm also not supported. Pre-processing # like padding or detrending already happened in the single trial # compute functions. @@ -37,14 +37,16 @@ def normalize_csd_cF(csd_av_dat, """ Given the trial averaged cross spectral densities, calculates the normalizations to arrive at the - channel x channel coherencies. If S_ij(f) is the - averaged cross-spectrum between channel i and j, the + channel x channel coherencies. If ``S_ij(f)`` is the + averaged cross-spectrum between channel `i` and `j`, the coherency [1]_ is defined as: - C_ij = S_ij(f) / (|S_ii| |S_jj|) + .. math:: - The coherence is now defined as either |C_ij| - or |C_ij|^2, this can be controlled with the `output` + C_{ij} = S_{ij}(f) / (|S_{ii}| |S_{jj}|) + + The coherence is now defined as either ``|C_ij|`` + or ``|C_ij|^2``, this can be controlled with the `output` parameter. Parameters @@ -54,7 +56,7 @@ def normalize_csd_cF(csd_av_dat, and `nFreq` frequencies averaged over trials. output : {'abs', 'pow', 'fourier'}, default: 'abs' Also after normalization the coherency is still complex (`'fourier'`), - to get the real valued coherence 0 < C_ij(f) < 1 one can either take the + to get the real valued coherence ``0 < C_ij(f) < 1`` one can either take the absolute (`'abs'`) or the absolute squared (`'pow'`) values of the coherencies. The definitions are not uniform in the literature, hence multiple output types are supported. @@ -66,7 +68,7 @@ def normalize_csd_cF(csd_av_dat, Returns ------- CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` - Coherence for all channel combinations i,j. + Coherence for all channel combinations ``i,j``. `N` corresponds to number of input channels. Notes @@ -214,7 +216,7 @@ def normalize_ccov_cF(trl_av_dat, Returns ------- Corr_ij : (nLag, 1, N, N) :class:`numpy.ndarray` - Cross-correlations for all channel combinations i,j. + Cross-correlations for all channel combinations ``i,j``. `N` corresponds to number of input channels. Notes @@ -357,7 +359,7 @@ def granger_cF(csd_av_dat, for spectral matrix factorization. Default should be fine for most cases, handle with care! nIter : int - Maximum Number of iterations for CSD factorization. A result + Maximum number of iterations for CSD factorization. A result is returned if exhausted also if error tolerance was not met. cond_max : float The maximal condition number of the spectral matrix. @@ -377,8 +379,8 @@ def granger_cF(csd_av_dat, Granger : (1, nFreq, N, N) :class:`numpy.ndarray` Spectral Granger-Geweke causality between all channel combinations. Directionality follows array - notation: causality from i->j is Granger[0,:,i,j], - causality from j->i is Granger[0,:,j,i] + notation: causality from ``i -> j`` is ``Granger[0,:,i,j]``, + causality from ``j -> i`` is ``Granger[0,:,j,i]`` Notes ----- @@ -429,7 +431,6 @@ def granger_cF(csd_av_dat, # if this is not enough! CSDreg, factor = regularize_csd(CSD, cond_max=cond_max, eps_max=1e-3) # call Wilson - H, Sigma, conv = wilson_sf(CSDreg, nIter=nIter, rtol=rtol) # calculate G-causality diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index a63982ee9..d6686588f 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# computeFunctions and -Routines to parallel calculate -# single trial measures needed for the averaged +# computeFunctions and -Routines for parallel calculation +# of single trial measures needed for the averaged # measures like cross spectral densities # @@ -44,8 +44,8 @@ def cross_spectra_cF(trl_dat, for multi-taper analysis with `taper="dpss"`. Output consists of all (nChannels x nChannels+1)/2 different complex - estimates aranged in a symmetric fashion (CS_ij == CS_ji*). The - elements on the main diagonal (CS_ii) are the (real) auto-spectra. + estimates arranged in a symmetric fashion (``CS_ij == CS_ji*``). The + elements on the main diagonal (`CS_ii`) are the (real) auto-spectra. This is NOT the same as what is commonly referred to as "cross spectral density" as there is no (time) averaging!! @@ -60,7 +60,7 @@ def cross_spectra_cF(trl_dat, Uniformly sampled multi-channel time-series data The 1st dimension is interpreted as the time axis, columns represent individual channels. - Dimensions can be transposed to (N, K) with the `timeAxis` parameter. + Dimensions can be transposed to `(N, K)` with the `timeAxis` parameter. samplerate : float Samplerate in Hz foi : 1D :class:`numpy.ndarray` or None, optional @@ -102,7 +102,7 @@ def cross_spectra_cF(trl_dat, Returns ------- CS_ij : (1, nFreq, N, N) :class:`numpy.ndarray` - Complex cross spectra for all channel combinations i,j. + Complex cross spectra for all channel combinations ``i,j``. `N` corresponds to number of input channels. freqs : (nFreq,) :class:`numpy.ndarray` @@ -213,12 +213,12 @@ class ST_CrossSpectra(ComputationalRoutine): computeFunction = staticmethod(cross_spectra_cF) backends = [mtmfft] - # 1st argument,the data, gets omitted + # 1st argument,the data, gets omitted valid_kws = list(signature(mtmfft).parameters.keys())[1:] valid_kws += list(signature(cross_spectra_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend - valid_kws += ['tapsmofrq', 'nTaper'] - + valid_kws += ['tapsmofrq', 'nTaper'] + def process_metadata(self, data, out): # Some index gymnastics to get trial begin/end "samples" @@ -261,10 +261,10 @@ def cross_covariance_cF(trl_dat, """ Single trial covariance estimates between all channels - of the input data. Output consists of all (nChannels x nChannels+1)/2 - different estimates aranged in a symmetric fashion - (COV_ij == COV_ji). The elements on the - main diagonal (CS_ii) are the channel variances. + of the input data. Output consists of all ``(nChannels x nChannels+1)/2`` + different estimates arranged in a symmetric fashion + (``COV_ij == COV_ji``). The elements on the + main diagonal (`CS_ii`) are the channel variances. Parameters ---------- @@ -272,7 +272,7 @@ def cross_covariance_cF(trl_dat, Uniformly sampled multi-channel time-series data The 1st dimension is interpreted as the time axis, columns represent individual channels. - Dimensions can be transposed to (N, K) with the `timeAxis` parameter. + Dimensions can be transposed to `(N, K)` with the `timeAxis` parameter. samplerate : float Samplerate in Hz padding_opt : dict @@ -299,7 +299,7 @@ def cross_covariance_cF(trl_dat, Returns ------- CC_ij : (K, 1, N, N) :class:`numpy.ndarray` - Cross covariance for all channel combinations i,j. + Cross covariance for all channel combinations ``i,j``. `N` corresponds to number of input channels. lags : (M,) :class:`numpy.ndarray` diff --git a/syncopy/connectivity/granger.py b/syncopy/connectivity/granger.py index dbb7c29a2..a755a5253 100644 --- a/syncopy/connectivity/granger.py +++ b/syncopy/connectivity/granger.py @@ -1,30 +1,29 @@ # -*- coding: utf-8 -*- -# +# # Implementation of Granger-Geweke causality -# +# # # Builtin/3rd party package imports import numpy as np def granger(CSD, Hfunc, Sigma): - - ''' + """ Computes the pairwise Granger-Geweke causalities for all (non-symmetric!) channel combinations - according to equation 8 in [1]_. + according to Equation 8 in [1]_. - The transfer functions `Hfunc` and noise covariance + The transfer functions `Hfunc` and noise covariance `Sigma` are expected to have been already computed. Parameters ---------- CSD : (nFreq, N, N) :class:`numpy.ndarray` - Complex cross spectra for all channel combinations i,j - `N` corresponds to number of input channels. + Complex cross spectra for all channel combinations ``i,j`` + `N` corresponds to number of input channels. Hfunc : (nFreq, N, N) :class:`numpy.ndarray` - Spectral transfer functions for all channel combinations i,j - Sigma : (N, N) :class:`numpy.ndarray` + Spectral transfer functions for all channel combinations ``i,j`` + Sigma : (N, N) :class:`numpy.ndarray` The noise covariances Returns @@ -32,41 +31,41 @@ def granger(CSD, Hfunc, Sigma): Granger : (nFreq, N, N) :class:`numpy.ndarray` Spectral Granger-Geweke causality between all channel combinations. Directionality follows array - notation: causality from i->j is Granger[:,i,j], - causality from j->i is Granger[:,j,i] + notation: causality from ``i -> j`` is ``Granger[:,i,j]``, + causality from ``j -> i`` is ``Granger[:,j,i]`` See also -------- wilson_sf : :func:`~syncopy.connectivity.wilson_sf.wilson_sf - Spectral matrix factorization that yields the + Spectral matrix factorization that yields the transfer functions and noise covariances from a cross spectral density. Notes ----- - .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. - "Estimating Granger causality from Fourier and wavelet transforms + .. [1] Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms of time series data." Physical review letters 100.1 (2008): 018701. - ''' + """ - nChannels = CSD.shape[1] + nChannels = CSD.shape[1] auto_spectra = CSD.transpose(1, 2, 0).diagonal() auto_spectra = np.abs(auto_spectra) # auto-spectra are real - + # we need the stacked auto-spectra of the form (nChannel=3): - # S_11 S_22 S_33 - # Smat(f) = S_11 S_22 S_33 + # S_11 S_22 S_33 + # Smat(f) = S_11 S_22 S_33 # S_11 S_22 S_33 Smat = auto_spectra[:, None, :] * np.ones(nChannels)[:, None] - # Granger i->j needs H_ji entry + # Granger i->j needs H_ji entry Hmat = np.abs(Hfunc.transpose(0, 2, 1))**2 # Granger i->j needs Sigma_ji entry SigmaIJ = np.abs(Sigma.T)**2 # imag part should be 0 - auto_cov = np.abs(Sigma.diagonal()) + auto_cov = np.abs(Sigma.diagonal()) # same stacking as for the auto spectra (without freq axis) SigmaII = auto_cov[:, None] * np.ones(nChannels)[:, None] @@ -77,4 +76,4 @@ def granger(CSD, Hfunc, Sigma): # linear causality i -> j Granger = np.log(Smat / denom) - return Granger + return Granger diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index feabb138f..ada2fa18a 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# +# # Performs the numerical inner-outer factorization of a spectral matrix, using # Wilsons method. This implementation here is a Python version of the original # Matlab implementation by M. Dhamala (mdhamala@bme.ufl.edu) & G. Rangarajan @@ -14,8 +14,7 @@ def wilson_sf(CSD, nIter=100, rtol=1e-9): - - ''' + """ Wilsons spectral matrix factorization ("analytic method") Converges extremely fast, so the default number of @@ -27,43 +26,43 @@ def wilson_sf(CSD, nIter=100, rtol=1e-9): Parameters ---------- CSD : (nFreq, N, N) :class:`numpy.ndarray` - Complex cross spectra for all channel combinations i,j. + Complex cross spectra for all channel combinations ``i,j``. `N` corresponds to number of input channels. Has to be positive definite and well conditioned. nIter : int - Maximum Number of iterations, factorization result + Maximum number of iterations, factorization result is returned also if error tolerance wasn't met. rtol : float - Tolerance of the relative maximal - error of the factorization. + Tolerance of the relative maximal + error of the factorization. Returns ------- Hfunc : (nFreq, N, N) :class:`numpy.ndarray` - The transfer functio + The transfer function Sigma : (N, N) :class:`numpy.ndarray` Noise covariance converged : bool - Indicates wether the algorithm converged. + Indicates wether the algorithm converged. If `False` result was returned after `nIter` iterations. - ''' + """ nFreq, nChannels = CSD.shape[:2] Ident = np.eye(*CSD.shape[1:]) - + # nChannel x nChannel psi0 = _psi0_initial(CSD) - + # initial choice of psi, constant for all z(~f) - psi = np.tile(psi0, (nFreq, 1, 1)) + psi = np.tile(psi0, (nFreq, 1, 1)) assert psi.shape == CSD.shape converged = False for _ in range(nIter): - - psi_inv = np.linalg.inv(psi) + + psi_inv = np.linalg.inv(psi) # the bracket of equation 3.1 g = psi_inv @ CSD @ psi_inv.conj().transpose(0, 2, 1) gplus, gplus_0 = _plusOperator(g + Ident) @@ -75,20 +74,20 @@ def wilson_sf(CSD, nIter=100, rtol=1e-9): # the next step psi_{tau+1} psi = psi @ (gplus + S) psi0 = psi0 @ (gplus_0 + S) - + # max relative error CSDfac = psi @ psi.conj().transpose(0, 2, 1) err = np.abs(CSD - CSDfac) err = (err / np.abs(CSD)).max() - + # converged if err < rtol: converged = True break - + # Noise Covariance Sigma = psi0 @ psi0.conj().T - + # Transfer function psi0_inv = np.linalg.inv(psi0) Hfunc = psi @ psi0_inv.conj().T @@ -98,48 +97,48 @@ def wilson_sf(CSD, nIter=100, rtol=1e-9): def _psi0_initial(CSD): - ''' + """ Initialize Wilson's algorithm with the Cholesky decomposition of the 1st Fourier series component of the cross spectral density matrix (CSD). This is explicitly proposed in section 4. of the original paper. - ''' + """ nSamples = CSD.shape[1] - + # perform ifft to obtain gammas. gamma = np.fft.ifft(CSD, axis=0) gamma0 = gamma[0, ...] - - # Remove any assymetry due to rounding error. + + # Remove any asymmetry due to rounding error. # This also will zero out any imaginary values # on the diagonal - real diagonals are required for cholesky. gamma0 = np.real((gamma0 + gamma0.conj()) / 2) # check for positive definiteness eivals = np.linalg.eigvals(gamma0) - if np.all(np.imag(eivals) == 0): + if np.all(np.imag(eivals) == 0): psi0 = np.linalg.cholesky(gamma0) # otherwise initialize with 1's as a fallback else: psi0 = np.ones((nSamples, nSamples)) - + return psi0.T - + def _plusOperator(g): - ''' + """ The []+ operator from definition 1.2, given by explicit Fourier transformations - The nFreq x nChannel x nChannel matrix `g` is given + The nFreq x nChannel x nChannel matrix `g` is given in the frequency domain. - ''' + """ # 'negative lags' from the ifft nLag = g.shape[0] // 2 - # the series expansion in beta_k + # the series expansion in beta_k beta = np.fft.ifft(g, axis=0) # take half of the zero lag @@ -159,14 +158,14 @@ def _plusOperator(g): def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): - ''' - Brute force regularize CSD matrix - by inspecting the maximal condition number + """ + Brute force regularization of CSD matrix + by inspecting the maximal condition number along the frequency axis. - Multiply with different epsilon * I, - starting with epsilon = 1e-10 until the + Multiply with different ``epsilon * I``, + starting with ``epsilon = 1e-10`` until the condition number is smaller than `cond_max`. - Raises a ValueError if the maximal regularization + Raises a `ValueError` if the maximal regularization factor `epx_max` was reached but `cond_max` still not met. @@ -174,15 +173,15 @@ def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): ---------- CSD : 3D :class:`numpy.ndarray` The cross spectral density matrix - with shape (nFreq, nChannel, nChannel) + with shape ``(nFreq, nChannel, nChannel)`` cond_max : float The maximal condition number after regularization eps_max : float The largest regularization factor to be used. If also this value does not regularize the CSD up - to `cond_max` a ValueError is raised. + to `cond_max` a `ValueError` is raised. nSteps : int - Number of steps between 1e-10 and eps_max. + Number of steps between 1e-10 and `eps_max`. Returns ------- @@ -192,7 +191,7 @@ def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): eps : float The regularization factor used - ''' + """ epsilons = np.logspace(-10, np.log10(eps_max), 25) I = np.eye(CSD.shape[1]) @@ -203,7 +202,7 @@ def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): if CondNum < cond_max: return CSD, 0 - for eps in epsilons: + for eps in epsilons: CSDreg = CSD + eps * I CondNum = np.linalg.cond(CSDreg).max() diff --git a/syncopy/specest/compRoutines.py b/syncopy/specest/compRoutines.py index 19794b50c..f4a366258 100644 --- a/syncopy/specest/compRoutines.py +++ b/syncopy/specest/compRoutines.py @@ -15,7 +15,7 @@ # method_keys : list of names of the backend method parameters # cF_keys : list of names of the parameters of the middleware computeFunctions # -# the backend method name als gets explictly attached as a class constant: +# the backend method name als gets explicitly attached as a class constant: # method: backend method name # Builtin/3rd party package imports @@ -47,7 +47,7 @@ # ----------------------- @unwrap_io -def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, +def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, pad="nextpow2", padtype="zero", padlength=None, polyremoval=None, output_fmt="pow", noCompute=False, chunkShape=None, method_kwargs=None): @@ -105,7 +105,8 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, Notes ----- - This method is intended to be used as :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + This method is intended to be used as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` inside a :class:`~syncopy.shared.computational_routine.ComputationalRoutine`. Thus, input parameters are presumed to be forwarded from a parent metafunction. Consequently, this function does **not** perform any error checking and operates @@ -153,7 +154,7 @@ def mtmfft_cF(trl_dat, foi=None, timeAxis=0, keeptapers=True, dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) elif polyremoval == 1: dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) - + # call actual specest method res, _ = mtmfft(dat, **method_kwargs) @@ -184,7 +185,7 @@ class MultiTaperFFT(ComputationalRoutine): computeFunction = staticmethod(mtmfft_cF) # 1st argument,the data, gets omitted - valid_kws = list(signature(mtmfft).parameters.keys())[1:] + valid_kws = list(signature(mtmfft).parameters.keys())[1:] valid_kws += list(signature(mtmfft_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend valid_kws += ['tapsmofrq', 'nTaper'] @@ -335,7 +336,7 @@ def mtmconvol_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Pad input array if necessary if padbegin > 0 or padend > 0: dat = padding(dat, "zero", pad="relative", padlength=None, @@ -363,7 +364,7 @@ def mtmconvol_cF( detrend = 'linear' else: detrend = False - + # additional keyword args for `stft` in dictionary method_kwargs.update({"boundary": stftBdry, "padded": stftPad, @@ -417,7 +418,7 @@ class MultiTaperFFTConvol(ComputationalRoutine): computeFunction = staticmethod(mtmconvol_cF) # 1st argument,the data, gets omitted - valid_kws = list(signature(mtmconvol).parameters.keys())[1:] + valid_kws = list(signature(mtmconvol).parameters.keys())[1:] valid_kws += list(signature(mtmconvol_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend valid_kws += ['tapsmofrq', 't_ftimwin', 'nTaper'] @@ -542,7 +543,7 @@ def wavelet_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Get shape of output for dry-run phase nChannels = dat.shape[1] if isinstance(toi, np.ndarray): # `toi` is an array of time-points @@ -559,7 +560,7 @@ def wavelet_cF( dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) elif polyremoval == 1: dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) - + # ------------------ # actual method call # ------------------ @@ -683,7 +684,7 @@ def superlet_cF( ------- gmean_spec : :class:`numpy.ndarray` Complex time-frequency representation of the input data. - Shape is (nTime, 1, nScales, nChannels). + Shape is ``(nTime, 1, nScales, nChannels)``. Notes ----- @@ -698,8 +699,8 @@ def superlet_cF( -------- syncopy.freqanalysis : parent metafunction SuperletTransform : :class:`~syncopy.shared.computational_routine.ComputationalRoutine` - instance that calls this method as - :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` + instance that calls this method as + :meth:`~syncopy.shared.computational_routine.ComputationalRoutine.computeFunction` """ @@ -708,7 +709,7 @@ def superlet_cF( dat = trl_dat.T # does not copy but creates view of `trl_dat` else: dat = trl_dat - + # Get shape of output for dry-run phase nChannels = trl_dat.shape[1] if isinstance(toi, np.ndarray): # `toi` is an array of time-points @@ -725,7 +726,7 @@ def superlet_cF( dat = signal.detrend(dat, type='constant', axis=0, overwrite_data=True) elif polyremoval == 1: dat = signal.detrend(dat, type='linear', axis=0, overwrite_data=True) - + # ------------------ # actual method call # ------------------ @@ -751,7 +752,7 @@ class SuperletTransform(ComputationalRoutine): computeFunction = staticmethod(superlet_cF) - # 1st argument,the data, gets omitted + # 1st argument,the data, gets omitted valid_kws = list(signature(superlet).parameters.keys())[1:] valid_kws += list(signature(superlet_cF).parameters.keys())[1:-1] @@ -768,9 +769,6 @@ def process_metadata(self, data, out): # Construct trialdef array and compute new sampling rate trl, srate = _make_trialdef(self.cfg, trl, data.samplerate) - # Construct trialdef array and compute new sampling rate - trl, srate = _make_trialdef(self.cfg, trl, data.samplerate) - # If trial-averaging was requested, use the first trial as reference # (all trials had to have identical lengths), and average onset timings if not self.keeptrials: diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 778c19b15..84d9c0251 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# +# # Spectral estimation with (multi-)tapered FFT -# +# # Builtin/3rd party package imports import numpy as np @@ -9,12 +9,11 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): - - ''' + """ (Multi-)tapered fast Fourier transform. Returns full complex Fourier transform for each taper. Multi-tapering only supported with Slepian windwows (`taper="dpss"`). - + Parameters ---------- data_arr : (N,) :class:`numpy.ndarray` @@ -26,10 +25,10 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): Taper function to use, one of scipy.signal.windows Set to `None` for no tapering. taper_opt : dict or None - Additional keyword arguments passed to the `taper` function. - For multi-tapering with `taper='dpss'` set the keys + Additional keyword arguments passed to the `taper` function. + For multi-tapering with `taper='dpss'` set the keys `'Kmax'` and `'NW'`. - For further details, please refer to the + For further details, please refer to the `SciPy docs `_ Returns @@ -39,7 +38,7 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): Complex output has shape (nTapers x nFreq x nChannels). freqs : 1D :class:`numpy.ndarray` Array of Fourier frequencies - + Notes ----- @@ -48,8 +47,8 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): Sxx = np.real(ftr * ftr.conj()).mean(axis=0) - The FFT result is normalized such that this yields the squared amplitudes. - ''' + The FFT result is normalized such that this yields the squared amplitudes. + """ # attach dummy channel axis in case only a # single signal/channel is the input @@ -68,20 +67,20 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): if taper_opt is None: taper_opt = {} - - taper_func = getattr(signal.windows, taper) + + taper_func = getattr(signal.windows, taper) # only really 2d if taper='dpss' with Kmax > 1 windows = np.atleast_2d(taper_func(nSamples, **taper_opt)) - + # only(!!) slepian windows are already normalized # still have to normalize by number of tapers # such that taper-averaging yields correct amplitudes if taper == 'dpss': - windows = windows * np.sqrt(taper_opt.get('Kmax', 1)) - # per pedes L2 normalisation for all other tapers + windows = windows * np.sqrt(taper_opt.get('Kmax', 1)) + # per pedes L2 normalisation for all other tapers else: windows = windows * np.sqrt(nSamples) / np.sum(windows) - + # Fourier transforms (nTapers x nFreq x nChannels) ftr = np.zeros((windows.shape[0], nFreq, nChannels), dtype='complex64') @@ -90,7 +89,7 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): # real fft takes only 'half the energy'/positive frequencies, # multiply by 2 to correct for this ftr[taperIdx] = 2 * np.fft.rfft(data_arr * win, axis=0) - # normalization + # normalization ftr[taperIdx] /= np.sqrt(nSamples) return ftr, freqs From c1d03a92e9cd7e500fbf462f95a9ae973bc23d40 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 15 Dec 2021 15:30:47 +0100 Subject: [PATCH 101/109] FIX : Remove debugging print statements --- syncopy/connectivity/AV_compRoutines.py | 2 +- syncopy/connectivity/ST_compRoutines.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index 76abc45c7..f195a66cf 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -235,7 +235,7 @@ def normalize_ccov_cF(trl_av_dat, Single trial cross covariances. """ - print('AV call, input shape', trl_av_dat.shape) + # it's the same as the input shape! outShape = trl_av_dat.shape diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index d6686588f..8673269a0 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -35,7 +35,7 @@ def cross_spectra_cF(trl_dat, fullOutput=False): """ - Single trial Fourier cross spectra estimates between all channels + Single trial Fourier cross spectral estimates between all channels of the input data. First all the individual Fourier transforms are calculated via a (multi-)tapered FFT, then the pairwise cross-spectra are computed. @@ -315,7 +315,7 @@ def cross_covariance_cF(trl_dat, under the assumption that all inputs have been externally validated and cross-checked. """ - print('ST call, input shape:', trl_dat.shape) + # Re-arrange array if necessary and get dimensional information if timeAxis != 0: dat = trl_dat.T # does not copy but creates view of `trl_dat` @@ -370,7 +370,6 @@ def cross_covariance_cF(trl_dat, N = STDs[:, None] * STDs[None, :] CC = CC / N - print("ST done, output shape:", CC.shape) if not fullOutput: return CC else: From 7673b4db64b20d9b53e206df5fc718cb2cd3591c Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 15 Dec 2021 15:56:55 +0100 Subject: [PATCH 102/109] NEW : Support for single trial cross correlations - for connectivity analysis keeptrials=True only maybe makes sense for the cross correlations. In this case no AV CR needs to be called after the ST CR call. Changes to be committed: modified: connectivity_analysis.py --- syncopy/connectivity/connectivity_analysis.py | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index f3dada1a6..fa7480509 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -42,7 +42,15 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", nTaper=None, out=None, **kwargs): """ - coming soon.. + Perform connectivity analysis of Syncopy :class:`~syncopy.AnalogData` objects + + **Usage Summary** + + Options available in all analysis methods: + + * **foi**/**foilim** : frequencies of interest; either array of frequencies or + frequency window (not both) + * **polyremoval** : de-trending method to use (0 = mean, 1 = linear) """ # Make sure our one mandatory input object can be processed @@ -204,16 +212,24 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", if method == 'corr': check_effective_parameters(ST_CrossCovariance, defaults, lcls) + + # single trial cross-correlations + if keeptrials: + av_compRoutine = None # no trial average + norm = True # normalize individual trials within the ST CR + else: + av_compRoutine = NormalizeCrossCov() + norm = False + # parallel computation over trials st_compRoutine = ST_CrossCovariance(samplerate=data.samplerate, padding_opt=padding_opt, polyremoval=polyremoval, - timeAxis=timeAxis) + timeAxis=timeAxis, + norm=norm) # hard coded as class attribute st_dimord = ST_CrossCovariance.dimord - av_compRoutine = NormalizeCrossCov() - # ------------------------------------------------- # Call the chosen single trial ComputationalRoutine # ------------------------------------------------- @@ -224,10 +240,18 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", # Perform the trial-parallelized computation of the matrix quantity st_compRoutine.initialize(data, st_out._stackingDim, - chan_per_worker=None, # no parallelisation over channel possible + chan_per_worker=None, # no parallelisation over channels possible keeptrials=keeptrials) # we most likely need trial averaging! st_compRoutine.compute(data, st_out, parallel=kwargs.get("parallel"), log_dict=log_dict) + # if ever needed.. + # for single trial cross-corr results <-> keeptrials is True + if keeptrials and av_compRoutine is None: + if out is not None: + msg = "Single trial processing does not support `out` argument but directly returns the results" + SPYWarning(msg) + return st_out + # ---------------------------------------------------------------------------------- # Sanitize output and call the chosen ComputationalRoutine on the averaged ST output # ---------------------------------------------------------------------------------- @@ -244,7 +268,7 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", else: out = CrossSpectralData(dimord=st_dimord) new_out = True - + # now take the trial average from the single trial CR as input av_compRoutine.initialize(st_out, out._stackingDim, chan_per_worker=None) av_compRoutine.pre_check() # make sure we got a trial_average From 96f66311f12940e80d567f6fb24684eb7d0845d2 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Wed, 15 Dec 2021 17:30:33 +0100 Subject: [PATCH 103/109] FIX: Fixed more typos - yet another pedantic commit On branch coherence Changes to be committed: modified: syncopy/specest/mtmconvol.py modified: syncopy/specest/mtmfft.py modified: syncopy/tests/backend/test_connectivity.py modified: syncopy/tests/backend/test_timefreq.py --- syncopy/specest/mtmconvol.py | 22 ++---- syncopy/specest/mtmfft.py | 11 +-- syncopy/tests/backend/test_connectivity.py | 85 +++++++++++----------- syncopy/tests/backend/test_timefreq.py | 80 ++++++++++---------- 4 files changed, 94 insertions(+), 104 deletions(-) diff --git a/syncopy/specest/mtmconvol.py b/syncopy/specest/mtmconvol.py index 42b844cfd..fe273b2e0 100644 --- a/syncopy/specest/mtmconvol.py +++ b/syncopy/specest/mtmconvol.py @@ -11,7 +11,7 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", taper_opt={}, boundary='zeros', padded=True, detrend=False): - ''' + """ (Multi-)tapered short time fast Fourier transform. Returns full complex Fourier transform for each taper. Multi-tapering only supported with Slepian windwows (`taper="dpss"`). @@ -26,14 +26,14 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", nperseg : int Sliding window size in sample units noverlap : int - Overlap between consecutive windows, set to nperseg -1 + Overlap between consecutive windows, set to ``nperseg - 1`` to cover the whole signal taper : str or None - Taper function to use, one of scipy.signal.windows + Taper function to use, one of `scipy.signal.windows` Set to `None` for no tapering. taper_opt : dict Additional keyword arguments passed to the `taper` function. - For multi-tapering with `taper='dpss'` set the keys + For multi-tapering with ``taper='dpss'`` set the keys `'Kmax'` and `'NW'`. For further details, please refer to the `SciPy docs `_ @@ -42,28 +42,27 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", sample. If set to `False` half the window size (`nperseg`) will be lost on each side of the signal. padded : bool - Additional padding in case `noverlap != nperseg - 1` to fit an integer number + Additional padding in case ``noverlap != nperseg - 1`` to fit an integer number of windows. Returns ------- ftr : 4D :class:`numpy.ndarray` The Fourier transforms, complex output has shape: - (nTime, nTapers x nFreq x nChannels) + ``(nTime, nTapers x nFreq x nChannels)`` freqs : 1D :class:`numpy.ndarray` Array of Fourier frequencies Notes ----- - For a (MTM) power spectral estimate average the absolute squared transforms across tapers: - Sxx = np.real(ftr * ftr.conj()).mean(axis=0) + ``Sxx = np.real(ftr * ftr.conj()).mean(axis=0)`` The short time FFT result is normalized such that this yields the squared harmonic amplitudes. - ''' + """ # attach dummy channel axis in case only a # single signal/channel is the input @@ -119,8 +118,3 @@ def mtmconvol(data_arr, samplerate, nperseg, noverlap=None, taper="hann", ftr[:, taperIdx, ...] = 2 * pxx.transpose(2, 0, 1)[:nTime, ...] return ftr, freqs - - - - - diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 84d9c0251..26bba7e98 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -22,30 +22,28 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): samplerate : float Samplerate in Hz taper : str or None - Taper function to use, one of scipy.signal.windows + Taper function to use, one of `scipy.signal.windows` Set to `None` for no tapering. taper_opt : dict or None Additional keyword arguments passed to the `taper` function. - For multi-tapering with `taper='dpss'` set the keys + For multi-tapering with ``taper='dpss'`` set the keys `'Kmax'` and `'NW'`. For further details, please refer to the `SciPy docs `_ Returns ------- - ftr : 3D :class:`numpy.ndarray` - Complex output has shape (nTapers x nFreq x nChannels). + Complex output has shape ``(nTapers x nFreq x nChannels)``. freqs : 1D :class:`numpy.ndarray` Array of Fourier frequencies Notes ----- - For a (MTM) power spectral estimate average the absolute squared transforms across tapers: - Sxx = np.real(ftr * ftr.conj()).mean(axis=0) + ``Sxx = np.real(ftr * ftr.conj()).mean(axis=0)`` The FFT result is normalized such that this yields the squared amplitudes. """ @@ -93,4 +91,3 @@ def mtmfft(data_arr, samplerate, taper="hann", taper_opt=None): ftr[taperIdx] /= np.sqrt(nSamples) return ftr, freqs - diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index cfd1cc219..246dff4b9 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -10,15 +10,15 @@ def test_coherence(): - ''' - Tests the normalization cF to + """ + Tests the normalization cF to arrive at the coherence given a trial averaged csd - ''' + """ nSamples = 1001 fs = 1000 - tvec = np.arange(nSamples) / fs + tvec = np.arange(nSamples) / fs harm_freq = 40 phase_shifts = np.array([0, np.pi / 2, np.pi]) @@ -28,12 +28,12 @@ def test_coherence(): nFreq = nSamples // 2 + 1 nChannel = len(phase_shifts) avCSD = np.zeros((1, nFreq, nChannel, nChannel), dtype=np.complex64) - + for i in range(nTrials): - + # 1 phase phase shifted harmonics + white noise + constant, SNR = 1 trl_dat = [10 + np.cos(harm_freq * 2 * np. pi * tvec + ps) - for ps in phase_shifts] + for ps in phase_shifts] trl_dat = np.array(trl_dat).T trl_dat = np.array(trl_dat) + np.random.randn(nSamples, len(phase_shifts)) @@ -49,7 +49,7 @@ def test_coherence(): # this is the trial average avCSD /= nTrials - + # perform the normalisation on the trial averaged csd's Cij = avCR.normalize_csd_cF(avCSD) @@ -85,23 +85,23 @@ def test_coherence(): assert np.all(coh[:peak_idx - 2] < level) assert np.all(coh[peak_idx + 2:] < level) - + def test_csd(): - ''' + """ Tests multi-tapered single trial cross spectral densities - ''' + """ nSamples = 1001 fs = 1000 - tvec = np.arange(nSamples) / fs + tvec = np.arange(nSamples) / fs harm_freq = 40 phase_shifts = np.array([0, np.pi / 2, np.pi]) # 1 phase phase shifted harmonics + white noise + constant, SNR = 1 data = [10 + np.cos(harm_freq * 2 * np. pi * tvec + ps) - for ps in phase_shifts] + for ps in phase_shifts] data = np.array(data).T data = np.array(data) + np.random.randn(nSamples, len(phase_shifts)) @@ -145,9 +145,9 @@ def test_csd(): def test_cross_cov(): nSamples = 1001 - fs = 1000 + fs = 1000 tvec = np.arange(nSamples) / fs - + cosine = np.cos(2 * np.pi * 30 * tvec) sine = np.sin(2 * np.pi * 30 * tvec) data = np.c_[cosine, sine] @@ -157,23 +157,22 @@ def test_cross_cov(): # test for result is returned in the [0, np.ceil(nSamples / 2)] lag interval nLags = int(np.ceil(nSamples / 2)) - + # output has shape (nLags, 1, nChannels, nChannels) assert CC.shape == (nLags, 1, data.shape[1], data.shape[1]) - + # cross-correlation (normalized cross-covariance) between - # cosine and sine analytically equals minus sine + # cosine and sine analytically equals minus sine assert np.all(CC[:, 0, 0, 1] + sine[:nLags] < 1e-5) def test_wilson(): - - ''' + """ Test Wilson's spectral matrix factorization. As the routine has relative error-checking inbuild, we just need to check for convergence. - ''' + """ # --- create test data --- fs = 1000 @@ -182,14 +181,14 @@ def test_wilson(): f1, f2 = [30 , 40] # 30Hz and 60Hz data = np.zeros((nSamples, nChannels)) for i in range(nChannels): - # more phase diffusion in the 60Hz band + # more phase diffusion in the 60Hz band p1 = phase_evo(f1 * 2 * np.pi, eps=0.1, fs=fs, N=nSamples) p2 = phase_evo(f2 * 2 * np.pi, eps=0.35, fs=fs, N=nSamples) - + data[:, i] = np.cos(p1) + 2 * np.sin(p2) + .5 * np.random.randn(nSamples) - + # --- get the (single trial) CSD --- - + bw = 5 # 5Hz smoothing NW = bw * nSamples / (2 * fs) Kmax = int(2 * NW - 1) # optimal number of tapers @@ -207,15 +206,15 @@ def test_wilson(): assert CN > 1e6 # --- regularize CSD --- - + CSDreg, fac = regularize_csd(CSD, cond_max=1e6, nSteps=25) CNreg = np.linalg.cond(CSDreg).max() assert CNreg < 1e6 # check that 'small' regularization factor is enough - assert fac < 1e-5 - + assert fac < 1e-5 + # --- factorize CSD with Wilson's algorithm --- - + H, Sigma, conv = wilson_sf(CSDreg, rtol=1e-9) # converged - \Psi \Psi^* \approx CSD, @@ -224,12 +223,12 @@ def test_wilson(): # reconstitute CSDfac = H @ Sigma @ H.conj().transpose(0, 2, 1) - + fig, ax = ppl.subplots(figsize=(6, 4)) ax.set_xlabel('frequency (Hz)') ax.set_ylabel(r'$|CSD_{ij}(f)|$') chan = nChannels // 2 - # show (real) auto-spectra + # show (real) auto-spectra assert ax.plot(freqs, np.abs(CSD[:, chan, chan]), '-o', label='original CSD', ms=3) assert ax.plot(freqs, np.abs(CSDreg[:, chan, chan]), @@ -241,20 +240,20 @@ def test_wilson(): def test_granger(): - ''' + """ Test the granger causality measure with uni-directionally coupled AR(2) processes akin to the source publication: - Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. - "Estimating Granger causality from Fourier and wavelet transforms + Dhamala, Mukeshwar, Govindan Rangarajan, and Mingzhou Ding. + "Estimating Granger causality from Fourier and wavelet transforms of time series data." Physical review letters 100.1 (2008): 018701. - ''' + """ fs = 200 # Hz nSamples = 2500 nTrials = 50 - + # both AR(2) processes have same parameters # and yield a spectral peak at 40Hz alpha1, alpha2 = 0.55, -0.8 @@ -264,17 +263,17 @@ def test_granger(): for _ in range(nTrials): # -- simulate 2 AR(2) processes -- - + sol = np.zeros((nSamples, 2)) # pick the 1st values at random xs_ini = np.random.randn(2, 2) sol[:2, :] = xs_ini for i in range(1, nSamples): - sol[i, 1] = alpha1 * sol[i - 1, 1] + alpha2 * sol[i - 2, 1] + sol[i, 1] = alpha1 * sol[i - 1, 1] + alpha2 * sol[i - 2, 1] sol[i, 1] += np.random.randn() # X2 drives X1 sol[i, 0] = alpha1 * sol[i - 1, 0] + alpha2 * sol[i - 2, 0] - sol[i, 0] += sol[i - 1, 1] * coupling + sol[i, 0] += sol[i - 1, 1] * coupling sol[i, 0] += np.random.randn() # --- get CSD --- @@ -288,12 +287,12 @@ def test_granger(): CSD = CS2[0, ...] CSDav += CSD - + CSDav /= nTrials # with only 2 channels this CSD is well conditioned assert np.linalg.cond(CSDav).max() < 1e2 H, Sigma, conv = wilson_sf(CSDav) - + G = granger(CSDav, H, Sigma) assert G.shape == CSDav.shape @@ -305,7 +304,7 @@ def test_granger(): assert G[freq_idx, 0, 1] < 0.1 # check high causality for 2->1 assert G[freq_idx, 1, 0] > 0.8 - + fig, ax = ppl.subplots(figsize=(6, 4)) ax.set_xlabel('frequency (Hz)') ax.set_ylabel(r'Granger causality(f)') @@ -318,7 +317,7 @@ def test_granger(): # noisy phase evolution -> phase diffusion def phase_evo(omega0, eps, fs=1000, N=1000): - wn = np.random.randn(N) + wn = np.random.randn(N) delta_ts = np.ones(N) * 1 / fs phase = np.cumsum(omega0 * delta_ts + eps * wn) return phase diff --git a/syncopy/tests/backend/test_timefreq.py b/syncopy/tests/backend/test_timefreq.py index 559fb8290..95af47d0d 100644 --- a/syncopy/tests/backend/test_timefreq.py +++ b/syncopy/tests/backend/test_timefreq.py @@ -12,42 +12,42 @@ def gen_testdata(freqs=[20, 40, 60], cycles=11, fs=1000, eps = 0): - ''' + """ Harmonic superposition of multiple few-cycle oscillations akin to the example of Figure 3 in Moca et al. 2021 NatComm Each harmonic has a frequency neighbor with +10Hz and a time neighbor after 2 cycles(periods). - ''' + """ signal = [] for freq in freqs: - + # 10 cycles of f1 tvec = np.arange(cycles / freq, step=1 / fs) harmonic = np.cos(2 * np.pi * freq * tvec) # frequency neighbor - f_neighbor = np.cos(2 * np.pi * (freq + 10) * tvec) + f_neighbor = np.cos(2 * np.pi * (freq + 10) * tvec) packet = harmonic + f_neighbor # 2 cycles time neighbor delta_t = np.zeros(int(2 / freq * fs)) - + # 5 cycles break pad = np.zeros(int(5 / freq * fs)) signal.extend([pad, packet, delta_t, harmonic]) - # stack the packets together with some padding + # stack the packets together with some padding signal.append(pad) signal = np.concatenate(signal) # additive white noise if eps > 0: signal = np.random.randn(len(signal)) * eps + signal - + return signal @@ -60,7 +60,7 @@ def gen_testdata(freqs=[20, 40, 60], # signal_freqs = np.array([20, 70]) cycles = 12 A = 5 # signal amplitude -signal = A * gen_testdata(freqs=signal_freqs, cycles=cycles, fs=fs, eps=0.) +signal = A * gen_testdata(freqs=signal_freqs, cycles=cycles, fs=fs, eps=0.) # define frequencies of interest for wavelet methods foi = np.arange(1, 101, step=1) @@ -69,18 +69,18 @@ def gen_testdata(freqs=[20, 40, 60], freq_idx = [] for frequency in signal_freqs: freq_idx.append(np.argmax(foi >= frequency)) - + def test_mtmconvol(): - # 10 cycles of 40Hz are 250 samples + # 10 cycles of 40Hz are 250 samples window_size = 750 # default - stft pads with 0's to make windows fit # we choose N-1 overlap to retrieve a time-freq estimate # for each epoch in the signal - # the transforms have shape (nTime, nTaper, nFreq, nChannel) + # the transforms have shape (nTime, nTaper, nFreq, nChannel) ftr, freqs = mtmconvol.mtmconvol(signal, samplerate=fs, taper='cosine', nperseg=window_size, @@ -96,11 +96,11 @@ def test_mtmconvol(): gridspec_kw={"height_ratios": [1, 3]}, figsize=(6, 6)) - ax1.set_title("Short Time Fourier Transform") + ax1.set_title("Short Time Fourier Transform") ax1.plot(np.arange(signal.size) / fs, signal, c='cornflowerblue') ax1.set_ylabel('signal (a.u.)') - ax2.set_xlabel("time (s)") + ax2.set_xlabel("time (s)") ax2.set_ylabel("frequency (Hz)") df = freqs[1] - freqs[0] @@ -129,7 +129,7 @@ def test_mtmconvol(): for frequency in signal_freqs: freq_idx.append(np.argmax(freqs >= frequency)) - # test amplitude normalization + # test amplitude normalization for idx, frequency in zip(freq_idx, signal_freqs): ax2.plot([0, len(signal) / fs], @@ -145,7 +145,7 @@ def test_mtmconvol(): # assert cycle_num > 2 * cycles # power should decay fast, so we don't detect more cycles # assert cycle_num < 3 * cycles - + fig.tight_layout() # ------------------------- @@ -169,11 +169,11 @@ def test_mtmconvol(): gridspec_kw={"height_ratios": [1, 3]}, figsize=(6, 6)) - ax1.set_title("Multi-Taper STFT") + ax1.set_title("Multi-Taper STFT") ax1.plot(np.arange(signal.size) / fs, signal, c='cornflowerblue') ax1.set_ylabel('signal (a.u.)') - ax2.set_xlabel("time (s)") + ax2.set_xlabel("time (s)") ax2.set_ylabel("frequency (Hz)") # test also the plotting @@ -213,10 +213,10 @@ def test_mtmconvol(): def test_superlet(): - + scalesSL = superlet.scale_from_period(1 / foi) - # spec shape is nScales x nTime (x nChannels) + # spec shape is nScales x nTime (x nChannels) spec = superlet.superlet(signal, samplerate=fs, scales=scalesSL, @@ -231,13 +231,13 @@ def test_superlet(): sharex=True, gridspec_kw={"height_ratios": [1, 3]}, figsize=(6, 6)) - - ax1.set_title("Superlet Transform") + + ax1.set_title("Superlet Transform") ax1.plot(np.arange(signal.size) / fs, signal, c='cornflowerblue') ax1.set_ylabel('signal (a.u.)') - - ax2.set_xlabel("time (s)") - ax2.set_ylabel("frequency (Hz)") + + ax2.set_xlabel("time (s)") + ax2.set_ylabel("frequency (Hz)") extent = [0, len(signal) / fs, foi[0], foi[-1]] # test also the plotting # scale with amplitude @@ -248,7 +248,7 @@ def test_superlet(): origin='lower', vmin=0, vmax=1.2 * A) - + # get the 'mappable' im = ax2.images[0] fig.colorbar(im, ax = ax2, orientation='horizontal', @@ -272,7 +272,7 @@ def test_superlet(): fig.tight_layout() - + def test_wavelet(): # get a wavelet function @@ -294,8 +294,8 @@ def test_wavelet(): ax1.set_title("Wavelet Transform") ax1.plot(np.arange(signal.size) / fs, signal, c='cornflowerblue') ax1.set_ylabel('signal (a.u.)') - - ax2.set_xlabel("time (s)") + + ax2.set_xlabel("time (s)") ax2.set_ylabel("frequency (Hz)") extent = [0, len(signal) / fs, foi[0], foi[-1]] @@ -332,7 +332,7 @@ def test_wavelet(): fig.tight_layout() - + def test_mtmfft(): # superposition 40Hz and 100Hz oscillations A1:A2 for 1s @@ -346,8 +346,8 @@ def test_mtmfft(): # -------------------- # -- test untapered -- # -------------------- - - # the transforms have shape (nTaper, nFreq, nChannel) + + # the transforms have shape (nTaper, nFreq, nChannel) ftr, freqs = mtmfft.mtmfft(signal, fs, taper=None) # with 1000Hz sampling frequency and 1000 samples this gives @@ -359,7 +359,7 @@ def test_mtmfft(): spec = np.real(ftr * ftr.conj()).mean(axis=0) amplitudes = np.sqrt(spec)[:, 0] # only 1 channel # our FFT normalisation recovers the signal amplitudes: - assert np.allclose([A1, A2], amplitudes[[f1, f2]]) + assert np.allclose([A1, A2], amplitudes[[f1, f2]]) fig, ax = ppl.subplots() ax.set_title(f"Amplitude spectrum {A1} x 40Hz + {A2} x 100Hz") @@ -370,10 +370,10 @@ def test_mtmfft(): # ------------------------- # test multi-taper analysis # ------------------------- - + taper_opt = {'Kmax' : 8, 'NW' : 1} ftr, freqs = mtmfft.mtmfft(signal, fs, taper="dpss", taper_opt=taper_opt) - # average over tapers + # average over tapers dpss_spec = np.real(ftr * ftr.conj()).mean(axis=0) dpss_amplitudes = np.sqrt(dpss_spec)[:, 0] # only 1 channel # check for amplitudes (and taper normalisation) @@ -385,19 +385,19 @@ def test_mtmfft(): # ----------------- # test kaiser taper (is boxcar for beta -> inf) # ----------------- - + taper_opt = {'beta' : 2} ftr, freqs = mtmfft.mtmfft(signal, fs, taper="kaiser", taper_opt=taper_opt) # average over tapers (only 1 here) kaiser_spec = np.real(ftr * ftr.conj()).mean(axis=0) kaiser_amplitudes = np.sqrt(kaiser_spec)[:, 0] # only 1 channel # check for amplitudes (and taper normalisation) - assert np.allclose(kaiser_amplitudes[[f1, f2]], [A1, A2], atol=1e-2) + assert np.allclose(kaiser_amplitudes[[f1, f2]], [A1, A2], atol=1e-2) # ------------------------------- # test all other window functions (which don't need a parameter) # ------------------------------- - + for win in windows.__all__: taper_opt = {} # that guy isn't symmetric @@ -405,14 +405,14 @@ def test_mtmfft(): continue # that guy is deprecated if win == 'hanning': - continue + continue try: ftr, freqs = mtmfft.mtmfft(signal, fs, taper=win, taper_opt=taper_opt) # average over tapers (only 1 here) spec = np.real(ftr * ftr.conj()).mean(axis=0) amplitudes = np.sqrt(spec)[:, 0] # only 1 channel - # print(win, amplitudes[[f1, f2]]) - assert np.allclose(amplitudes[[f1, f2]], [A1, A2], atol=1e-3) + # print(win, amplitudes[[f1, f2]]) + assert np.allclose(amplitudes[[f1, f2]], [A1, A2], atol=1e-3) except TypeError: # we didn't provide default parameters.. pass From 3ee081016c6eace5f39b1c4bfb8326ef123500de Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 15 Dec 2021 17:16:44 +0100 Subject: [PATCH 104/109] WIP : Connectivity doc - also changed freqanalysis header doc to refer to the method strings - additionally captured padding for cross-correlation with a Value error Changes to be committed: modified: ST_compRoutines.py modified: connectivity_analysis.py modified: ../specest/freqanalysis.py --- syncopy/connectivity/ST_compRoutines.py | 2 +- syncopy/connectivity/connectivity_analysis.py | 36 +++++++++++++++++++ syncopy/specest/freqanalysis.py | 10 +++--- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index 8673269a0..de8bb6e7c 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -217,7 +217,7 @@ class ST_CrossSpectra(ComputationalRoutine): valid_kws = list(signature(mtmfft).parameters.keys())[1:] valid_kws += list(signature(cross_spectra_cF).parameters.keys())[1:] # hardcode some parameter names which got digested from the frontend - valid_kws += ['tapsmofrq', 'nTaper'] + valid_kws += ['tapsmofrq', 'nTaper', 'pad_to_length'] def process_metadata(self, data, out): diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index fa7480509..2bb76bb7d 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -51,6 +51,37 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", * **foi**/**foilim** : frequencies of interest; either array of frequencies or frequency window (not both) * **polyremoval** : de-trending method to use (0 = mean, 1 = linear) + + List of available analysis methods and respective distinct options: + + "coh" : (Multi-) tapered coherency estimate + Compute the normalized cross spectral densities + between all channel combinations + + * **taper** : one of :data:`~syncopy.shared.const_def.availableTapers` + * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) + * **nTaper** : (optional) number of orthogonal tapers for slepian tapers + * **pad_to_length**: either pad to an absolute length or set to `'nextpow2'` + + "corr" : Cross-correlations + Computes the one sided (positive lags) cross-correlations + between all channel combinations. The maximal lag is half + the trial lenghts. + + * **keeptrials** : set to `True` for single trial cross-correlations + + "granger" : Spectral Granger-Geweke causality + Computes linear causality estimates between + all channel combinations. The needed cross-spectral + densities can be computed via multi-tapering. + + * **taper** : one of :data:`~syncopy.shared.const_def.availableTapers` + * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) + * **nTaper** : (optional) number of orthogonal tapers for slepian tapers + * **pad_to_length**: either pad to an absolute length or set to `'nextpow2'` + + **Full documentation below** + """ # Make sure our one mandatory input object can be processed @@ -107,6 +138,11 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", # --- Padding --- + if method == "corr" and pad_to_length: + lgl = "`None`, no padding for cross-correlations" + actual = f"{pad_to_length}" + raise SPYValueError(legal=lgl, varname="pad_to_length", actual=actual) + # manual symmetric zero padding of ALL trials the same way if isinstance(pad_to_length, Number): diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 21153a499..3a2237f5d 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -72,13 +72,13 @@ def freqanalysis(data, method='mtmfft', output='fourier', List of available analysis methods and respective distinct options: - :func:`~syncopy.specest.mtmfft.mtmfft` : (Multi-)tapered Fourier transform + "mtmfft" : (Multi-)tapered Fourier transform Perform frequency analysis on time-series trial data using either a single taper window (Hanning) or many tapers based on the discrete prolate spheroidal sequence (DPSS) that maximize energy concentration in the main lobe. - * **taper** : one of :data:`~syncopy.specest.const_def.availableTapers` + * **taper** : one of :data:`~syncopy.shared.const_def.availableTapers` * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) * **nTaper** : number of orthogonal tapers for slepian tapers * **keeptapers** : return individual tapers or average @@ -91,7 +91,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', * **prepadlength** : number of samples to pre-pend to each trial * **postpadlength** : number of samples to append to each trial - :func:`~syncopy.specest.mtmconvol.mtmconvol` : (Multi-)tapered sliding window Fourier transform + "mtmconvol" : (Multi-)tapered sliding window Fourier transform Perform time-frequency analysis on time-series trial data based on a sliding window short-time Fourier transform using either a single Hanning taper or multiple DPSS tapers. @@ -109,7 +109,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', a window on every sample in the data. * **t_ftimwin** : sliding window length (in sec) - :func:`~syncopy.specest.wavelet.wavelet` : (Continuous non-orthogonal) wavelet transform + "wavelet" : (Continuous non-orthogonal) wavelet transform Perform time-frequency analysis on time-series trial data using a non-orthogonal continuous wavelet transform. @@ -122,7 +122,7 @@ def freqanalysis(data, method='mtmfft', output='fourier', * **order** : Order of Paul wavelet function (>= 4) or derivative order of real-valued DOG wavelets (2 = mexican hat) - :func:`~syncopy.specest.superlet.superlet` : Superlet transform + "superlet" : Superlet transform Perform time-frequency analysis on time-series trial data using the super-resolution superlet transform (SLT) from [Moca2021]_. From 0decee4e15fd236fc88585c30aa463780dabb3f4 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 16 Dec 2021 11:07:50 +0100 Subject: [PATCH 105/109] FIX: Docstrings for the input validators + removing (in-)sane tapsmofrq default - user now has to supply a smoothing bandwidth for multi-tapering, if None is given the range 1-10Hz is proposed in the error message Changes to be committed: modified: shared/input_validators.py --- syncopy/shared/input_validators.py | 60 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index ed5e42507..e870f0dd0 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -9,7 +9,7 @@ import numpy as np from syncopy.shared.errors import SPYValueError, SPYWarning, SPYInfo -from syncopy.shared.parsers import data_parser, scalar_parser, array_parser +from syncopy.shared.parsers import scalar_parser, array_parser from syncopy.shared.const_def import availableTapers, generalParameters @@ -17,15 +17,15 @@ def validate_foi(foi, foilim, samplerate): ''' - Parameters to check - ------------------- + Parameters + ---------- foi : 'all' or array like or None frequencies of interest foilim : 2-element sequence or None foi limits - Auxiliary arguments - ------------------- + Other Parameters + ---------------- samplerate : float the samplerate in Hz @@ -99,17 +99,17 @@ def validate_taper(taper, "The Effective Bandwidth of a Multitaper Spectral Estimator, A. T. Walden, E. J. McCoy and D. B. Percival" - Parameters to check - ------------------- + Parameters + ---------- taper : str Windowing function, one of :data:`~syncopy.shared.const_def.availableTapers` tapsmofrq : float or None Taper smoothing bandwidth for `taper='dpss'` nTaper : int_like or None - Number of tapers to user for multi-tapering (not recommended) + Number of tapers to use for multi-tapering (not recommended) - Auxiliary arguments - ------------------- + Other Parameters + ---------------- keeptapers : bool foimax : float Maximum frequency for the analysis @@ -155,27 +155,15 @@ def validate_taper(taper, # Set/get `tapsmofrq` if we're working w/Slepian tapers elif taper == "dpss": + # --- minimal smoothing bandwidth --- # --- such that Kmax/nTaper is at least 1 minBw = 2 * samplerate / nSamples # ----------------------------------- - - # Try to derive "sane" settings by using 3/4 octave - # smoothing of highest `foi` - # following Hill et al. "Oscillatory Synchronization in Large-Scale - # Cortical Networks Predicts Perception", Neuron, 2011 - # FIX ME: This "sane setting" seems quite excessive (huuuge bwidths) - - if tapsmofrq is None: - tapsmofrq = (foimax * 2**(3 / 4 / 2) - foimax * 2**(-3 / 4 / 2)) / 2 - if tapsmofrq < minBw: # *should* not happen but just in case - tapsmofrq = minBw - msg = f'Automatic setting of `tapsmofrq` to {tapsmofrq:.2f}' - SPYInfo(msg) # user set tapsmofrq directly - elif tapsmofrq is not None: + if tapsmofrq is not None: try: scalar_parser(tapsmofrq, varname="tapsmofrq", lims=[0, np.inf]) except Exception as exc: @@ -185,6 +173,21 @@ def validate_taper(taper, msg = f'Setting tapsmofrq to the minimal attainable bandwidth of {minBw:.2f}Hz' SPYInfo(msg) tapsmofrq = minBw + + # we now enforce a user submitted smoothing bw + else: + lgl = "smoothing bandwidth in Hz, typical values are in the range 1-10Hz" + raise SPYValueError(legal=lgl, varname="tapsmofrq", actual=tapsmofrq) + + # Try to derive "sane" settings by using 3/4 octave + # smoothing of highest `foi` + # following Hill et al. "Oscillatory Synchronization in Large-Scale + # Cortical Networks Predicts Perception", Neuron, 2011 + # FIX ME: This "sane setting" seems quite excessive (huuuge bwidths) + + # tapsmofrq = (foimax * 2**(3 / 4 / 2) - foimax * 2**(-3 / 4 / 2)) / 2 + # msg = f'Automatic setting of `tapsmofrq` to {tapsmofrq:.2f}' + # SPYInfo(msg) # -------------------------------------------- # set parameters for scipy.signal.windows.dpss @@ -192,9 +195,9 @@ def validate_taper(taper, # from the minBw setting NW always is at least 1 Kmax = int(2 * NW - 1) # optimal number of tapers # -------------------------------------------- - + # the recommended way: - # set nTaper automatically to maximize effective smoothing bandwidth + # set nTaper automatically to achieve exact effective smoothing bandwidth if nTaper is None: msg = f'Using {Kmax} taper(s) for multi-tapering' SPYInfo(msg) @@ -212,7 +215,7 @@ def validate_taper(taper, if nTaper != Kmax: msg = f''' Manually setting the number of tapers is not recommended - and may (strongly) distort the spectral estimation!\n + and may (strongly) distort the effective smoothing bandwidth!\n The optimal number of tapers is {Kmax}, you have chosen to use {nTaper}. ''' SPYWarning(msg) @@ -230,14 +233,13 @@ def check_effective_parameters(CR, defaults, lcls): Parameters ---------- - CR : :class:`~syncopy.shared.computational_routine.ComputationalRoutine Needs to have a `valid_kws` attribute defaults : dict Result of :func:`~syncopy.shared.tools.get_defaults`, the frontend parameter names plus values with default values lcls : dict - Result of `locals()`, all names and values of the local name space + Result of `locals()`, all names and values of the local (frontend-)name space ''' # list of possible parameter names of the CR expected = CR.valid_kws + ["parallel", "select"] From 43d03b5a2b7909bd5a8d4f38749596c18c42f984 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 16 Dec 2021 13:49:17 +0100 Subject: [PATCH 106/109] FIX: Repaired specest tests w/o tapsmofrq default - additionally shortened the sample info digestion from selections Changes to be committed: modified: specest/freqanalysis.py modified: tests/test_specest.py --- syncopy/specest/freqanalysis.py | 11 ++--------- syncopy/tests/test_specest.py | 10 ++++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 3a2237f5d..f739360bd 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -345,17 +345,10 @@ def freqanalysis(data, method='mtmfft', output='fourier', raise SPYTypeError(lcls[vname], varname=vname, expected="Bool") # If only a subset of `data` is to be processed, make some necessary adjustments - # and compute minimal sample-count across (selected) trials + # of the sampleinfo and trial lengths if data._selection is not None: + sinfo = data._selection.trialdefinition[:, :2] trialList = data._selection.trials - sinfo = np.zeros((len(trialList), 2)) - for tk, trlno in enumerate(trialList): - trl = data._preview_trial(trlno) - tsel = trl.idx[timeAxis] - if isinstance(tsel, list): - sinfo[tk, :] = [0, len(tsel)] - else: - sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] else: trialList = list(range(len(data.trials))) sinfo = data.sampleinfo diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index 99c9fb322..ff073c857 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -191,7 +191,7 @@ def test_allocout(self): # keep trials but throw away tapers out = SpectralData(dimord=SpectralData._defaultDimord) freqanalysis(self.adata, method="mtmfft", taper="dpss", - keeptapers=False, output="pow", out=out) + tapsmofrq=3, keeptapers=False, output="pow", out=out) assert out.sampleinfo.shape == (self.nTrials, 2) assert out.taper.size == 1 @@ -199,6 +199,7 @@ def test_allocout(self): cfg.dataset = self.adata cfg.out = SpectralData(dimord=SpectralData._defaultDimord) cfg.taper = "dpss" + cfg.tapsmofrq = 3 cfg.output = "pow" cfg.keeptapers = False freqanalysis(cfg) @@ -258,7 +259,7 @@ def test_dpss(self): # ensure default setting results in single taper spec = freqanalysis(self.adata, method="mtmfft", - taper="dpss", output="pow", select=select) + taper="dpss", tapsmofrq=3, output="pow", select=select) assert spec.taper.size == 1 assert spec.channel.size == len(chanList) @@ -396,7 +397,7 @@ def test_vdata(self): avdata = AnalogData(vdata, samplerate=self.fs, trialdefinition=self.trialdefinition) spec = freqanalysis(avdata, method="mtmfft", taper="dpss", - keeptapers=False, output="abs", pad="relative", + tapsmofrq=3, keeptapers=False, output="abs", pad="relative", padlength=npad) assert (np.diff(avdata.sampleinfo)[0][0] + npad) / 2 + 1 == spec.freq.size del avdata, vdata, dmap, spec @@ -560,7 +561,7 @@ def test_tf_allocout(self): # keep trials but throw away tapers out = SpectralData(dimord=SpectralData._defaultDimord) - freqanalysis(self.tfData, method="mtmconvol", taper="dpss", + freqanalysis(self.tfData, method="mtmconvol", taper="dpss", tapsmofrq=3, keeptapers=False, output="pow", toi=0.0, t_ftimwin=1.0, out=out) assert out.sampleinfo.shape == (self.nTrials, 2) @@ -570,6 +571,7 @@ def test_tf_allocout(self): cfg.dataset = self.tfData cfg.out = SpectralData(dimord=SpectralData._defaultDimord) cfg.taper = "dpss" + cfg.tapsmofrq = 3 cfg.keeptapers = False cfg.output = "pow" freqanalysis(cfg) From faf0c8a20d977ccd6132af64a4b2317f0a3d4783 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 16 Dec 2021 14:09:41 +0100 Subject: [PATCH 107/109] CHG: Allow unequal lengths trials in connectivity - user still can manually set an absolute padding lengths for all trials - in case this isn't done and unequal trial lengths are present enforce a 'maxlen' like padding with a warning msg Changes to be committed: modified: connectivity/connectivity_analysis.py --- syncopy/connectivity/connectivity_analysis.py | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 2bb76bb7d..4c2f690b1 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -14,7 +14,6 @@ from syncopy.datatype.methods.padding import _nextpow2 from syncopy.shared.errors import ( SPYValueError, - SPYTypeError, SPYWarning, SPYInfo) from syncopy.shared.kwarg_decorators import (unwrap_cfg, unwrap_select, @@ -102,48 +101,39 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", if method not in availableMethods: lgl = "'" + "or '".join(opt + "' " for opt in availableMethods) raise SPYValueError(legal=lgl, varname="method", actual=method) - - # If only a subset of `data` is to be processed, - # make some necessary adjustments - # and compute minimal sample-count across (selected) trials + + # if a subset selection is present + # get sampleinfo and check for equidistancy if data._selection is not None: + sinfo = data._selection.trialdefinition[:, :2] trialList = data._selection.trials - sinfo = np.zeros((len(trialList), 2)) - for tk, trlno in enumerate(trialList): - trl = data._preview_trial(trlno) - tsel = trl.idx[timeAxis] - - # user picked discrete set of time points - if isinstance(tsel, list): - lgl = "equidistant time points (toi) or time slice (toilim)" - actual = "non-equidistant set of time points" - raise SPYValueError(legal=lgl, varname="select", actual=actual) - - sinfo[tk, :] = [trl.idx[timeAxis].start, trl.idx[timeAxis].stop] + # user picked discrete set of time points + if isinstance(data._selection.time[0], list): + lgl = "equidistant time points (toi) or time slice (toilim)" + actual = "non-equidistant set of time points" + raise SPYValueError(legal=lgl, varname="select", actual=actual) else: trialList = list(range(len(data.trials))) sinfo = data.sampleinfo lenTrials = np.diff(sinfo).squeeze() - - # here we enforce equal lengths trials as is required for - # sensical trial averaging - user is responsible for trial - # specific padding and time axis alignments - # OR we do a brute force 'maxlen' padding if there is unequal lengths?! - if not lenTrials.min() == lenTrials.max(): - lgl = "trials of same lengths" - actual = "trials of different lengths - please pre-pad!" - raise SPYValueError(legal=lgl, varname="lenTrials", actual=actual) - numTrials = len(trialList) # --- Padding --- if method == "corr" and pad_to_length: - lgl = "`None`, no padding for cross-correlations" + lgl = "`None`, no padding needed/allowed for cross-correlations" actual = f"{pad_to_length}" raise SPYValueError(legal=lgl, varname="pad_to_length", actual=actual) - - # manual symmetric zero padding of ALL trials the same way + + # here we check for equal lengths trials as is required for + # trial averaging, in case of no user specified absolute padding length + # we do a rough 'maxlen' padding, nextpow2 will be overruled in this case + if lenTrials.min() != lenTrials.max() and not isinstance(pad_to_length, Number): + pad_to_length = int(lenTrials.max()) + msg = f"Unequal trial lengths present, automatic padding to {pad_to_length} samples" + SPYWarning(msg) + + # symmetric zero padding of ALL trials the same way if isinstance(pad_to_length, Number): scalar_parser(pad_to_length, @@ -157,7 +147,9 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", } # after padding! nSamples = pad_to_length + # or pad to optimal FFT lengths + # (not possible for unequal lengths trials) elif pad_to_length == 'nextpow2': padding_opt = { 'padtype' : 'zero', From 10a2ff960c7a2e2af2c0a396eebca901c77cac0f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 16 Dec 2021 14:29:04 +0100 Subject: [PATCH 108/109] NEW: Check polyremoval parameter - check via a simple scalar_parser is done in both connectivity and freqanalysis - also finally used """ instead of ''' in the input_validators.py docstrings Changes to be committed: modified: connectivity/connectivity_analysis.py modified: shared/input_validators.py modified: specest/freqanalysis.py --- syncopy/connectivity/connectivity_analysis.py | 4 ++++ syncopy/shared/input_validators.py | 16 ++++++++-------- syncopy/specest/freqanalysis.py | 4 ++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/syncopy/connectivity/connectivity_analysis.py b/syncopy/connectivity/connectivity_analysis.py index 4c2f690b1..e74c57065 100644 --- a/syncopy/connectivity/connectivity_analysis.py +++ b/syncopy/connectivity/connectivity_analysis.py @@ -118,6 +118,10 @@ def connectivity(data, method="coh", keeptrials=False, output="abs", lenTrials = np.diff(sinfo).squeeze() numTrials = len(trialList) + # check polyremoval + if polyremoval is not None: + scalar_parser(polyremoval, varname="polyremoval", ntype="int_like", lims=[0, 1]) + # --- Padding --- if method == "corr" and pad_to_length: diff --git a/syncopy/shared/input_validators.py b/syncopy/shared/input_validators.py index e870f0dd0..be40ce21e 100644 --- a/syncopy/shared/input_validators.py +++ b/syncopy/shared/input_validators.py @@ -15,8 +15,7 @@ def validate_foi(foi, foilim, samplerate): - ''' - + """ Parameters ---------- foi : 'all' or array like or None @@ -32,13 +31,15 @@ def validate_foi(foi, foilim, samplerate): Returns ------- foi, foilim : tuple + Either both are `None` or the + user submitted one is parsed and returned Notes ----- Setting both `foi` and `foilim` to `None` is valid, the subsequent analysis methods should all have a default way to select a standard set of frequencies (e.g. np.fft.fftfreq). - ''' + """ if foi is not None and foilim is not None: lgl = "either `foi` or `foilim` specification" @@ -91,7 +92,7 @@ def validate_taper(taper, nSamples, output): - ''' + """ General taper validation and Slepian/dpss input sanitization. The default is to max out `nTaper` to achieve the desired frequency smoothing bandwidth. For details about the Slepion settings see @@ -126,7 +127,7 @@ def validate_taper(taper, For multi-tapering (`taper='dpss'`) contains the parameters `NW` and `Kmax` for `scipy.signal.windows.dpss`. For all other tapers this is an empty dictionary. - ''' + """ # See if taper choice is supported if taper not in availableTapers: @@ -156,7 +157,6 @@ def validate_taper(taper, # Set/get `tapsmofrq` if we're working w/Slepian tapers elif taper == "dpss": - # --- minimal smoothing bandwidth --- # --- such that Kmax/nTaper is at least 1 minBw = 2 * samplerate / nSamples @@ -226,7 +226,7 @@ def validate_taper(taper, def check_effective_parameters(CR, defaults, lcls): - ''' + """ For a given ComputationalRoutine, compare set parameters (*lcls*) with the accepted parameters and the frontend meta function *defaults* to warn if any ineffective parameters are set. @@ -240,7 +240,7 @@ def check_effective_parameters(CR, defaults, lcls): parameter names plus values with default values lcls : dict Result of `locals()`, all names and values of the local (frontend-)name space - ''' + """ # list of possible parameter names of the CR expected = CR.valid_kws + ["parallel", "select"] relevant = [name for name in defaults if name not in generalParameters] diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index f739360bd..131bc7f4c 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -357,6 +357,10 @@ def freqanalysis(data, method='mtmfft', output='fourier', lenTrials = lenTrials[None] numTrials = len(trialList) + # check polyremoval + if polyremoval is not None: + scalar_parser(polyremoval, varname="polyremoval", ntype="int_like", lims=[0, 1]) + # Sliding window FFT does not support "fancy" padding if method == "mtmconvol" and isinstance(pad, str): msg = "method 'mtmconvol' only supports in-place padding for windows " +\ From e4409b5dee933b52241fbd3a410a32015dd03783 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 16 Dec 2021 16:02:50 +0100 Subject: [PATCH 109/109] FIX: All the nitty gritty details like unused imports, print statements etc. Changes to be committed: modified: ../../connectivity/AV_compRoutines.py modified: ../../connectivity/ST_compRoutines.py deleted: ../../connectivity/const_def.py modified: ../../connectivity/wilson_sf.py modified: test_connectivity.py modified: test_timefreq.py modified: ../test_specest.py --- syncopy/connectivity/AV_compRoutines.py | 7 +-- syncopy/connectivity/ST_compRoutines.py | 2 +- syncopy/connectivity/const_def.py | 6 --- syncopy/connectivity/wilson_sf.py | 13 ++--- syncopy/tests/backend/test_connectivity.py | 26 ++++----- syncopy/tests/backend/test_timefreq.py | 62 +++++++++++----------- syncopy/tests/test_specest.py | 5 +- 7 files changed, 50 insertions(+), 71 deletions(-) delete mode 100644 syncopy/connectivity/const_def.py diff --git a/syncopy/connectivity/AV_compRoutines.py b/syncopy/connectivity/AV_compRoutines.py index f195a66cf..f23228b13 100644 --- a/syncopy/connectivity/AV_compRoutines.py +++ b/syncopy/connectivity/AV_compRoutines.py @@ -16,14 +16,11 @@ # syncopy imports from syncopy.shared.const_def import spectralDTypes, spectralConversions -from syncopy.shared.errors import SPYWarning from syncopy.shared.computational_routine import ComputationalRoutine from syncopy.shared.kwarg_decorators import unwrap_io from syncopy.shared.errors import ( SPYValueError, - SPYTypeError, - SPYWarning, - SPYInfo) +) from syncopy.connectivity.wilson_sf import wilson_sf, regularize_csd from syncopy.connectivity.granger import granger @@ -98,12 +95,10 @@ def normalize_csd_cF(csd_av_dat, # For initialization of computational routine, # just return output shape and dtype - # cross spectra are complex! if noCompute: return outShape, spectralDTypes[output] # re-shape to (nChannels x nChannels x nFreq) - CS_ij = csd_av_dat.transpose(0, 2, 3, 1)[0, ...] # main diagonal has shape (nFreq x nChannels): the auto spectra diff --git a/syncopy/connectivity/ST_compRoutines.py b/syncopy/connectivity/ST_compRoutines.py index de8bb6e7c..54968b863 100644 --- a/syncopy/connectivity/ST_compRoutines.py +++ b/syncopy/connectivity/ST_compRoutines.py @@ -13,7 +13,7 @@ # syncopy imports from syncopy.specest.mtmfft import mtmfft from syncopy.shared.const_def import spectralDTypes -from syncopy.shared.errors import SPYWarning, SPYValueError +from syncopy.shared.errors import SPYValueError from syncopy.datatype import padding from syncopy.shared.tools import best_match from syncopy.shared.computational_routine import ComputationalRoutine diff --git a/syncopy/connectivity/const_def.py b/syncopy/connectivity/const_def.py deleted file mode 100644 index 5684353f5..000000000 --- a/syncopy/connectivity/const_def.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Constant definitions specific for connectivity - None so far -# - -#: available spectral estimation methods of :func:`~syncopy.connectivity_analysis` diff --git a/syncopy/connectivity/wilson_sf.py b/syncopy/connectivity/wilson_sf.py index ada2fa18a..de80a1851 100644 --- a/syncopy/connectivity/wilson_sf.py +++ b/syncopy/connectivity/wilson_sf.py @@ -48,7 +48,7 @@ def wilson_sf(CSD, nIter=100, rtol=1e-9): iterations. """ - nFreq, nChannels = CSD.shape[:2] + nFreq = CSD.shape[0] Ident = np.eye(*CSD.shape[1:]) @@ -156,7 +156,7 @@ def _plusOperator(g): # --- End of Wilson's Algorithm --- -def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): +def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=15): """ Brute force regularization of CSD matrix @@ -193,7 +193,7 @@ def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): """ - epsilons = np.logspace(-10, np.log10(eps_max), 25) + epsilons = np.logspace(-10, np.log10(eps_max), nSteps) I = np.eye(CSD.shape[1]) CondNum = np.linalg.cond(CSD).max() @@ -211,10 +211,3 @@ def regularize_csd(CSD, cond_max=1e6, eps_max=1e-3, nSteps=50): msg = f"CSD matrix not regularizable with a max epsilon of {eps_max}!" raise ValueError(msg) - - -def _mem_size(arr): - ''' - Gives array size in MB - ''' - return f'{arr.size * arr.itemsize / 1e6:.2f} MB' diff --git a/syncopy/tests/backend/test_connectivity.py b/syncopy/tests/backend/test_connectivity.py index 246dff4b9..d041421ce 100644 --- a/syncopy/tests/backend/test_connectivity.py +++ b/syncopy/tests/backend/test_connectivity.py @@ -65,14 +65,14 @@ def test_coherence(): ax.set_ylim((-.02,1.05)) ax.set_title('Trial average coherence, SNR=1') - assert ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') + ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') # we test for the highest peak sitting at # the vicinity (± 5Hz) of the harmonic peak_val = np.max(coh) peak_idx = np.argmax(coh) peak_freq = freqs[peak_idx] - print(peak_freq, peak_val) + assert harm_freq - 5 < peak_freq < harm_freq + 5 # we test that the peak value @@ -127,14 +127,13 @@ def test_csd(): ax.set_ylim((-.02,1.05)) ax.set_title(f'MTM coherence, {Kmax} tapers, SNR=1') - assert ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') + ax.plot(freqs, coh, lw=1.5, alpha=0.8, c='cornflowerblue') # we test for the highest peak sitting at # the vicinity (± 5Hz) of one the harmonic peak_val = np.max(coh) peak_idx = np.argmax(coh) peak_freq = freqs[peak_idx] - print(peak_freq, peak_val) assert harm_freq - 5 < peak_freq < harm_freq + 5 # we test that the peak value @@ -229,15 +228,16 @@ def test_wilson(): ax.set_ylabel(r'$|CSD_{ij}(f)|$') chan = nChannels // 2 # show (real) auto-spectra - assert ax.plot(freqs, np.abs(CSD[:, chan, chan]), - '-o', label='original CSD', ms=3) - assert ax.plot(freqs, np.abs(CSDreg[:, chan, chan]), - '-o', label='regularized CSD', ms=3) - assert ax.plot(freqs, np.abs(CSDfac[:, chan, chan]), - '-o', label='factorized CSD', ms=3) + ax.plot(freqs, np.abs(CSD[:, chan, chan]), + '-o', label='original CSD', ms=3) + ax.plot(freqs, np.abs(CSDreg[:, chan, chan]), + '-o', label='regularized CSD', ms=3) + ax.plot(freqs, np.abs(CSDfac[:, chan, chan]), + '-o', label='factorized CSD', ms=3) ax.set_xlim((f1 - 5, f2 + 5)) + ax.legend() - + def test_granger(): """ @@ -308,8 +308,8 @@ def test_granger(): fig, ax = ppl.subplots(figsize=(6, 4)) ax.set_xlabel('frequency (Hz)') ax.set_ylabel(r'Granger causality(f)') - assert ax.plot(freqs, G[:, 0, 1], label=r'Granger $1\rightarrow2$') - assert ax.plot(freqs, G[:, 1, 0], label=r'Granger $2\rightarrow1$') + ax.plot(freqs, G[:, 0, 1], label=r'Granger $1\rightarrow2$') + ax.plot(freqs, G[:, 1, 0], label=r'Granger $2\rightarrow1$') ax.legend() # --- Helper routines --- diff --git a/syncopy/tests/backend/test_timefreq.py b/syncopy/tests/backend/test_timefreq.py index 95af47d0d..929f293a6 100644 --- a/syncopy/tests/backend/test_timefreq.py +++ b/syncopy/tests/backend/test_timefreq.py @@ -108,13 +108,13 @@ def test_mtmconvol(): extent = [0, len(signal) / fs, freqs[0] - df / 2, freqs[-1] - df / 2] # test also the plotting # scale with amplitude - assert ax2.imshow(ampls.T, - cmap='magma', - aspect='auto', - origin='lower', - extent=extent, - vmin=0, - vmax=1.2 * A) + ax2.imshow(ampls.T, + cmap='magma', + aspect='auto', + origin='lower', + extent=extent, + vmin=0, + vmax=1.2 * A) # zoom into foi region ax2.set_ylim((foi[0], foi[-1])) @@ -142,9 +142,9 @@ def test_mtmconvol(): cycle_num = (ampls[:, idx] > A / np.e).sum() / fs * frequency print(f'{cycle_num} cycles for the {frequency} band') # we have 2 times the cycles for each frequency (temporal neighbor) - # assert cycle_num > 2 * cycles + assert cycle_num > 2 * cycles # power should decay fast, so we don't detect more cycles - # assert cycle_num < 3 * cycles + assert cycle_num < 3 * cycles fig.tight_layout() @@ -178,13 +178,13 @@ def test_mtmconvol(): # test also the plotting # scale with amplitude - assert ax2.imshow(ampls2.T, - cmap='magma', - aspect='auto', - origin='lower', - extent=extent, - vmin=0, - vmax=1.2 * A) + ax2.imshow(ampls2.T, + cmap='magma', + aspect='auto', + origin='lower', + extent=extent, + vmin=0, + vmax=1.2 * A) # zoom into foi region ax2.set_ylim((foi[0], foi[-1])) @@ -205,7 +205,7 @@ def test_mtmconvol(): # for multi-taper stft we can't # check for the whole time domain - # due to too much spectral broadening + # due to too much spectral broadening/smearing # so we just check that the maximum estimated # amplitude is within 10% boundsof the real amplitude @@ -241,13 +241,13 @@ def test_superlet(): extent = [0, len(signal) / fs, foi[0], foi[-1]] # test also the plotting # scale with amplitude - assert ax2.imshow(ampls, - cmap='magma', - aspect='auto', - extent=extent, - origin='lower', - vmin=0, - vmax=1.2 * A) + ax2.imshow(ampls, + cmap='magma', + aspect='auto', + extent=extent, + origin='lower', + vmin=0, + vmax=1.2 * A) # get the 'mappable' im = ax2.images[0] @@ -301,13 +301,13 @@ def test_wavelet(): # test also the plotting # scale with amplitude - assert ax2.imshow(ampls, - cmap='magma', - aspect='auto', - extent=extent, - origin='lower', - vmin=0, - vmax=1.2 * A) + ax2.imshow(ampls, + cmap='magma', + aspect='auto', + extent=extent, + origin='lower', + vmin=0, + vmax=1.2 * A) # get the 'mappable' im = ax2.images[0] diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index ff073c857..3a48839cd 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -266,8 +266,6 @@ def test_dpss(self): # specify tapers spec = freqanalysis(self.adata, method="mtmfft", taper="dpss", tapsmofrq=7, keeptapers=True, select=select) - # automatic nTaper settings, can be removed?! - # assert spec.taper.size == 7 assert spec.channel.size == len(chanList) # non-equidistant data w/multiple tapers @@ -729,9 +727,8 @@ def test_tf_toi(self): cfg.toi = "all" cfg.t_ftimwin = 0.05 tfSpec = freqanalysis(cfg, self.tfData) - print(self.tfData) assert tfSpec.taper.size >= 1 - dt = 1/self.tfData.samplerate + dt = 1 / self.tfData.samplerate timeArr = np.arange(cfg.select["toilim"][0], cfg.select["toilim"][1] + dt, dt) assert np.allclose(tfSpec.time[0], timeArr) cfg.toi = 1.0