Refactor index, cache, log and preprocessing systems

CHANGELOG.md

@@ -13,6 +13,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 
+## [0.10.0] - 2020-09-28
+### Added
+* Added new dataset versioning system,
+  you can work with multiple versions
+  of the same dataset during creation
+* Added new in-Python download system,
+  you can download a specific version
+  of the dataset during initialization
+
+### Changed
+* Refactored dataset indexing system, 
+  a dedicated directory will be created
+  for the index database during initialization
+* Refactored preprocessing cache system,
+  a dedicated directory will be created
+  and a faster version for cache coherence
+  check has been implemented
+* Refactored logging system
+* Update package requirements
+* Update README
+
+### Removed
+* Deprecated Makefile
+
+
 ## [0.9.3] - 2020-07-05
 
 ### Change

README.md

@@ -35,10 +35,6 @@ PyEEGLab is distributed using the pip repository:
 
     pip install PyEEGLab
 
-If you use Python 3.6, the dataclasses package must be installed as backport of Python 3.7 dataclasses:
-
-    pip install dataclasses
-
 If you need a bleeding edge version, you can install it directly from GitHub:
 
     pip install git+https://github.com/AlessioZanga/PyEEGLab@develop
@@ -99,11 +95,18 @@ The following datasets will work upon downloading:
 
 > **WARNING**: Retriving the TUH EEG datasets require valid credentials, you can get your own at: https://www.isip.piconepress.com/projects/tuh_eeg/html/request_access.php.
 
-In the root directory of this project there is a Makefile, by typing:
+Given the dataset instance, trigger the download using the "download" method:
+
+    from pyeeglab import *
+    dataset = TUHEEGAbnormalDataset()
+    dataset.download(user='USER', password='PASSWORD')
+    dataset.index()
+
+then index the new downloaded files.
 
-    make tuh_eeg_abnormal
+It should be noted that the download mechanism work on Unix-like systems given the following packages:
 
-you will trigger the dataset download.
+    sudo apt install sshpass rsync wget
 
 ## Documentation
 

examples/tensorboard/example_tensorboard.py

@@ -37,8 +37,6 @@
 from pyeeglab import *
 
 def build_data(dataset):
-    dataset.set_cache_manager(PickleCache('../../export'))
-
     preprocessing = Pipeline([
         CommonChannelSet(),
         LowestFrequency(),
@@ -212,18 +210,18 @@ def tune_model(dataset_name, data):
 if __name__ == '__main__':
     dataset = {}
 
-    dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
+    dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
-    dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/v1.0.0/edf')
+    dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/')
     dataset['tuh_eeg_artifact'].set_minimum_event_duration(4)
 
-    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/v1.5.2/edf')
+    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/')
     dataset['tuh_eeg_seizure'].set_minimum_event_duration(4)
 
-    # dataset['eegmmidb'] = EEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/1.0.0')
+    # dataset['eegmmidb'] = PhysioNetEEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/')
     # dataset['eegmmidb'].set_minimum_event_duration(4)
 
-    dataset['chbmit'] = CHBMITDataset('../../data/physionet.org/files/chbmit/1.0.0')
+    dataset['chbmit'] = PhysioNetCHBMITDataset('../../data/physionet.org/files/chbmit/')
     dataset['chbmit'].set_minimum_event_duration(4)
 
     """

examples/tensorboard/example_tensorboard_cnn.py

@@ -39,8 +39,6 @@
 from pyeeglab import *
 
 def build_data(dataset):
-    dataset.set_cache_manager(PickleCache('../../export'))
-
     preprocessing = Pipeline([
         CommonChannelSet(),
         LowestFrequency(),
@@ -201,18 +199,18 @@ def tune_model(dataset_name, data):
 if __name__ == '__main__':
     dataset = {}
 
-    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
+    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
-    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/v1.0.0/edf')
+    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/')
     # dataset['tuh_eeg_artifact'].set_minimum_event_duration(4)
 
-    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/v1.5.2/edf')
+    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/')
     dataset['tuh_eeg_seizure'].set_minimum_event_duration(4)
 
-    # dataset['eegmmidb'] = EEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/1.0.0')
+    # dataset['eegmmidb'] = PhysioNetEEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/')
     # dataset['eegmmidb'].set_minimum_event_duration(4)
 
-    dataset['chbmit'] = CHBMITDataset('../../data/physionet.org/files/chbmit/1.0.0')
+    dataset['chbmit'] = PhysioNetCHBMITDataset('../../data/physionet.org/files/chbmit/')
     dataset['chbmit'].set_minimum_event_duration(4)
 
     """

examples/tensorboard/example_tensorboard_gat.py

@@ -37,8 +37,6 @@
 from pyeeglab import *
 
 def build_data(dataset):
-    dataset.set_cache_manager(PickleCache('../../export'))
-
     preprocessing = Pipeline([
         CommonChannelSet(),
         LowestFrequency(),
@@ -206,18 +204,18 @@ def tune_model(dataset_name, data):
 if __name__ == '__main__':
     dataset = {}
 
-    dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
+    dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
-    dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/v1.0.0/edf')
+    dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/')
     dataset['tuh_eeg_artifact'].set_minimum_event_duration(4)
 
-    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/v1.5.2/edf')
+    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/')
     dataset['tuh_eeg_seizure'].set_minimum_event_duration(4)
 
-    # dataset['eegmmidb'] = EEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/1.0.0')
+    # dataset['eegmmidb'] = PhysioNetEEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/')
     # dataset['eegmmidb'].set_minimum_event_duration(4)
 
-    dataset['chbmit'] = CHBMITDataset('../../data/physionet.org/files/chbmit/1.0.0')
+    dataset['chbmit'] = PhysioNetCHBMITDataset('../../data/physionet.org/files/chbmit/')
     dataset['chbmit'].set_minimum_event_duration(4)
 
     """

examples/tensorboard/example_tensorboard_gnn.py

@@ -39,8 +39,6 @@
 from pyeeglab import *
 
 def build_data(dataset):
-    dataset.set_cache_manager(PickleCache('../../export'))
-
     preprocessing = Pipeline([
         CommonChannelSet(),
         LowestFrequency(),
@@ -208,18 +206,18 @@ def tune_model(dataset_name, data):
 if __name__ == '__main__':
     dataset = {}
 
-    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
+    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
-    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/v1.0.0/edf')
+    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/')
     # dataset['tuh_eeg_artifact'].set_minimum_event_duration(4)
 
-    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/v1.5.2/edf')
+    dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/')
     dataset['tuh_eeg_seizure'].set_minimum_event_duration(4)
 
-    # dataset['eegmmidb'] = EEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/1.0.0')
+    # dataset['eegmmidb'] = PhysioNetEEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/')
     # dataset['eegmmidb'].set_minimum_event_duration(4)
 
-    dataset['chbmit'] = CHBMITDataset('../../data/physionet.org/files/chbmit/1.0.0')
+    dataset['chbmit'] = PhysioNetCHBMITDataset('../../data/physionet.org/files/chbmit/')
     dataset['chbmit'].set_minimum_event_duration(4)
 
     """

examples/tensorboard/example_validation.py

@@ -38,8 +38,6 @@
 from pyeeglab import *
 
 def build_data(dataset):
-    dataset.set_cache_manager(PickleCache('../../export'))
-
     preprocessing = Pipeline([
         CommonChannelSet(),
         LowestFrequency(),
@@ -220,18 +218,18 @@ def tune_model(dataset_name, data, metrics, folds, val_size=0.1):
 if __name__ == '__main__':
     dataset = {}
 
-    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
+    # dataset['tuh_eeg_abnormal'] = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
-    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/v1.0.0/edf')
+    # dataset['tuh_eeg_artifact'] = TUHEEGArtifactDataset('../../data/tuh_eeg_artifact/')
     # dataset['tuh_eeg_artifact'].set_minimum_event_duration(4)
 
-    # dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/v1.5.2/edf')
+    # dataset['tuh_eeg_seizure'] = TUHEEGSeizureDataset('../../data/tuh_eeg_seizure/')
     # dataset['tuh_eeg_seizure'].set_minimum_event_duration(4)
 
-    # dataset['eegmmidb'] = EEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/1.0.0')
+    # dataset['eegmmidb'] = PhysioNetEEGMMIDBDataset('../../data/physionet.org/files/eegmmidb/')
     # dataset['eegmmidb'].set_minimum_event_duration(4)
 
-    dataset['chbmit'] = CHBMITDataset('../../data/physionet.org/files/chbmit/1.0.0')
+    dataset['chbmit'] = PhysioNetCHBMITDataset('../../data/physionet.org/files/chbmit/')
     dataset['chbmit'].set_minimum_event_duration(4)
 
     """

examples/tuh_eeg_abnormal/example_cnn_dense_classification.py

@@ -21,12 +21,11 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, ToDataframe, DynamicWindow, BinarizedSpearmanCorrelation, \
                         ToNumpy
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

examples/tuh_eeg_abnormal/example_cnn_dense_classification_with_filter.py

@@ -21,12 +21,11 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, BandPassFrequency, ToDataframe, DynamicWindow, \
                         BinarizedSpearmanCorrelation, ToNumpy
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

examples/tuh_eeg_abnormal/example_cnn_lstm_classification.py

@@ -21,12 +21,11 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, ToDataframe, DynamicWindow, BinarizedSpearmanCorrelation, \
                         ToNumpy
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

examples/tuh_eeg_abnormal/example_cnn_lstm_classification_with_filter.py

@@ -21,12 +21,11 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, BandPassFrequency, ToDataframe, DynamicWindow, \
                         BinarizedSpearmanCorrelation, ToNumpy
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

examples/tuh_eeg_abnormal/example_gat_lstm_classification.py

@@ -23,12 +23,11 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, ToDataframe, DynamicWindow, BinarizedSpearmanCorrelation, \
                         CorrelationToAdjacency, Bandpower, GraphWithFeatures, ForkedPreprocessor
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

examples/tuh_eeg_abnormal/example_gat_lstm_classification_with_filter.py

@@ -23,13 +23,12 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
-from pyeeglab import    TUHEEGAbnormalDataset, PickleCache, Pipeline, CommonChannelSet, \
+from pyeeglab import    TUHEEGAbnormalDataset, Pipeline, CommonChannelSet, \
                         LowestFrequency, BandPassFrequency, ToDataframe, DynamicWindow, \
                         BinarizedSpearmanCorrelation, CorrelationToAdjacency, Bandpower, \
                         GraphWithFeatures, ForkedPreprocessor
 
-dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/v2.0.0/edf')
-dataset.set_cache_manager(PickleCache('../../export'))
+dataset = TUHEEGAbnormalDataset('../../data/tuh_eeg_abnormal/')
 
 preprocessing = Pipeline([
     CommonChannelSet(),

pyeeglab/__init__.py

@@ -1,15 +1,11 @@
 import logging
 import warnings
 
-from importlib.util import find_spec
-from mne.utils import set_config
+logging.basicConfig(format="%(asctime)s %(levelname)7s: %(message)s", datefmt="%Y/%m/%d %H:%M:%S")
 
 from .dataset import *
-from .io import Raw
-from .cache import PickleCache
-from .pipeline import Pipeline, ForkedPreprocessor
+from .pipeline import *
 from .preprocess import *
 
 logging.getLogger().setLevel(logging.DEBUG)
-
 warnings.filterwarnings("ignore", category=RuntimeWarning)