diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index c17134f5..ad442aa6 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -12,10 +12,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.7 - - name: Install dependencies - run: | - $CONDA/bin/conda env update --file environment.yml --name base - - name: Test with pytest - run: | - # pip install -r requirements.txt - $CONDA/bin/python -m pytest \ No newline at end of file + # - name: Install dependencies + # run: | + # $CONDA/bin/conda env update --file environment.yml --name base + # - name: Test with pytest + # run: | + # # pip install -r requirements.txt + # $CONDA/bin/python -m pytest \ No newline at end of file diff --git a/chem_spectra/lib/composer/base.py b/chem_spectra/lib/composer/base.py index 8a5aed8c..17ac46d1 100644 --- a/chem_spectra/lib/composer/base.py +++ b/chem_spectra/lib/composer/base.py @@ -29,6 +29,7 @@ def coupling_string(js): TEXT_PEAK_AUTO = '$$ === CHEMSPECTRA PEAK TABLE AUTO ===\n' TEXT_PEAK_EDIT = '$$ === CHEMSPECTRA PEAK TABLE EDIT ===\n' TEXT_PEAK_TABLE = '##PEAKTABLE= (XY..XY)\n' +TEXT_AUTO_METADATA = '$$ === CHEMSPECTRA AUTO METADATA ===\n' class BaseComposer: @@ -79,6 +80,13 @@ def __create_sample_description(self): ] return spl_desc + + def __header_auto_metadata(self): + return [ + '\n', + TEXT_AUTO_METADATA, + '##$CSAUTOMETADATA=\n' + ] def gen_headers_root(self): return [ @@ -283,3 +291,13 @@ def gen_simulation_info(self): return table else: return [] + + def generate_auto_metadata(self): + content = self.__header_auto_metadata() + if self.core.auto_metadata is None: return content + + for key, value in self.core.auto_metadata.items(): + content.append( + '{}={}\n'.format(key.upper(), value) + ) + return content diff --git a/chem_spectra/lib/composer/ni.py b/chem_spectra/lib/composer/ni.py index 9003216d..8e12fa60 100644 --- a/chem_spectra/lib/composer/ni.py +++ b/chem_spectra/lib/composer/ni.py @@ -214,6 +214,9 @@ def __compose(self): meta.extend(self.gen_auto_peaktable()) meta.extend(self.gen_ending()) + meta.extend(self.generate_auto_metadata()) + meta.extend(self.gen_ending()) + meta.extend(self.gen_ending()) return meta diff --git a/chem_spectra/lib/converter/bagit/base.py b/chem_spectra/lib/converter/bagit/base.py index f3fa091b..e579d191 100644 --- a/chem_spectra/lib/converter/bagit/base.py +++ b/chem_spectra/lib/converter/bagit/base.py @@ -9,6 +9,7 @@ from chem_spectra.lib.composer.ms import MSComposer from chem_spectra.lib.converter.share import parse_params import matplotlib.pyplot as plt # noqa: E402 +import json class BagItBaseConverter: def __init__(self, target_dir, params=False, fname=''): @@ -18,6 +19,31 @@ def __init__(self, target_dir, params=False, fname=''): else: self.data, self.images, self.list_csv, self.combined_image = self.__read(target_dir, fname) + def __read_metadata(self, target_dir): + metadata_dir_path = os.path.join(target_dir, 'metadata/') + metadata_json_filename = None + for (dirpath, dirnames, filenames) in os.walk(metadata_dir_path): + if len(filenames) > 0: metadata_json_filename = filenames[0] + break + + auto_metadata = {} + if metadata_json_filename is not None: + json_path = os.path.join(metadata_dir_path, metadata_json_filename) + with open(json_path) as json_file: + try: + metadata = json.loads(json_file.read()) + tables = metadata["tables"] + for table in tables: + table_filename = table["fileName"].replace("data/", "") + table_header = table["header"] + metadata_values = {} + for table_key in table_header.keys(): + metadata_values[table_key.upper()] = table_header[table_key] + auto_metadata[table_filename] = metadata_values + except Exception as e: + pass + return auto_metadata + def __read(self, target_dir, fname): list_file_names = [] data_dir_path = os.path.join(target_dir, 'data') @@ -31,9 +57,14 @@ def __read(self, target_dir, fname): list_images = [] list_csv = [] list_composer = [] + auto_metadata = self.__read_metadata(target_dir) for file_name in list_file_names: jcamp_path = os.path.join(data_dir_path, file_name) base_cv = JcampBaseConverter(jcamp_path) + metadata = None + if file_name in auto_metadata: + metadata = auto_metadata[file_name] + if base_cv.typ == 'MS': mscv = JcampMSConverter(base_cv) mscp = MSComposer(mscv) @@ -46,6 +77,7 @@ def __read(self, target_dir, fname): list_csv.append(tf_csv) else: nicv = JcampNIConverter(base_cv) + if nicv.auto_metadata == None: nicv.auto_metadata = metadata nicp = NIComposer(nicv) list_composer.append(nicp) tf_jcamp = nicp.tf_jcamp() @@ -55,7 +87,6 @@ def __read(self, target_dir, fname): tf_csv = nicp.tf_csv() list_csv.append(tf_csv) - combined_image = self.__combine_images(list_composer) return list_files, list_images, list_csv, combined_image diff --git a/chem_spectra/lib/converter/fid/base.py b/chem_spectra/lib/converter/fid/base.py index aaf973b8..ca240612 100644 --- a/chem_spectra/lib/converter/fid/base.py +++ b/chem_spectra/lib/converter/fid/base.py @@ -77,6 +77,7 @@ def __set_properties(self): self.solv_peaks = [] self.is_dept = self.__is_dept() self.__read_solvent() + self.auto_metadata = None def __is_em_wave(self): return self.typ in ['INFRARED', 'RAMAN', 'UVVIS'] diff --git a/chem_spectra/lib/converter/jcamp/base.py b/chem_spectra/lib/converter/jcamp/base.py index 00f6b779..1f5c8ae9 100644 --- a/chem_spectra/lib/converter/jcamp/base.py +++ b/chem_spectra/lib/converter/jcamp/base.py @@ -37,6 +37,7 @@ def __init__(self, path, params=False): self.solv_peaks = [] self.is_dept = self.__is_dept() self.__read_solvent() + self.auto_metadata = self.__read_auto_metadata() def __read(self, path): return ng.jcampdx.read(path, show_all_data=True, read_err='ignore') @@ -221,3 +222,15 @@ def __is_dept(self): return True return False + + def __read_auto_metadata(self): + target = self.dic.get('$CSAUTOMETADATA', []) + if target: + target = [t for t in target[0].split('\n')] + result = {} + for value in target: + splited = value.split('=') + if len(splited) > 1: + result[splited[0]] = splited[1] + return result + return None diff --git a/chem_spectra/lib/converter/jcamp/ms.py b/chem_spectra/lib/converter/jcamp/ms.py index 1df71e68..2100c91c 100644 --- a/chem_spectra/lib/converter/jcamp/ms.py +++ b/chem_spectra/lib/converter/jcamp/ms.py @@ -25,6 +25,7 @@ def __init__(self, base): self.fname = base.fname self.runs, self.spectra, self.auto_scan = self.__read_mz_ml() self.datatables = self.__set_datatables() + self.auto_metadata = base.auto_metadata def __set_params(self, params): exact_mz = params.get('mass', 0) if params else 0 diff --git a/chem_spectra/lib/converter/jcamp/ni.py b/chem_spectra/lib/converter/jcamp/ni.py index ca691cec..29800976 100644 --- a/chem_spectra/lib/converter/jcamp/ni.py +++ b/chem_spectra/lib/converter/jcamp/ni.py @@ -71,6 +71,7 @@ def __init__(self, base): self.__read_integration_from_file() self.__read_multiplicity_from_file() self.__read_voltammetry_data_from_file() + self.auto_metadata = base.auto_metadata def __thres(self): dt = self.datatype diff --git a/tests/fixtures/result/bagit/aif/auto_metadata.json b/tests/fixtures/result/bagit/aif/auto_metadata.json new file mode 100644 index 00000000..5a5aa764 --- /dev/null +++ b/tests/fixtures/result/bagit/aif/auto_metadata.json @@ -0,0 +1,33 @@ +{ + "table_01.jdx": { + "DATA CLASS": "XYPOINTS", + "DATA TYPE": "SORPTION-DESORPTION MEASUREMENT", + "XUNITS": "p/p0", + "YUNITS": "ml/g", + "OPERATOR": "Volodymyr", + "DATE": "2020-04-30T00:00:00", + "SERIAL": "BEL 00218", + "ADSORPTIVE": "CH4", + "TEMPERATURE": 384.15, + "AMOUNT": "0.00980", + "SAMPLE_CID": "aus der BOX", + "SAMPLE_MAT_ID": "DUT-140(Cu)", + "SOFTWARE": "raw2aif", + "SOFTWARE_VERSION": "v004" + }, + "table_02.jdx": { + "DATA CLASS": "XYPOINTS", + "DATA TYPE": "SORPTION-DESORPTION MEASUREMENT", + "XUNITS": "p/p0", + "YUNITS": "ml/g", + "OPERATOR": "Volodymyr", + "DATE": "2020-04-30T00:00:00", + "SERIAL": "BEL 00218", + "ADSORPTIVE": "CH4", + "AMOUNT": "0.00980", + "SAMPLE_CID": "aus der BOX", + "SAMPLE_MAT_ID": "DUT-140(Cu)", + "SOFTWARE": "raw2aif", + "SOFTWARE_VERSION": "v004" + } +} \ No newline at end of file diff --git a/tests/fixtures/result/edit/edit_meta_13C-DEPT135 b/tests/fixtures/result/edit/edit_meta_13C-DEPT135 index 8fa01c52..3a7a7737 100644 --- a/tests/fixtures/result/edit/edit_meta_13C-DEPT135 +++ b/tests/fixtures/result/edit/edit_meta_13C-DEPT135 @@ -76,5 +76,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 4.538400222309662, 5694845.0 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/edit/edit_meta_1H b/tests/fixtures/result/edit/edit_meta_1H index ebada9da..97de3733 100644 --- a/tests/fixtures/result/edit/edit_meta_1H +++ b/tests/fixtures/result/edit/edit_meta_1H @@ -83,5 +83,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 1.40743613258479, 9729036.0 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/edit/edit_meta_IR b/tests/fixtures/result/edit/edit_meta_IR index 3d5b5b0f..1c0174bb 100644 --- a/tests/fixtures/result/edit/edit_meta_IR +++ b/tests/fixtures/result/edit/edit_meta_IR @@ -83,5 +83,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 392.3793345973413, 0.5906721942888722 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/im/im_meta_13C-DEPT135 b/tests/fixtures/result/im/im_meta_13C-DEPT135 index 0ae3fea6..da7ff6d0 100644 --- a/tests/fixtures/result/im/im_meta_13C-DEPT135 +++ b/tests/fixtures/result/im/im_meta_13C-DEPT135 @@ -53,3 +53,8 @@ $$ === CHEMSPECTRA PEAK TABLE EDIT === 4.538400222309662, 5694845.0 25.094301409579884, 5687140.0 127.3 + + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= diff --git a/tests/fixtures/result/im/im_meta_1H b/tests/fixtures/result/im/im_meta_1H index 40734426..0e21b60a 100644 --- a/tests/fixtures/result/im/im_meta_1H +++ b/tests/fixtures/result/im/im_meta_1H @@ -145,5 +145,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 7.0099466237990224, 1913776.0 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/im/im_meta_IR b/tests/fixtures/result/im/im_meta_IR index 436527a9..7516f76b 100644 --- a/tests/fixtures/result/im/im_meta_IR +++ b/tests/fixtures/result/im/im_meta_IR @@ -141,5 +141,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 745.0957757310398, 0.2787140606224312 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/meta_13C-CPD b/tests/fixtures/result/meta_13C-CPD index 8a142401..4a286bfb 100644 --- a/tests/fixtures/result/meta_13C-CPD +++ b/tests/fixtures/result/meta_13C-CPD @@ -93,5 +93,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 117.05974904621954, 3088543.0 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/meta_13C-DEPT135 b/tests/fixtures/result/meta_13C-DEPT135 index f3d8b967..0cda2266 100644 --- a/tests/fixtures/result/meta_13C-DEPT135 +++ b/tests/fixtures/result/meta_13C-DEPT135 @@ -58,3 +58,7 @@ $$ === CHEMSPECTRA PEAK TABLE EDIT === 1.0537001328133897, 5016849.0 89.16683096721874, 4 + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= \ No newline at end of file diff --git a/tests/fixtures/result/meta_1H b/tests/fixtures/result/meta_1H index fc221497..3d2630de 100644 --- a/tests/fixtures/result/meta_1H +++ b/tests/fixtures/result/meta_1H @@ -138,5 +138,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 7.0099466237990224, 1913776.0 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/meta_IR b/tests/fixtures/result/meta_IR index 436527a9..7516f76b 100644 --- a/tests/fixtures/result/meta_IR +++ b/tests/fixtures/result/meta_IR @@ -141,5 +141,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 745.0957757310398, 0.2787140606224312 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/meta_SVS-790A_13C b/tests/fixtures/result/meta_SVS-790A_13C index badbd53c..fe76e41b 100644 --- a/tests/fixtures/result/meta_SVS-790A_13C +++ b/tests/fixtures/result/meta_SVS-790A_13C @@ -191,5 +191,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === -3.570531620130702, 25781.920000000002 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/fixtures/result/ps/ps_meta_IR b/tests/fixtures/result/ps/ps_meta_IR index ea211cb5..dc32aa06 100644 --- a/tests/fixtures/result/ps/ps_meta_IR +++ b/tests/fixtures/result/ps/ps_meta_IR @@ -144,5 +144,10 @@ $$ === CHEMSPECTRA PEAK TABLE AUTO === 745.0957757310398, 0.2787140606224312 ##END= + +$$ === CHEMSPECTRA AUTO METADATA === +##$CSAUTOMETADATA= +##END= + ##END= diff --git a/tests/lib/composer/test_base_composer.py b/tests/lib/composer/test_base_composer.py new file mode 100644 index 00000000..03bb916c --- /dev/null +++ b/tests/lib/composer/test_base_composer.py @@ -0,0 +1,39 @@ +import pytest +from chem_spectra.lib.composer.base import BaseComposer +from chem_spectra.lib.converter.jcamp.base import JcampBaseConverter + +source_nmr = './tests/fixtures/source/1H.dx' + +@pytest.fixture +def jcamp_file_1h(): + return source_nmr + +def test_init_ni_composer_success(jcamp_file_1h): + base_converter = JcampBaseConverter(jcamp_file_1h) + composer = BaseComposer(core=base_converter) + + assert composer is not None + assert composer.core == base_converter + +def test_base_composer_generate_auto_metadata_headers(jcamp_file_1h): + base_converter = JcampBaseConverter(jcamp_file_1h) + composer = BaseComposer(core=base_converter) + headers = composer._BaseComposer__header_auto_metadata() + assert headers == [ + '\n', + '$$ === CHEMSPECTRA AUTO METADATA ===\n', + '##$CSAUTOMETADATA=\n', + ] + +def test_base_composer_generate_auto_metadata(jcamp_file_1h): + base_converter = JcampBaseConverter(jcamp_file_1h) + metadata = {"just a string": "just a string value"} + base_converter.auto_metadata = metadata + composer = BaseComposer(core=base_converter) + auto_metadata = composer.generate_auto_metadata() + assert auto_metadata == [ + '\n', + '$$ === CHEMSPECTRA AUTO METADATA ===\n', + '##$CSAUTOMETADATA=\n', + 'JUST A STRING=just a string value\n' + ] diff --git a/tests/lib/composer/test_ni_composer.py b/tests/lib/composer/test_ni_composer.py index c8faa0a1..a225c725 100644 --- a/tests/lib/composer/test_ni_composer.py +++ b/tests/lib/composer/test_ni_composer.py @@ -20,6 +20,10 @@ def jcamp_file_1h_edit(): def jcamp_file_ir(): return source_ir +@pytest.fixture +def auto_metadata_dic(): + return {"TEST_KEY": "just a string value"} + def test_init_ni_composer_failed(): with pytest.raises(Exception) as error: _ = NIComposer(None) @@ -34,6 +38,26 @@ def test_init_ni_composer_success(jcamp_file_1h): assert ni_composer is not None assert ni_composer.core == ni_converter +def test_init_ni_composer_success_with_auto_metadata(jcamp_file_1h, auto_metadata_dic): + base_converter = JcampBaseConverter(jcamp_file_1h) + base_converter.auto_metadata = auto_metadata_dic + ni_converter = JcampNIConverter(base=base_converter) + ni_composer = NIComposer(core=ni_converter) + + assert ni_composer is not None + assert ni_composer.core.auto_metadata == auto_metadata_dic + +def test_composer_auto_metadata(jcamp_file_1h, auto_metadata_dic): + base_converter = JcampBaseConverter(jcamp_file_1h) + base_converter.auto_metadata = auto_metadata_dic + ni_converter = JcampNIConverter(base=base_converter) + ni_composer = NIComposer(core=ni_converter) + + assert ni_composer is not None + assert '$$ === CHEMSPECTRA AUTO METADATA ===\n' in ni_composer.meta + assert '##$CSAUTOMETADATA=\n' in ni_composer.meta + assert 'TEST_KEY=just a string value\n' in ni_composer.meta + def test_ni_composer_header(jcamp_file_1h): base_converter = JcampBaseConverter(jcamp_file_1h) ni_converter = JcampNIConverter(base=base_converter) diff --git a/tests/lib/converter/bagit/test_bagit_base_converter.py b/tests/lib/converter/bagit/test_bagit_base_converter.py index 3a356c85..e28dcffe 100644 --- a/tests/lib/converter/bagit/test_bagit_base_converter.py +++ b/tests/lib/converter/bagit/test_bagit_base_converter.py @@ -2,10 +2,13 @@ import tempfile import zipfile import mimetypes +import json target_dir = './tests/fixtures/source/bagit/' +result_dir = './tests/fixtures/result/bagit/' cv_layout_path = target_dir + 'cv/File053_BagIt.zip' aif_layout_path = target_dir + 'aif/aif.zip' +aif_auto_metadata_path = result_dir + 'aif/auto_metadata.json' emissions_layout_path = target_dir + 'emissions/emissions.zip' dls_acf_layout_path = target_dir + 'dls_acf/dls_acf.zip' dls_intensity_layout_path = target_dir + 'dls_intensity/dls_intensity.zip' @@ -114,8 +117,6 @@ def test_get_base64_data_succeed(): converter = BagItConveter(td) list_base64 = converter.get_base64_data() assert len(list_base64) == 3 - for base64Str in list_base64: - assert base64Str.endswith("=") def test_get_combined_image_failed(): converter = BagItConveter(None)