diff --git a/CHANGES.txt b/CHANGES.txt index b18189dd94..cb099ca3e8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,13 @@ -* V2.5.3.1 +* V2.5.4 * added support to automatically add simple packages from Git: ck add package:my-git-repo --git={URL} --tags=my-repo --env_ext=MY_GIT_REPO * added module:mlperf.result to abstract official MLPerf results + * added key "skip_global_deps" to program meta to skip global dependencies + for a given command line (for example, only to install python deps + for a given program) + * improved handling of a situation when CK environment has changed + and make it possible to continue running a workflow at user risk + (useful for debugging) * V2.5.3 * added "pull" function to module:docker diff --git a/ck/kernel.py b/ck/kernel.py index 7e7f4ceeca..f1fdb94bf0 100755 --- a/ck/kernel.py +++ b/ck/kernel.py @@ -28,7 +28,7 @@ # We use 3 digits for the main (released) version and 4th digit for development revision -__version__ = "2.5.3.1" +__version__ = "2.5.4" # Do not use characters (to detect outdated version)! # Import packages that are global for the whole kernel diff --git a/ck/repo/module/env/module.py b/ck/repo/module/env/module.py index ddc62eff74..e41d0844de 100644 --- a/ck/repo/module/env/module.py +++ b/ck/repo/module/env/module.py @@ -896,11 +896,19 @@ def env_set(i): ck.out('') rx=ck.inp({'text':'Would you like to remove outdated environment entry from CK (Y/n)? '}) - x=rx['string'].strip() + x=rx['string'].strip().lower() if x=='n' or x=='no': - return {'return':1, 'error':err} - to_delete=True + x='' + if o=='con': + ck.out('') + rx=ck.inp({'text':'Would you like to continue at your own risk (y/N)? '}) + x=rx['string'].strip().lower() + + if x!='y': + return {'return':1, 'error':err} + else: + to_delete=True # Deleting outdated environment if to_delete: diff --git a/ck/repo/module/program/module.py b/ck/repo/module/program/module.py index 671f8bb9b9..ba54c8386b 100644 --- a/ck/repo/module/program/module.py +++ b/ck/repo/module/program/module.py @@ -1868,6 +1868,7 @@ def process_in_dir(i): 'preset_deps':preset_deps, 'random':ran, 'safe':safe, + 'skip_global_deps':vcmd.get('skip_global_deps',''), 'quiet':quiet}) if rx['return']>0: return rx @@ -6751,6 +6752,9 @@ def update_run_time_deps(i): (to have more deterministic build) (preset_deps) - to preset deps + + (skip_global_deps) - if 'yes', skip global deps + (useful to install deps) } Output: { @@ -6800,7 +6804,8 @@ def update_run_time_deps(i): rr={'return':0} if len(rdeps)==0: - rdeps.update(meta.get('run_deps',{})) + if i.get('skip_global_deps','')!='yes': + rdeps.update(meta.get('run_deps',{})) rdeps.update(vcmd.get('run_deps',{})) # Prune if needed diff --git a/incubator/README.md b/incubator/README.md index 518ee61b1b..9ac52b3ddf 100644 --- a/incubator/README.md +++ b/incubator/README.md @@ -2,6 +2,7 @@ This directory contains community projects to improve the CK framework or design the new one: -* [CK lite](ck-lite) -* [cBench](cbench) -* [connectMe](connectme) +* [CK lite](ck-lite) - light CMD for CK repos +* [cDataBase](cdatabase) - Pythonic interface to CK repos +* [cBench](cbench) - collaborative benchmarking toolset connected with cKnowledge.io +* [connectMe](connectme) - CK3 prototyping diff --git a/incubator/cdatabase/CONTRIBUTING.md b/incubator/cdatabase/CONTRIBUTING.md new file mode 100644 index 0000000000..82a2a343ee --- /dev/null +++ b/incubator/cdatabase/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contributing to cDataBase + +TBD diff --git a/incubator/cdatabase/LICENSE b/incubator/cdatabase/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/incubator/cdatabase/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/incubator/cdatabase/README.md b/incubator/cdatabase/README.md new file mode 100644 index 0000000000..a60b48a15d --- /dev/null +++ b/incubator/cdatabase/README.md @@ -0,0 +1,12 @@ +# Collective DataBase or cDB + + + +## License +cDB is licensed under the Apache License 2.0. + +See [LICENSE](https://github.com/gfursin/database/blob/master/LICENSE) for more information. + +## Support + +[Create a GitHub issue](https://github.com/gfursin/cdatabase/issues/new/choose) diff --git a/incubator/cdatabase/_clean.bat b/incubator/cdatabase/_clean.bat new file mode 100644 index 0000000000..63013b6294 --- /dev/null +++ b/incubator/cdatabase/_clean.bat @@ -0,0 +1 @@ +rmdir /Q /S build dist diff --git a/incubator/cdatabase/cdatabase/__init__.py b/incubator/cdatabase/cdatabase/__init__.py new file mode 100644 index 0000000000..c957b9b6cb --- /dev/null +++ b/incubator/cdatabase/cdatabase/__init__.py @@ -0,0 +1,5 @@ +# +# cDatabase init +# + +#from connectme.cm import CM diff --git a/incubator/cdatabase/cdatabase/cdata.py b/incubator/cdatabase/cdatabase/cdata.py new file mode 100644 index 0000000000..eb53bceeef --- /dev/null +++ b/incubator/cdatabase/cdatabase/cdata.py @@ -0,0 +1,291 @@ +# +# Low-level data abstraction in cDatabase +# + +import os +import json +import yaml + +from cdatabase.config import cfg + +class cData(object): + ########################################################################### + def __init__(self, path: str): + + """ + Initialize cData + + Args: + path (str): Path to cData. + """ + + self.path = path + self.valid = False + self.loaded = False + self.meta = {} + + # If a data was loaded in the old CK format + self.ck = {} + self.ck_format = False + + # Extra info + self.object_id = '' + self.object_name = '' + self.data_id = '' + self.data_name = '' + self.title = '' + + self.error = '' + + ########################################################################### + def is_valid(self): + + """ + Check if there is a valid cData object in a given path + + Args: + None + + Returns + (bool) : True if valid or False otherwise + """ + + valid = False + + if os.path.isdir(self.path): + p_yaml = os.path.join(self.path, cfg['cdb_meta_desc_yaml']) + if os.path.isfile(p_yaml): + valid = True + else: + p_json = os.path.join(self.path, cfg['cdb_meta_desc_json']) + if os.path.isfile(p_json): + valid = True + else: + p_old = os.path.join(self.path, cfg['ck_dir']) + if os.path.isdir(p_old): + valid = True + self.ck_format = True + + self.valid = valid + + return valid + + ########################################################################### + def load(self, force_ck_format: bool = False, + ignore_errors: bool = False): + + """ + Load cData from a given path. + + Args: + force_ck_format (bool) = False - if True, load data in the old CK format. + ignore_errors (bool) = False - if True, ignore file errors + """ + + if not os.path.isdir(self.path): + raise RuntimeError("Path to cData ("+self.path+") doesn't exit") + + self.meta = {} + + p_yaml = os.path.join(self.path, cfg['cdb_meta_desc_yaml']) + p_json = os.path.join(self.path, cfg['cdb_meta_desc_json']) + + if not force_ck_format and (os.path.isfile(p_yaml) or os.path.isfile(p_json)): + # First load YAML as a base (optional) + if os.path.isfile(p_yaml): + try: + with open(p_yaml, 'rt' , encoding='utf8') as yaml_file: + tmp_data = yaml.load(yaml_file, Loader=yaml.FullLoader) + self.meta.update(tmp_data) + self.loaded = True + except Exception as e: + self.error = e + if not ignore_errors: + raise e + + if os.path.isfile(p_json): + try: + with open(p_json, 'rt' , encoding='utf8') as json_file: + tmp_data = json.load(json_file) + self.meta.update(tmp_data) + self.loaded = True + except Exception as e: + self.error = e + if not ignore_errors: + raise e + + else: + p_old = os.path.join(self.path, cfg['ck_dir']) + + # Check old CK version + if os.path.isdir(p_old): + files = [(cfg['ck_desc_file'], 'desc'), + (cfg['ck_info_file'], 'info'), + (cfg['ck_meta_file'], 'meta')] + + for f in files: + ck_file = os.path.join(p_old, f[0]) + if os.path.isfile(ck_file): + try: + with open(ck_file, 'rt' , encoding='utf8') as json_file: + tmp_data = json.load(json_file) + self.meta.update(tmp_data) + + self.ck[f[1]]=tmp_data + + self.loaded = True + self.ck_format = True + except Exception as e: + self.error = e + if not ignore_errors: + raise e + + if self.loaded: + self.valid = True + + # Check special meta information + self.data_name = os.path.basename(self.path) + + self.object_id = self.meta.get('object_id', '') + if self.object_id == '': + self.object_id = self.meta.get('backup_module_uid','') + + self.object_name = self.meta.get('object_name', '') + if self.object_name == '': + self.object_name = self.meta.get('backup_module_uoa','') + + self.data_id = self.meta.get('data_id', '') + if self.data_id == '': + self.data_id = self.meta.get('backup_data_uid','') + + self.title = self.meta.get('title', '') + if self.title == '': + self.title = self.meta.get('data_name','') + if self.title !='' and self.title == self.data_name: + self.title = '' + + return self.loaded + + ########################################################################### + def is_loaded(self): + + """ + Args: + None + + Returns: + True if loaded or False otherwise. + """ + + return self.loaded + + ########################################################################### + def is_ck_format(self): + + """ + Args: + None + + Returns: + True if loaded in the old CK format. + """ + + return self.ck_format + + ########################################################################### + def get_meta(self): + + """ + Args: + None + + Returns: + (dict) : cData meta + """ + + return self.meta + + ########################################################################### + def get_path(self): + + """ + Delete cData from a given path. + + Args: + None + """ + + return self.path + + ########################################################################### + def save(self, force_yaml: bool = False): + + """ + Save cData to a given path. + Note that by default we record the whole meta into JSON file and not YAML. + If we force YAML, we delete JSON (if exists). + + Args: + None + """ + + if not os.path.isdir(self.path): + os.makedirs(self.path) + + p_json = os.path.join(self.path, cfg['cdb_meta_desc_json']) + + if force_yaml: + p_yaml = os.path.join(self.path, cfg['cdb_meta_desc_yaml']) + + with open(p_yaml, 'w' , encoding='utf8') as yaml_file: + yaml.dump(self. meta, yaml_file, sort_keys=True) + + if os.path.isfile(p_json): + os.remove(p_json) + + else: + with open(p_json, 'w' , encoding='utf8') as json_file: + json.dump(self. meta, json_file, indent=2, sort_keys=True) + + return True + + ########################################################################### + def belongs_to_object(self, object_id: str): + + """ + Check if data belong to a given object + + Args: + object_id (str) - ID of an cDB object + """ + + return self.object_id == object_id + + ########################################################################### + def belongs_to_id(self, data_id: str): + + """ + Check if data belong to a given ID + + Args: + data_id (str) - ID of an cDB object + """ + + return self.data_id == data_id + + ########################################################################### + def delete(self): + + """ + Delete cData from a given path. + + Args: + None + """ + + if os.path.isdir(self.path): + import shutil + shutil.rmtree(self.path) + + return True + diff --git a/incubator/cdatabase/cdatabase/ck.py b/incubator/cdatabase/cdatabase/ck.py new file mode 100644 index 0000000000..068a5dd870 --- /dev/null +++ b/incubator/cdatabase/cdatabase/ck.py @@ -0,0 +1,10 @@ +# +# Compatibility layer with previous CK framework +# + +########################################################################### +def access(i): + + print (i) + + return {'return':1, 'error':'TO BE IMPLEMENTED'} diff --git a/incubator/cdatabase/cdatabase/cobject.py b/incubator/cdatabase/cdatabase/cobject.py new file mode 100644 index 0000000000..1bcf2904f5 --- /dev/null +++ b/incubator/cdatabase/cdatabase/cobject.py @@ -0,0 +1,80 @@ +# +# Low-level object abstraction in cDatabase +# + +import os +import json +import yaml + +from cdatabase.config import cfg +from cdatabase.cdata import cData + +import cdatabase.utils as utils + +class cObject(object): + ########################################################################### + def __init__(self, path: str): + + """ + Initialize cObject in a given path + + Args: + path (str): Path to cObject.. + """ + + self.path = path + self.entries = {} + + ########################################################################### + def list(self, load_entries: bool = False, + object_id: str = "", + data_name: str = "", + data_id: str = ""): + + """ + List cData objects in a cObject directory + + Args: + None + """ + + if not os.path.isdir(self.path): + raise RuntimeError("Path to cObject ("+self.path+") doesn't exit") + + # List directories + dir_list = utils.dir_list(self.path, data_name, data_id) + + for name in dir_list: + # Attempt to load data + p = os.path.join(self.path, name) + + data = cData(p) + + # Check if valid format + if data.is_valid(): + add_data = False + + if load_entries: + data.load(ignore_errors=True) + + # Check if belongs to a requested object using ID + if data.is_loaded() and (object_id == '' or data.belongs_to_object(object_id)) \ + and (data_id == '' or data.belongs_to_id(data_id)): + add_data = True + + if add_data: + self.entries[name] = data + + return True + + ########################################################################### + def get_entries(self): + + """ + Get cData objects + + Args: + None + """ + + return self.entries diff --git a/incubator/cdatabase/cdatabase/config.py b/incubator/cdatabase/cdatabase/config.py new file mode 100644 index 0000000000..403bab99d3 --- /dev/null +++ b/incubator/cdatabase/cdatabase/config.py @@ -0,0 +1,44 @@ +# +# cDatabase configuration +# + +cfg = { + "ck_dir": ".cm", + "ck_meta_file": "meta.json", + "ck_info_file": "info.json", + "ck_desc_file": "desc.json", + + "cdb_meta_desc_json": "_cdb.json", + "cdb_meta_desc_yaml": "_cdb.yaml", + + "cache_id_dir": ".cm", + "cache_id_prefix": "alias-u-", + + + + + "line_before_json": "====== LINE BEFORE JSON ======", + "line_after_json": "====== LINE AFTER JSON ======", + + "name": "cb", + + "module_prefix": "cb_", + + "env_home": "CB_HOME", + + "default_home_dir": "CB", + + "env_debug": "CB_DEBUG", + + "file_repo_list": "repo_list.json", + + "local_repo_name": "local", + "local_repo_uid": "9a3280b14a4285c9", + "local_repo_desc": "local CB repository", + + "repo_desc_file": "cb.yaml", + + "error_prefix": "Error:", + + "config_file": "config.yaml", + } diff --git a/incubator/cdatabase/cdatabase/crepo.py b/incubator/cdatabase/cdatabase/crepo.py new file mode 100644 index 0000000000..ec068d76b4 --- /dev/null +++ b/incubator/cdatabase/cdatabase/crepo.py @@ -0,0 +1,65 @@ +# +# Low-level repository abstraction in cDatabase +# + +import os +import json +import yaml + +from cdatabase.config import cfg +from cdatabase.cobject import cObject + +import cdatabase.utils as utils + +class cRepo(object): + ########################################################################### + def __init__(self, path: str): + + """ + Initialize cObject in a given path + + Args: + path (str): Path to cDataBase + """ + + self.path = path + + ########################################################################### + def list(self, load_entries: bool = False, + object_name: str = "", + object_id: str = "", + data_name: str = "", + data_id: str = ""): + + """ + List cData objects in a cObject directory + + Args: + """ + + if not os.path.isdir(self.path): + raise RuntimeError("Path to cDatabase ("+self.path+") doesn't exit") + + entries = [] + + # List directories + dir_list = utils.dir_list(self.path, object_name, object_id) + + for obj_name in dir_list: + # Attempt to load data + p = os.path.join(self.path, obj_name) + + if os.path.isdir(p): + obj = cObject(p) + + if obj.list(load_entries=load_entries, + object_id=object_id, + data_name=data_name, + data_id=data_id): + + tmp_entries=obj.get_entries() + + for k in tmp_entries: + entries.append(tmp_entries[k]) + + return entries diff --git a/incubator/cdatabase/cdatabase/utils.py b/incubator/cdatabase/cdatabase/utils.py new file mode 100644 index 0000000000..9fe4faf96b --- /dev/null +++ b/incubator/cdatabase/cdatabase/utils.py @@ -0,0 +1,53 @@ +# +# Utils for cDatabase +# + +from cdatabase.config import cfg + +########################################################################### +def dir_list(path: str, name: str, xid: str = ""): + + import os + + wildcards = '*' in name or '?' in name + + # List directories + dir_list = [] + + if xid != '': + # REDO will global cache + # Check if UID for more than 1 name - inconsistency + + # Check if in cache (.cm) + cache_dir = os.path.join(path, cfg['cache_id_dir']) + if os.path.isdir(cache_dir): + cache_file = os.path.join(cache_dir, cfg['cache_id_prefix']+xid) + if os.path.isfile(cache_file): + with open(cache_file, 'r') as cf: + name = cf.read().strip() + wildcards = False + + # Check if entry exists + path2 = os.path.join(path, name) + if not os.path.isdir(path2): + name = '' + os.remove(cache_file) + + if name != '' and not wildcards: + dir_list = [name] + + if len(dir_list)==0: + dir_list = os.listdir(path) + + if wildcards: + import fnmatch + + new_dir_list = [] + + for d in dir_list: + if fnmatch.fnmatch(d, name): + new_dir_list.append(d) + + dir_list = new_dir_list + + return dir_list diff --git a/incubator/cdatabase/setup.py b/incubator/cdatabase/setup.py new file mode 100644 index 0000000000..ec1f96d7e7 --- /dev/null +++ b/incubator/cdatabase/setup.py @@ -0,0 +1,64 @@ +import os +import sys +import re + +############################################################ +from setuptools import find_packages, setup, convert_path + +try: + from io import open +except ImportError: + pass + +############################################################ +# Version +current_path = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(current_path, "cbase", "__init__.py"), encoding="utf-8") as f: + output = re.search(r'__version__ = ["\']([^"\']+)', f.read()) + version = output.group(1) + +############################################################ +setup( + name='cdatabase', + + author="Grigori Fursin", + author_email="gfursin@gmail.com", + + version=version, + + description="Collective DataBase", + + license="Apache 2.0", + + long_description=open(convert_path('./README.md'), encoding="utf-8").read(), + long_description_content_type="text/markdown", + + url="https://github.com/gfursin/cdatabase", + + python_requires="", # do not force for testing + + packages=find_packages(exclude=["tests*", "docs*"]), + + include_package_data=True, + + install_requires=[ + 'pyyaml', + 'ck' # temporal + ], + +# entry_points={"console_scripts": [ +# "cdb = cdatabase.cli:run" +# ]}, + + zip_safe=False, + + keywords="cdatabase,collective database", + + classifiers=[ + "Development Status :: 1 - Planning", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Intended Audience :: Developers", + ], +) diff --git a/incubator/cdatabase/tests/init.bat b/incubator/cdatabase/tests/init.bat new file mode 100644 index 0000000000..adf988b789 --- /dev/null +++ b/incubator/cdatabase/tests/init.bat @@ -0,0 +1,8 @@ +set curpath=%cd% + +cd .. + +set mypath=%cd% +set PYTHONPATH=%mypath%;%PYTHONPATH% + +cd %curpath% diff --git a/incubator/cdatabase/tests/test01.bat b/incubator/cdatabase/tests/test01.bat new file mode 100644 index 0000000000..cfd7c81e7e --- /dev/null +++ b/incubator/cdatabase/tests/test01.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test01.py diff --git a/incubator/cdatabase/tests/test01.py b/incubator/cdatabase/tests/test01.py new file mode 100644 index 0000000000..18361a25dd --- /dev/null +++ b/incubator/cdatabase/tests/test01.py @@ -0,0 +1,39 @@ +import os +import json + +# Get path to some CK entry +import ck.kernel as ck +r=ck.access({'action':'load', + 'module_uoa':'kernel', + 'data_uoa':'default'}) +if r['return']>0: ck.err(r) +path=r['path'] + +from cdatabase import cdata + +data=cdata.cData(path) + +if data.load(force_ck_format=True): + print ('') + print ('Old format:') + print ('') + print (json.dumps(data.get_meta(), indent=2)) + + print (data.ck) + +if data.load(): + print ('') + print ('New format:') + print ('') + print (json.dumps(data.get_meta(), indent=2)) + + +print ('') +print ('Save data:') +print ('') +print (data.save()) + +#data=cdata.cData("D:\\Work") +#data.load() +#data.meta['aaaaa']='bbbbb' +#data.save(force_yaml=True) diff --git a/incubator/cdatabase/tests/test02.bat b/incubator/cdatabase/tests/test02.bat new file mode 100644 index 0000000000..95fe1447f3 --- /dev/null +++ b/incubator/cdatabase/tests/test02.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test02.py diff --git a/incubator/cdatabase/tests/test02.py b/incubator/cdatabase/tests/test02.py new file mode 100644 index 0000000000..678a2fb47b --- /dev/null +++ b/incubator/cdatabase/tests/test02.py @@ -0,0 +1,29 @@ +import os +import json + +# Get path to some CK entry +import ck.kernel as ck +r=ck.access({'action':'load', + 'module_uoa':'kernel', + 'data_uoa':'default'}) +if r['return']>0: ck.err(r) +path=r['path'] + +path_obj=os.path.dirname(path) + +from cdatabase import cobject + +obj=cobject.cObject(path_obj) + +obj.list(load_entries=True, object_id='b1e99f6461424276') +entries=obj.get_entries() +for d in entries: + entry=entries[d] + + print ('') + print (entry.object_id) + print (entry.object_name) + print (entry.data_id) + print (entry.data_name) + print (entry.title) + diff --git a/incubator/cdatabase/tests/test03.bat b/incubator/cdatabase/tests/test03.bat new file mode 100644 index 0000000000..c7b23ca88e --- /dev/null +++ b/incubator/cdatabase/tests/test03.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test03.py diff --git a/incubator/cdatabase/tests/test03.py b/incubator/cdatabase/tests/test03.py new file mode 100644 index 0000000000..ac07091a36 --- /dev/null +++ b/incubator/cdatabase/tests/test03.py @@ -0,0 +1,24 @@ +import os +import json + +# Get path to some CK entry +import ck.kernel as ck + +path_repo=ck.work['dir_local_repo'] + +print (path_repo) + +from cdatabase import crepo + +repo=crepo.cRepo(path_repo) + +entries=repo.list(load_entries=True, object_name='kernel', object_id='b1e99f6461424276') +for entry in entries: + + print ('') + print (entry.object_id) + print (entry.object_name) + print (entry.data_id) + print (entry.data_name) + print (entry.title) + diff --git a/incubator/cdatabase/tests/test04.bat b/incubator/cdatabase/tests/test04.bat new file mode 100644 index 0000000000..7e05e87ff0 --- /dev/null +++ b/incubator/cdatabase/tests/test04.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test04.py diff --git a/incubator/cdatabase/tests/test04.py b/incubator/cdatabase/tests/test04.py new file mode 100644 index 0000000000..52c38538ef --- /dev/null +++ b/incubator/cdatabase/tests/test04.py @@ -0,0 +1,40 @@ +import os +import json +import sys +import time + +from cdatabase import crepo + +# Get path to some CK entry +import ck.kernel as ck + +r=ck.access({'action':'list', + 'module_uoa':'repo', + 'add_meta':'yes'}) +if r['return']>0: ck.err(r) + +lst=r['lst'] + +entries=[] + +start = time.time() +for l in lst: + m=l['meta'] + p=m.get('path','') + + if p!='' and os.path.isdir(p): + print (p) + repo=crepo.cRepo(p) + + tmp_entries=repo.list(load_entries=True) #, object_name='program') #, object_id='b1e99f6461424276') + entries+=tmp_entries + + end = time.time() + + print (' Entries: {}'.format(len(tmp_entries))) + print (' Total time: {:.2f}'.format(end-start)) + +print ('') +print ('Number of entries: {}'.format(len(entries))) + +print ('Size: {}'.format(sys.getsizeof(entries))) diff --git a/incubator/cdatabase/tests/test04a.bat b/incubator/cdatabase/tests/test04a.bat new file mode 100644 index 0000000000..22014b19ec --- /dev/null +++ b/incubator/cdatabase/tests/test04a.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test04a.py diff --git a/incubator/cdatabase/tests/test04a.py b/incubator/cdatabase/tests/test04a.py new file mode 100644 index 0000000000..d245afc8aa --- /dev/null +++ b/incubator/cdatabase/tests/test04a.py @@ -0,0 +1,74 @@ +import os +import json +import sys +import time + +from cdatabase import crepo +import ck.kernel as ck + + +import gc +import sys + +def get_obj_size(obj): + marked = {id(obj)} + obj_q = [obj] + sz = 0 + + while obj_q: + sz += sum(map(sys.getsizeof, obj_q)) + + # Lookup all the object referred to by the object in obj_q. + # See: https://docs.python.org/3.7/library/gc.html#gc.get_referents + all_refr = ((id(o), o) for o in gc.get_referents(*obj_q)) + + # Filter object that are already marked. + # Using dict notation will prevent repeated objects. + new_refr = {o_id: o for o_id, o in all_refr if o_id not in marked and not isinstance(o, type)} + + # The new obj_q will be the ones that were not marked, + # and we will update marked with their ids so we will + # not traverse them again. + obj_q = new_refr.values() + marked.update(new_refr.keys()) + + return sz + + +r=ck.access({'action':'list', + 'module_uoa':'repo', + 'add_meta':'yes'}) +if r['return']>0: ck.err(r) + +lst=r['lst'] + +entries=[] + +start = time.time() +for l in lst: + m=l['meta'] + p=m.get('path','') + + if p!='' and os.path.isdir(p): + print (p) + repo=crepo.cRepo(p) + + tmp_entries=repo.list(load_entries=True, object_name='program', object_id='b0ac08fe1d3c2615', + data_name='*corner*') + entries+=tmp_entries + + end = time.time() + + print (' Entries: {}'.format(len(tmp_entries))) + print (' Total time: {:.2f}'.format(end-start)) + +print ('') +print ('Number of entries: {}'.format(len(entries))) + +print ('Size: {}'.format(sys.getsizeof(entries))) +print ('Size2: {}'.format(get_obj_size(entries))) + +print ('') +for entry in entries: + print (entry.get_path()) + diff --git a/incubator/cdatabase/tests/test04b.bat b/incubator/cdatabase/tests/test04b.bat new file mode 100644 index 0000000000..e3e94801f2 --- /dev/null +++ b/incubator/cdatabase/tests/test04b.bat @@ -0,0 +1,7 @@ +call init.bat + +rem rmdir /Q /S CB + +set CDB_HOME=%curpath%\cdatabase + +python test04b.py diff --git a/incubator/cdatabase/tests/test04b.py b/incubator/cdatabase/tests/test04b.py new file mode 100644 index 0000000000..5a7065f555 --- /dev/null +++ b/incubator/cdatabase/tests/test04b.py @@ -0,0 +1,73 @@ +import os +import json +import sys +import time + +from cdatabase import crepo +import ck.kernel as ck + + +import gc +import sys + +def get_obj_size(obj): + marked = {id(obj)} + obj_q = [obj] + sz = 0 + + while obj_q: + sz += sum(map(sys.getsizeof, obj_q)) + + # Lookup all the object referred to by the object in obj_q. + # See: https://docs.python.org/3.7/library/gc.html#gc.get_referents + all_refr = ((id(o), o) for o in gc.get_referents(*obj_q)) + + # Filter object that are already marked. + # Using dict notation will prevent repeated objects. + new_refr = {o_id: o for o_id, o in all_refr if o_id not in marked and not isinstance(o, type)} + + # The new obj_q will be the ones that were not marked, + # and we will update marked with their ids so we will + # not traverse them again. + obj_q = new_refr.values() + marked.update(new_refr.keys()) + + return sz + + +r=ck.access({'action':'list', + 'module_uoa':'repo', + 'add_meta':'yes'}) +if r['return']>0: ck.err(r) + +lst=r['lst'] + +entries=[] + +start = time.time() +for l in lst: + m=l['meta'] + p=m.get('path','') + + if p!='' and os.path.isdir(p): + print (p) + repo=crepo.cRepo(p) + + tmp_entries=repo.list(load_entries=True, object_name='package', + data_id='04622c746f287473') + entries+=tmp_entries + + end = time.time() + + print (' Entries: {}'.format(len(tmp_entries))) + print (' Total time: {:.2f}'.format(end-start)) + +print ('') +for entry in entries: + print (entry.get_path()) + +print ('') +print ('Number of entries: {}'.format(len(entries))) + +print ('Size: {}'.format(sys.getsizeof(entries))) +print ('Size2: {}'.format(get_obj_size(entries)))