From df4013c1fd08639c343058e227aea0d17f5066f8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 10:54:18 +0200 Subject: [PATCH 01/27] remove sys deps from script "python hello world" --- script/print-hello-world-py/_cm.json | 3 --- script/print-hello-world-py/run.bat | 2 +- script/print-hello-world-py/run.sh | 4 ++-- script/print-hello-world/_cm.json | 32 ---------------------------- 4 files changed, 3 insertions(+), 38 deletions(-) delete mode 100644 script/print-hello-world/_cm.json diff --git a/script/print-hello-world-py/_cm.json b/script/print-hello-world-py/_cm.json index 5e6a0629c..7bb36e4b6 100644 --- a/script/print-hello-world-py/_cm.json +++ b/script/print-hello-world-py/_cm.json @@ -7,9 +7,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", diff --git a/script/print-hello-world-py/run.bat b/script/print-hello-world-py/run.bat index d1881d3a3..b4f5e3a7a 100644 --- a/script/print-hello-world-py/run.bat +++ b/script/print-hello-world-py/run.bat @@ -1,6 +1,6 @@ IF NOT DEFINED CM_TMP_CURRENT_SCRIPT_PATH SET CM_TMP_CURRENT_SCRIPT_PATH=%CD% -%CM_PYTHON_BIN_WITH_PATH% --version +rem %CM_PYTHON_BIN_WITH_PATH% --version %CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\code.py IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/script/print-hello-world-py/run.sh b/script/print-hello-world-py/run.sh index b7c69c790..26b72ba62 100644 --- a/script/print-hello-world-py/run.sh +++ b/script/print-hello-world-py/run.sh @@ -2,8 +2,8 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} -which ${CM_PYTHON_BIN_WITH_PATH} -${CM_PYTHON_BIN_WITH_PATH} --version +#which ${CM_PYTHON_BIN_WITH_PATH} +#${CM_PYTHON_BIN_WITH_PATH} --version ${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/code.py test $? -eq 0 || exit $? diff --git a/script/print-hello-world/_cm.json b/script/print-hello-world/_cm.json deleted file mode 100644 index c8ccffe53..000000000 --- a/script/print-hello-world/_cm.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "alias": "print-hello-world", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Tests", - "default_env": { - "CM_ENV_TEST1": "TEST1" - }, - "env": { - "CM_ENV_TEST2": "TEST2" - }, - "input_mapping": { - "test1": "CM_ENV_TEST1" - }, - "new_env_keys": [ - "CM_ENV_TEST*" - ], - "new_state_keys": [ - "hello_world*" - ], - "tags": [ - "print", - "hello-world", - "hello world", - "hello", - "world", - "native-script", - "native", - "script" - ], - "uid": "b9f0acba4aca4baa" -} From 9a7057e467b6ae76defb9c5fbb8123a6d27e250a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 11:52:40 +0200 Subject: [PATCH 02/27] * added "dummy" script to test Docker containers * added more standard Nvidia Docker configuration for PyTorch --- CHANGES.md | 4 ++++ ...-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml | 2 +- ...4-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml | 2 +- ...4-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml | 2 +- ...4-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml | 11 ++++++++++ ...4-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml | 11 ++++++++++ script/dummy/README-extra.md | 1 + script/dummy/_cm.yaml | 10 +++++++++ script/dummy/customize.py | 22 +++++++++++++++++++ script/dummy/run.bat | 1 + script/dummy/run.sh | 2 ++ 11 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml create mode 100644 cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml create mode 100644 script/dummy/README-extra.md create mode 100644 script/dummy/_cm.yaml create mode 100644 script/dummy/customize.py create mode 100644 script/dummy/run.bat create mode 100644 script/dummy/run.sh diff --git a/CHANGES.md b/CHANGES.md index c49755977..34d887627 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +### 20240927 + * added "dummy" script to test Docker containers + * added more standard Nvidia Docker configuration for PyTorch + ### 20240916 * fixed "cm add script" diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml index 1e71c67ce..c72268b15 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml @@ -1,6 +1,6 @@ uid: 854e65fb31584d63 -name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0" +name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0 (pytorch:22.10)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:22.10-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml index 4e146ca66..de9a1bc1d 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml @@ -1,6 +1,6 @@ uid: e0e7167139a74e36 -name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0 (pytorch:23.05)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:23.05-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml index a671d699a..f23bba2eb 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml @@ -1,6 +1,6 @@ uid: 49fc51f2999b4545 -name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0 (pytorch:24.03)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:24.03-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml new file mode 100644 index 000000000..b4e45d348 --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml @@ -0,0 +1,11 @@ +uid: 81879736ae5842f4 + +name: "Nvidia Ubuntu 22.04 CUDA 12.5 cuDNN 9.1.0 PyTorch 2.4.0 (pytorch:24.06)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-06.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.06-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml new file mode 100644 index 000000000..a9e2229ea --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml @@ -0,0 +1,11 @@ +uid: 203a68df99d44137 + +name: "Nvidia Ubuntu 22.04 CUDA 12.6 cuDNN 9.3.0 PyTorch 2.5.0 (pytorch:24.08)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.08-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/script/dummy/README-extra.md b/script/dummy/README-extra.md new file mode 100644 index 000000000..582991f6d --- /dev/null +++ b/script/dummy/README-extra.md @@ -0,0 +1 @@ +# CM script diff --git a/script/dummy/_cm.yaml b/script/dummy/_cm.yaml new file mode 100644 index 000000000..6e73e8ca8 --- /dev/null +++ b/script/dummy/_cm.yaml @@ -0,0 +1,10 @@ +alias: dummy +uid: 3ef5d69f929349bc + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: false + +tags: +- dummy diff --git a/script/dummy/customize.py b/script/dummy/customize.py new file mode 100644 index 000000000..d12f9b3e1 --- /dev/null +++ b/script/dummy/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} diff --git a/script/dummy/run.bat b/script/dummy/run.bat new file mode 100644 index 000000000..648302ca7 --- /dev/null +++ b/script/dummy/run.bat @@ -0,0 +1 @@ +rem native script diff --git a/script/dummy/run.sh b/script/dummy/run.sh new file mode 100644 index 000000000..05a7907cf --- /dev/null +++ b/script/dummy/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash + From 6b02a9db8b0b0213014cad49f8a2cd9e26e162ac Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 12:31:03 +0200 Subject: [PATCH 03/27] * added better support to select Docker configurations via UID * renamed dummy script to test-dummy --- CHANGES.md | 3 +- automation/script/module_misc.py | 13 +++-- automation/utils/module_cfg.py | 52 +++++++++++-------- .../_cm.yaml | 39 ++++++++++++++ script/{dummy => test-dummy}/README-extra.md | 0 script/{dummy => test-dummy}/_cm.yaml | 3 +- script/{dummy => test-dummy}/customize.py | 0 script/{dummy => test-dummy}/run.bat | 0 script/{dummy => test-dummy}/run.sh | 0 9 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml rename script/{dummy => test-dummy}/README-extra.md (100%) rename script/{dummy => test-dummy}/_cm.yaml (81%) rename script/{dummy => test-dummy}/customize.py (100%) rename script/{dummy => test-dummy}/run.bat (100%) rename script/{dummy => test-dummy}/run.sh (100%) diff --git a/CHANGES.md b/CHANGES.md index 34d887627..d640f53ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ ### 20240927 - * added "dummy" script to test Docker containers + * added "test dummy" script to test Docker containers * added more standard Nvidia Docker configuration for PyTorch + * added better support to select Docker configurations via UID ### 20240916 * fixed "cm add script" diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index b193ff5df..e143d5204 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1641,6 +1641,7 @@ def docker(i): (docker_save_script) (str): if !='' name of script to save docker command (docker_interactive) (bool): if True, run in interactive mode (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID Returns: (CM return dict): @@ -1693,13 +1694,19 @@ def docker(i): # Check available configurations docker_cfg = i.get('docker_cfg', '') - if docker_cfg != '': + docker_cfg_uid = i.get('docker_cfg_uid', '') + + if docker_cfg != '' or docker_cfg_uid != '': # Check if docker_cfg is turned on but not selected if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true','yes']: docker_cfg= '' - r = self_module.cmind.access({'action':'select_cfg', 'automation':'utils,dc2743f8450541e3', - 'tags':'basic,docker,configurations', 'title':'docker', 'alias':docker_cfg}) + r = self_module.cmind.access({'action':'select_cfg', + 'automation':'utils,dc2743f8450541e3', + 'tags':'basic,docker,configurations', + 'title':'docker', + 'alias':docker_cfg, + 'uid':docker_cfg_uid}) if r['return'] > 0: if r['return'] == 16: return {'return':1, 'error':'Docker configuration {} was not found'.format(docker_cfg)} diff --git a/automation/utils/module_cfg.py b/automation/utils/module_cfg.py index 145c388f2..994aa4329 100644 --- a/automation/utils/module_cfg.py +++ b/automation/utils/module_cfg.py @@ -230,6 +230,7 @@ def select_cfg(i): self_module = i['self_module'] tags = i['tags'] alias = i.get('alias', '') + uid = i.get('uid', '') title = i.get('title', '') # Check if alias is not provided @@ -240,6 +241,7 @@ def select_cfg(i): selector = [] + # Do coarse-grain search for CM artifacts for l in lst: p = l.path @@ -257,42 +259,50 @@ def select_cfg(i): if not f.startswith('_cm') and (f.endswith('.json') or f.endswith('.yaml')): selector.append({'path':os.path.join(p, f), 'alias':f[:-5]}) - if len(selector) == 0: + + # Load meta for name and UID + selector_with_meta = [] + for s in range(0, len(selector)): + ss = selector[s] + + path = ss['path'] + + full_path_without_ext = path[:-5] + + r = cmind.utils.load_yaml_and_json(full_path_without_ext) + if r['return']>0: + print ('Warning: problem loading configuration file {}'.format(path)) + + meta = r['meta'] + + if uid == '' or meta.get('uid', '') == uid: + ss['meta'] = meta + selector_with_meta.append(ss) + + # Quit if no configurations found + if len(selector_with_meta) == 0: return {'return':16, 'error':'configuration was not found'} select = 0 - if len(selector) > 1: + if len(selector_with_meta) > 1: xtitle = ' ' + title if title!='' else '' print ('') print ('Available{} configurations:'.format(xtitle)) print ('') - for s in range(0, len(selector)): - ss = selector[s] - - path = ss['path'] - - full_path_without_ext = path[:-5] - - r = cmind.utils.load_yaml_and_json(full_path_without_ext) - if r['return']>0: - print ('Warning: problem loading configuration file {}'.format(path)) - - meta = r['meta'] - ss['meta'] = meta - - selector = sorted(selector, key = lambda x: x['meta'].get('name','')) + selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name','')) s = 0 for ss in selector: alias = ss['alias'] - name = ss['meta'].get('name','') + uid = ss['meta'].get('uid', '') + name = ss['meta'].get('name', '') x = name if x!='': x+=' ' - x += '('+alias+')' + x += '(' + uid + ')' - print ('{}) {}'.format(s, x)) + print (f'{s}) {x}'.format(s, x)) s+=1 @@ -306,6 +316,6 @@ def select_cfg(i): if select<0 or select>=len(selector): return {'return':1, 'error':'selection is out of range'} - ss = selector[select] + ss = selector_with_meta[select] return {'return':0, 'selection':ss} diff --git a/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml new file mode 100644 index 000000000..716adc20b --- /dev/null +++ b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml @@ -0,0 +1,39 @@ +alias: benchmark-run-mlperf-inference-v4.1 +uid: b7e89771987d4168 + +automation_alias: cfg +automation_uid: 88dce9c160324c5d + +tags: +- benchmark +- run +- mlperf +- inference +- v4.1 + +name: "MLPerf inference - v4.1" + +supported_compute: +- ee8c568e0ac44f2b +- fe379ecd1e054a00 +- d8f06040f7294319 + +bench_uid: 39877bb63fb54725 + +view_dimensions: +- - input.device + - "MLPerf device" +- - input.implementation + - "MLPerf implementation" +- - input.backend + - "MLPerf backend" +- - input.model + - "MLPerf model" +- - input.scenario + - "MLPerf scenario" +- - input.host_os + - "Host OS" +- - output.state.cm-mlperf-inference-results-last.performance + - "Got performance" +- - output.state.cm-mlperf-inference-results-last.accuracy + - "Got accuracy" diff --git a/script/dummy/README-extra.md b/script/test-dummy/README-extra.md similarity index 100% rename from script/dummy/README-extra.md rename to script/test-dummy/README-extra.md diff --git a/script/dummy/_cm.yaml b/script/test-dummy/_cm.yaml similarity index 81% rename from script/dummy/_cm.yaml rename to script/test-dummy/_cm.yaml index 6e73e8ca8..09ba01f92 100644 --- a/script/dummy/_cm.yaml +++ b/script/test-dummy/_cm.yaml @@ -1,4 +1,4 @@ -alias: dummy +alias: test-dummy uid: 3ef5d69f929349bc automation_alias: script @@ -7,4 +7,5 @@ automation_uid: 5b4e0237da074764 cache: false tags: +- test - dummy diff --git a/script/dummy/customize.py b/script/test-dummy/customize.py similarity index 100% rename from script/dummy/customize.py rename to script/test-dummy/customize.py diff --git a/script/dummy/run.bat b/script/test-dummy/run.bat similarity index 100% rename from script/dummy/run.bat rename to script/test-dummy/run.bat diff --git a/script/dummy/run.sh b/script/test-dummy/run.sh similarity index 100% rename from script/dummy/run.sh rename to script/test-dummy/run.sh From 5c7492587da1ec2e3ac6e7c9bf87f6f5b46fb8e9 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 13:50:11 +0000 Subject: [PATCH 04/27] fixing docker cfg selection --- automation/script/module_misc.py | 9 ++++----- automation/utils/module_cfg.py | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index e143d5204..4718efc74 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1677,7 +1677,7 @@ def docker(i): # CAREFUL -> artifacts and parsed_artifacts are not supported in input (and should not be?) if 'artifacts' in i: del(i['artifacts']) if 'parsed_artifacts' in i: del(i['parsed_artifacts']) - + # Prepare "clean" input to replicate command r = self_module.cmind.access({'action':'prune_input', 'automation':'utils', 'input':i, 'extra_keys_starts_with':['docker_']}) i_run_cmd_arc = r['new_input'] @@ -1700,7 +1700,7 @@ def docker(i): # Check if docker_cfg is turned on but not selected if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true','yes']: docker_cfg= '' - + r = self_module.cmind.access({'action':'select_cfg', 'automation':'utils,dc2743f8450541e3', 'tags':'basic,docker,configurations', @@ -1715,10 +1715,9 @@ def docker(i): selection = r['selection'] docker_input_update = selection['meta']['input'] - + i.update(docker_input_update) - ######################################################################################## # Run dockerfile if not noregenerate_docker_file: @@ -1729,7 +1728,7 @@ def docker(i): cur_dir = os.getcwd() console = i.get('out') == 'con' - + # Search for script(s) r = aux_search({'self_module': self_module, 'input': i}) if r['return']>0: return r diff --git a/automation/utils/module_cfg.py b/automation/utils/module_cfg.py index 994aa4329..04ab0a9ad 100644 --- a/automation/utils/module_cfg.py +++ b/automation/utils/module_cfg.py @@ -236,7 +236,7 @@ def select_cfg(i): # Check if alias is not provided r = self_module.cmind.access({'action':'find', 'automation':'cfg', 'tags':'basic,docker,configurations'}) if r['return'] > 0: return r - + lst = r['list'] selector = [] @@ -288,12 +288,12 @@ def select_cfg(i): xtitle = ' ' + title if title!='' else '' print ('') print ('Available{} configurations:'.format(xtitle)) - + print ('') selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name','')) s = 0 - for ss in selector: + for ss in selector_with_meta: alias = ss['alias'] uid = ss['meta'].get('uid', '') name = ss['meta'].get('name', '') @@ -301,11 +301,11 @@ def select_cfg(i): x = name if x!='': x+=' ' x += '(' + uid + ')' - + print (f'{s}) {x}'.format(s, x)) s+=1 - + print ('') select = input ('Enter configuration number of press Enter for 0: ') From 8dbc03812f5cf7cc39f286234a789c4de645a1ed Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 15:52:31 +0200 Subject: [PATCH 05/27] clean up --- automation/script/module_misc.py | 49 +++++++++++++++++++++------- script/build-dockerfile/customize.py | 2 +- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index e143d5204..7c9552ea2 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1335,15 +1335,9 @@ def dockerfile(i): Args: (CM input dict): - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (repos) (str): list of repositories to search for automations - - (output_dir) (str): output directory (./ by default) + (out) (str): if 'con', output to console + (repos) (str): list of repositories to search for automations + (output_dir) (str): output directory (./ by default) Returns: (CM return dict): @@ -1636,12 +1630,45 @@ def docker(i): (docker_noregenerate) (bool): do not generate Dockerfiles (docker_norecreate) (bool): do not recreate Docker image + (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID + (docker_path) (str): where to create or find Dockerfile (docker_gh_token) (str): GitHub token for private repositories (docker_save_script) (str): if !='' name of script to save docker command (docker_interactive) (bool): if True, run in interactive mode - (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one - (docker_cfg_uid) (str): if True, select docker configuration with this UID + (docker_it) (bool): the same as `docker_interactive` + (docker_detached) (bool): detach Docker + (docker_dt) (bool) the same as `docker_detached` + + (docker_base_image) (str): force base image + (docker_os) (str): force docker OS (default: ubuntu) + (docker_os_version) (str): force docker OS version (default: 22.04) + (docker_image_tag_extra) (str): add extra tag (default:-latest) + + (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) + (docker_cm_repos) + (docker_cm_repo_flags) + + (dockerfile_env) + + (docker_skip_cm_sys_upgrade) + + (docker_extra_sys_deps) + + (fake_run_deps) + (docker_run_final_cmds) + + (all_gpus) + (num_gpus) + + (docker_device) + + (docker_port_maps) + + (docker_shm_size) + + (docker_extra_run_args) Returns: (CM return dict): diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 41300cdde..d99e24392 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -55,7 +55,7 @@ def preprocess(i): if env.get("CM_MLOPS_REPO", "") != "": cm_mlops_repo = env["CM_MLOPS_REPO"] else: - cm_mlops_repo = "mlcommons@ck" + cm_mlops_repo = "mlcommons@cm4mlops" if env.get("CM_MLOPS_REPO_BRANCH", '') != '': cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}" From 2087704d00aa012b3478c62e3313106004400e40 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 14:00:57 +0000 Subject: [PATCH 06/27] clean up --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 777340d39..92f7f07fe 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,11 @@ cm run script \ [Apache 2.0](LICENSE.md) +## CM concept + +* https://doi.org/10.5281/zenodo.8105339 +* https://arxiv.org/abs/2406.16791 + ## Authors [Grigori Fursin](https://cKnowledge.org/gfursin) and [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) From 773ef26f9fdf18c2446dab0d5204ccd64697029c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 14:02:11 +0000 Subject: [PATCH 07/27] removed ^M from setup.py --- setup.py | 294 +++++++++++++++++++++++++++---------------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/setup.py b/setup.py index e7c96a97c..2ff411286 100644 --- a/setup.py +++ b/setup.py @@ -1,147 +1,147 @@ -# setup.py -from setuptools import setup -from setuptools._distutils.dist import Distribution -from setuptools.command.install import install -import subprocess -import sys -import importlib.util -import platform -import os - -# Try to use importlib.metadata for Python 3.8+ -try: - if sys.version_info >= (3, 8): - from importlib.metadata import version, PackageNotFoundError - else: - # Fallback to pkg_resources for Python < 3.8 - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound -except ImportError: - # If importlib.metadata is unavailable, fall back to pkg_resources - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound - - - -class CustomInstallCommand(install): - def run(self): - self.get_sys_platform() - self.install_system_packages() - - # Call the standard run method - install.run(self) - - # Call the custom function - return self.custom_function() - - def is_package_installed(self, package_name): - try: - if sys.version_info >= (3, 8): - version(package_name) # Tries to get the version of the package - else: - pkg_resources.get_distribution(package_name) # Fallback for < 3.8 - return True - except PackageNotFoundError: - return False - - def install_system_packages(self): - # List of packages to install via system package manager - packages = [] - - git_status = self.command_exists('git') - if not git_status: - packages.append("git") - wget_status = self.command_exists('wget') - if not wget_status: - packages.append("wget") - curl_status = self.command_exists('curl') - if not curl_status: - packages.append("curl") - - name='venv' - - if name in sys.modules: - pass #nothing needed - elif self.is_package_installed(name): - pass - else: - packages.append("python3-venv") - - if packages: - if self.system == 'Linux' or self.system == 'Darwin': - manager, details = self.get_package_manager_details() - if manager: - if manager == "apt-get": - subprocess.check_call(['sudo', 'apt-get', 'update']) - subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) - elif self.system == 'Windows': - print(f"Please install the following packages manually: {packages}") - - - - def detect_package_manager(self): - package_managers = { - 'apt-get': '/usr/bin/apt-get', - 'yum': '/usr/bin/yum', - 'dnf': '/usr/bin/dnf', - 'pacman': '/usr/bin/pacman', - 'zypper': '/usr/bin/zypper', - 'brew': '/usr/local/bin/brew' - } - - for name, path in package_managers.items(): - if os.path.exists(path): - return name - - return None - - def get_package_manager_details(self): - manager = self.detect_package_manager() - if manager: - try: - version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') - return manager, version_output.split('\n')[0] - except subprocess.CalledProcessError: - return manager, 'Version information not available' - else: - return None, 'No supported package manager found' - - # Checks if command exists(for installing required packages). - # If the command exists, which returns 0, making the function return True. - # If the command does not exist, which returns a non-zero value, making the function return False. - # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. - def command_exists(self, command): - if self.system == "Linux" or self.system == 'Darwin': - return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 - elif self.system == "Windows": - return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 - - def custom_function(self): - import cmind - #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) - r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) - print(r) - if r['return'] > 0: - return r['return'] - - def get_sys_platform(self): - self.system = platform.system() - -setup( - name='cm4mlops', - version='0.1', - long_description='CM automations and scripts for MLOps', - long_description_content_type='text/x-rst', - packages=[], - install_requires=[ - "setuptools>=60", - "wheel", - "cmind", - "giturlparse", - "requests", - "pyyaml" - ], - cmdclass={ - 'install': CustomInstallCommand, - }, -) +# setup.py +from setuptools import setup +from setuptools._distutils.dist import Distribution +from setuptools.command.install import install +import subprocess +import sys +import importlib.util +import platform +import os + +# Try to use importlib.metadata for Python 3.8+ +try: + if sys.version_info >= (3, 8): + from importlib.metadata import version, PackageNotFoundError + else: + # Fallback to pkg_resources for Python < 3.8 + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound +except ImportError: + # If importlib.metadata is unavailable, fall back to pkg_resources + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound + + + +class CustomInstallCommand(install): + def run(self): + self.get_sys_platform() + self.install_system_packages() + + # Call the standard run method + install.run(self) + + # Call the custom function + return self.custom_function() + + def is_package_installed(self, package_name): + try: + if sys.version_info >= (3, 8): + version(package_name) # Tries to get the version of the package + else: + pkg_resources.get_distribution(package_name) # Fallback for < 3.8 + return True + except PackageNotFoundError: + return False + + def install_system_packages(self): + # List of packages to install via system package manager + packages = [] + + git_status = self.command_exists('git') + if not git_status: + packages.append("git") + wget_status = self.command_exists('wget') + if not wget_status: + packages.append("wget") + curl_status = self.command_exists('curl') + if not curl_status: + packages.append("curl") + + name='venv' + + if name in sys.modules: + pass #nothing needed + elif self.is_package_installed(name): + pass + else: + packages.append("python3-venv") + + if packages: + if self.system == 'Linux' or self.system == 'Darwin': + manager, details = self.get_package_manager_details() + if manager: + if manager == "apt-get": + subprocess.check_call(['sudo', 'apt-get', 'update']) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + elif self.system == 'Windows': + print(f"Please install the following packages manually: {packages}") + + + + def detect_package_manager(self): + package_managers = { + 'apt-get': '/usr/bin/apt-get', + 'yum': '/usr/bin/yum', + 'dnf': '/usr/bin/dnf', + 'pacman': '/usr/bin/pacman', + 'zypper': '/usr/bin/zypper', + 'brew': '/usr/local/bin/brew' + } + + for name, path in package_managers.items(): + if os.path.exists(path): + return name + + return None + + def get_package_manager_details(self): + manager = self.detect_package_manager() + if manager: + try: + version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') + return manager, version_output.split('\n')[0] + except subprocess.CalledProcessError: + return manager, 'Version information not available' + else: + return None, 'No supported package manager found' + + # Checks if command exists(for installing required packages). + # If the command exists, which returns 0, making the function return True. + # If the command does not exist, which returns a non-zero value, making the function return False. + # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. + def command_exists(self, command): + if self.system == "Linux" or self.system == 'Darwin': + return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + elif self.system == "Windows": + return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 + + def custom_function(self): + import cmind + #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) + r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) + print(r) + if r['return'] > 0: + return r['return'] + + def get_sys_platform(self): + self.system = platform.system() + +setup( + name='cm4mlops', + version='0.1', + long_description='CM automations and scripts for MLOps', + long_description_content_type='text/x-rst', + packages=[], + install_requires=[ + "setuptools>=60", + "wheel", + "cmind", + "giturlparse", + "requests", + "pyyaml" + ], + cmdclass={ + 'install': CustomInstallCommand, + }, +) From de55cc158ad14fd0276526f80aa3736c6cc7dd2a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 16:28:30 +0200 Subject: [PATCH 08/27] improving docker docs --- automation/script/module.py | 49 +++++++++++++++++++++++++++++--- automation/script/module_misc.py | 43 ---------------------------- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index e40cf8b35..9747639cb 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4028,14 +4028,55 @@ def docker(self, i): (out) (str): if 'con', output to console - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - (repos) (str): list of repositories to search for automations (output_dir) (str): output directory (./ by default) + (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) + (docker_noregenerate) (bool): do not generate Dockerfiles + (docker_norecreate) (bool): do not recreate Docker image + + (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID + + (docker_path) (str): where to create or find Dockerfile + (docker_gh_token) (str): GitHub token for private repositories + (docker_save_script) (str): if !='' name of script to save docker command + (docker_interactive) (bool): if True, run in interactive mode + (docker_it) (bool): the same as `docker_interactive` + (docker_detached) (bool): detach Docker + (docker_dt) (bool) the same as `docker_detached` + + (docker_base_image) (str): force base image + (docker_os) (str): force docker OS (default: ubuntu) + (docker_os_version) (str): force docker OS version (default: 22.04) + (docker_image_tag_extra) (str): add extra tag (default:-latest) + + (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) + (docker_cm_repos) + (docker_cm_repo_flags) + + (dockerfile_env) + + (docker_skip_cm_sys_upgrade) (bool): if True, do not install CM sys deps + + (docker_extra_sys_deps) + + (fake_run_deps) + (docker_run_final_cmds) + + (all_gpus) + (num_gpus) + + (docker_device) + + (docker_port_maps) + + (docker_shm_size) + + (docker_extra_run_args) + + Returns: (CM return dict): diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index c2a1cf740..98a4c667d 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1626,49 +1626,6 @@ def docker(i): (out) (str): if 'con', output to console - (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) - (docker_noregenerate) (bool): do not generate Dockerfiles - (docker_norecreate) (bool): do not recreate Docker image - - (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one - (docker_cfg_uid) (str): if True, select docker configuration with this UID - - (docker_path) (str): where to create or find Dockerfile - (docker_gh_token) (str): GitHub token for private repositories - (docker_save_script) (str): if !='' name of script to save docker command - (docker_interactive) (bool): if True, run in interactive mode - (docker_it) (bool): the same as `docker_interactive` - (docker_detached) (bool): detach Docker - (docker_dt) (bool) the same as `docker_detached` - - (docker_base_image) (str): force base image - (docker_os) (str): force docker OS (default: ubuntu) - (docker_os_version) (str): force docker OS version (default: 22.04) - (docker_image_tag_extra) (str): add extra tag (default:-latest) - - (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) - (docker_cm_repos) - (docker_cm_repo_flags) - - (dockerfile_env) - - (docker_skip_cm_sys_upgrade) - - (docker_extra_sys_deps) - - (fake_run_deps) - (docker_run_final_cmds) - - (all_gpus) - (num_gpus) - - (docker_device) - - (docker_port_maps) - - (docker_shm_size) - - (docker_extra_run_args) Returns: (CM return dict): From c5aeb3dc4c3d213f9838fed63b32b4f69f4545ea Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 16:37:59 +0200 Subject: [PATCH 09/27] minor fix in get-cuda-devices --- script/get-cuda-devices/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/get-cuda-devices/run.sh b/script/get-cuda-devices/run.sh index 91d0a9e1b..3d208dd6b 100644 --- a/script/get-cuda-devices/run.sh +++ b/script/get-cuda-devices/run.sh @@ -29,8 +29,7 @@ echo "" echo "Running program ..." echo "" -cd - -#${CM_TMP_CURRENT_PATH} +cd ${CM_TMP_CURRENT_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/a.out > tmp-run.out test $? -eq 0 || exit 1 From 9b79c55b671686cd864a4848cb696da1cf8da394 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:14:35 +0200 Subject: [PATCH 10/27] added --docker.key = value to cm docker script --- automation/script/module.py | 3 +++ automation/script/module_misc.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 9747639cb..9494e7a39 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4032,6 +4032,9 @@ def docker(self, i): (output_dir) (str): output directory (./ by default) + (docker) (dict): convert keys into docker_{key} strings for CM >= 2.3.8.1 + + (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) (docker_noregenerate) (bool): do not generate Dockerfiles (docker_norecreate) (bool): do not recreate Docker image diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 98a4c667d..18ea9b826 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1638,6 +1638,18 @@ def docker(i): import copy import re + from cmind import __version__ as current_cm_version + + self_module = i['self_module'] + + if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0 and type(i.get('docker', None)) == dict: + # Grigori started cleaning and refactoring this code on 20240929 + # + # 1. use --docker dictionary instead of --docker_{keys} + + docker_params = utils.convert_dictionary(i['docker'], 'docker') + i.update(docker_params) + quiet = i.get('quiet', False) detached = i.get('docker_detached', '') @@ -1655,7 +1667,6 @@ def docker(i): # Check simplified CMD: cm docker script "python app image-classification onnx" # If artifact has spaces, treat them as tags! - self_module = i['self_module'] self_module.cmind.access({'action':'detect_tags_in_artifact', 'automation':'utils', 'input':i}) # CAREFUL -> artifacts and parsed_artifacts are not supported in input (and should not be?) From 07ef382427132b55c1656754cc4a259d7603d2f8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:33:26 +0200 Subject: [PATCH 11/27] docker fixes --- automation/script/module_misc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 18ea9b826..b68d90e27 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1642,13 +1642,15 @@ def docker(i): self_module = i['self_module'] - if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0 and type(i.get('docker', None)) == dict: + if type(i.get('docker', None)) == dict: # Grigori started cleaning and refactoring this code on 20240929 # # 1. use --docker dictionary instead of --docker_{keys} - - docker_params = utils.convert_dictionary(i['docker'], 'docker') - i.update(docker_params) + + if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0: + docker_params = utils.convert_dictionary(i['docker'], 'docker') + i.update(docker_params) + del(i['docker']) quiet = i.get('quiet', False) From 2924ce68597283aa5d7172714515869553cca811 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:46:18 +0200 Subject: [PATCH 12/27] added ubuntu 24.04 config --- cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml | 9 +++++++++ ...buntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml | 2 ++ ...ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml | 2 ++ ...ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml | 2 ++ 4 files changed, 15 insertions(+) create mode 100644 cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml diff --git a/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml new file mode 100644 index 000000000..d949d5519 --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml @@ -0,0 +1,9 @@ +uid: 12e86eb386314866 + +name: "Basic Ubuntu 24.04" + +input: + docker_base_image: 'ubuntu:24.04' + docker_os: ubuntu + docker_os_version: '24.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml index c72268b15..16107d8d5 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0 (pytorch:22.10)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-22-10.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:22.10-py3' docker_os: ubuntu diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml index de9a1bc1d..66b9efd0d 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0 (pytorch:23.05)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-23-05.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:23.05-py3' docker_os: ubuntu diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml index f23bba2eb..38bcff694 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0 (pytorch:24.03)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-03.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:24.03-py3' docker_os: ubuntu From e3b43b852c739a2daa3d988e03c45f13c75dffde Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 10:39:33 +0200 Subject: [PATCH 13/27] removed sys deps from image-classification examples --- script/app-image-classification-onnx-py/_cm.yaml | 2 +- script/app-image-classification-torch-py/_cm.json | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/script/app-image-classification-onnx-py/_cm.yaml b/script/app-image-classification-onnx-py/_cm.yaml index 825be9128..2e2241a07 100644 --- a/script/app-image-classification-onnx-py/_cm.yaml +++ b/script/app-image-classification-onnx-py/_cm.yaml @@ -22,7 +22,7 @@ default_env: deps: - tags: detect,os -- tags: get,sys-utils-cm +#- tags: get,sys-utils-cm - names: - python - python3 diff --git a/script/app-image-classification-torch-py/_cm.json b/script/app-image-classification-torch-py/_cm.json index a6a78a679..4e30d3f5f 100644 --- a/script/app-image-classification-torch-py/_cm.json +++ b/script/app-image-classification-torch-py/_cm.json @@ -11,9 +11,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", From 62c38ef9e9e6214360a0da14a9a868ae33c18344 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 10:47:27 +0200 Subject: [PATCH 14/27] removed sys deps from image-classification examples and added YAML --- .../_cm.json | 73 ------------------- .../_cm.yaml | 48 ++++++++++++ 2 files changed, 48 insertions(+), 73 deletions(-) delete mode 100644 script/app-image-classification-tvm-onnx-py/_cm.json create mode 100644 script/app-image-classification-tvm-onnx-py/_cm.yaml diff --git a/script/app-image-classification-tvm-onnx-py/_cm.json b/script/app-image-classification-tvm-onnx-py/_cm.json deleted file mode 100644 index 1ae2e5c32..000000000 --- a/script/app-image-classification-tvm-onnx-py/_cm.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "alias": "app-image-classification-tvm-onnx-py", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Modular AI/ML application pipeline", - "default_env": { - "CM_BATCH_COUNT": "1", - "CM_BATCH_SIZE": "1" - }, - "deps": [ - { - "tags": "detect,os" - }, - { - "tags": "detect,cpu" - }, - { - "names": [ - "python", - "python3" - ], - "tags": "get,python3" - }, - { - "tags": "get,dataset,image-classification,original" - }, - { - "tags": "get,dataset-aux,image-classification" - }, - { - "tags": "get,raw,ml-model,image-classification,resnet50,_onnx" - }, - { - "tags": "get,generic-python-lib,_onnxruntime" - }, - { - "names": [ "tvm" ], - "tags": "get,tvm" - } - ], - "tags": [ - "app", - "image-classification", - "tvm-onnx", - "python" - ], - "tags_help":"app image-classification python tvm-onnx", - "uid": "63080407db4d4ac4", - "variations": { - "llvm": { - "add_deps_recursive": { - "tvm": { - "tags": "_llvm" - } - } - }, - "cuda": { - "add_deps_recursive": { - "tvm": { - "tags": "_cuda" - } - }, - "env": { - "USE_CUDA": "yes" - }, - "deps": [ - { - "tags": "get,cuda" - } - ] - } - } -} diff --git a/script/app-image-classification-tvm-onnx-py/_cm.yaml b/script/app-image-classification-tvm-onnx-py/_cm.yaml new file mode 100644 index 000000000..2b5cc9cca --- /dev/null +++ b/script/app-image-classification-tvm-onnx-py/_cm.yaml @@ -0,0 +1,48 @@ +alias: app-image-classification-tvm-onnx-py +uid: 63080407db4d4ac4 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Modular AI/ML application pipeline + +default_env: + CM_BATCH_COUNT: '1' + CM_BATCH_SIZE: '1' + +deps: +- tags: detect,os +- tags: detect,cpu +- names: + - python + - python3 + tags: get,python3 +- tags: get,dataset,image-classification,original +- tags: get,dataset-aux,image-classification +- tags: get,raw,ml-model,image-classification,resnet50,_onnx +- tags: get,generic-python-lib,_onnxruntime +- names: + - tvm + tags: get,tvm + +tags: +- app +- image-classification +- tvm-onnx +- python + +tags_help: app image-classification python tvm-onnx + +variations: + cuda: + add_deps_recursive: + tvm: + tags: _cuda + deps: + - tags: get,cuda + env: + USE_CUDA: 'yes' + llvm: + add_deps_recursive: + tvm: + tags: _llvm From f8a4e7a2ec59758197183bec01cebd115cc629c8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 17:37:59 +0200 Subject: [PATCH 15/27] fixing debug examples for customize.py and wrapped Python code (external to CM) --- COPYRIGHT.txt | 2 +- README.md | 2 +- debug.py | 9 +++++++++ script/print-hello-world-py/app.py | 16 ++++++++++++++++ script/print-hello-world-py/code.py | 6 ------ script/print-hello-world-py/customize.py | 19 +++++++++++++++++++ script/print-hello-world-py/run.bat | 2 +- script/print-hello-world-py/run.sh | 2 +- script/test-debug/_demo.py | 2 +- 9 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 debug.py create mode 100644 script/print-hello-world-py/app.py delete mode 100644 script/print-hello-world-py/code.py create mode 100644 script/print-hello-world-py/customize.py diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 80e8c506b..7b62e067d 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -1,5 +1,5 @@ Copyright (c) 2021-2024 MLCommons -The cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. +Grigori Fursin, the cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. Copyright (c) 2014-2021 cTuning foundation diff --git a/README.md b/README.md index 92f7f07fe..06c2cd2bc 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ cm run script \ [Apache 2.0](LICENSE.md) -## CM concept +## CM concepts * https://doi.org/10.5281/zenodo.8105339 * https://arxiv.org/abs/2406.16791 diff --git a/debug.py b/debug.py new file mode 100644 index 000000000..9fd54803c --- /dev/null +++ b/debug.py @@ -0,0 +1,9 @@ +# Developer: Grigori Fursin + +import cmind +import sys + +print(sys.executable) + +r = cmind.access('run script "print hello-world python" --debug_uid=f52670e5f3f345a2') +print(r) diff --git a/script/print-hello-world-py/app.py b/script/print-hello-world-py/app.py new file mode 100644 index 000000000..9dfd48b33 --- /dev/null +++ b/script/print-hello-world-py/app.py @@ -0,0 +1,16 @@ +def main(): + print ('') + + # Import cmind to test break points + import cmind.utils + import os + if os.environ.get('CM_TMP_DEBUG_UID', '') == 'f52670e5f3f345a2': + cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + + print ('HELLO WORLD from Python') + + x = 1 + print (x) + +if __name__ == '__main__': + main() diff --git a/script/print-hello-world-py/code.py b/script/print-hello-world-py/code.py deleted file mode 100644 index 735a89062..000000000 --- a/script/print-hello-world-py/code.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print ('') - print ('HELLO WORLD from Python') - -if __name__ == '__main__': - main() diff --git a/script/print-hello-world-py/customize.py b/script/print-hello-world-py/customize.py new file mode 100644 index 000000000..830869367 --- /dev/null +++ b/script/print-hello-world-py/customize.py @@ -0,0 +1,19 @@ +# Developer(s): Grigori Fursin + +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + + return {'return':0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + return {'return':0} diff --git a/script/print-hello-world-py/run.bat b/script/print-hello-world-py/run.bat index b4f5e3a7a..c0980c59b 100644 --- a/script/print-hello-world-py/run.bat +++ b/script/print-hello-world-py/run.bat @@ -2,7 +2,7 @@ IF NOT DEFINED CM_TMP_CURRENT_SCRIPT_PATH SET CM_TMP_CURRENT_SCRIPT_PATH=%CD% rem %CM_PYTHON_BIN_WITH_PATH% --version -%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\code.py +%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\app.py IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% echo CM_NEW_VAR_FROM_RUN=XYZ > tmp-run-env.out diff --git a/script/print-hello-world-py/run.sh b/script/print-hello-world-py/run.sh index 26b72ba62..bc7e2c301 100644 --- a/script/print-hello-world-py/run.sh +++ b/script/print-hello-world-py/run.sh @@ -5,7 +5,7 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} #which ${CM_PYTHON_BIN_WITH_PATH} #${CM_PYTHON_BIN_WITH_PATH} --version -${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/code.py +${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/app.py test $? -eq 0 || exit $? echo "CM_NEW_VAR_FROM_RUN=$MLPERF_XYZ" > tmp-run-env.out diff --git a/script/test-debug/_demo.py b/script/test-debug/_demo.py index 781bed321..878249a74 100644 --- a/script/test-debug/_demo.py +++ b/script/test-debug/_demo.py @@ -5,5 +5,5 @@ print(sys.executable) -r = cmind.access('run "cm-debug"') +r = cmind.access('run script "test cm-debug"') print(r) From afaee880508738e5bf08a13a59174e7ff4a1d1bf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 30 Sep 2024 17:14:10 +0100 Subject: [PATCH 16/27] Update customize.py | Fix download-file on windows when downloaded file already exists --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 97969418c..6e7aa0786 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -206,7 +206,7 @@ def preprocess(i): if not pre_clean: env['CM_PRE_DOWNLOAD_CMD'] = '' - if os_info['platform'] == 'windows': + if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD, '') != '': env['CM_DOWNLOAD_CMD'] = env['CM_DOWNLOAD_CMD'].replace('&', '^&').replace('|', '^|').replace('(', '^(').replace(')', '^)') if pre_clean: env['CM_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %CM_DOWNLOAD_FILENAME%" From 07da36145a8ccd947fe39725e0e111c783f00e8d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 30 Sep 2024 17:17:57 +0100 Subject: [PATCH 17/27] Update customize.py --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 6e7aa0786..9918de7c3 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -206,7 +206,7 @@ def preprocess(i): if not pre_clean: env['CM_PRE_DOWNLOAD_CMD'] = '' - if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD, '') != '': + if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD', '') != '': env['CM_DOWNLOAD_CMD'] = env['CM_DOWNLOAD_CMD'].replace('&', '^&').replace('|', '^|').replace('(', '^(').replace(')', '^)') if pre_clean: env['CM_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %CM_DOWNLOAD_FILENAME%" From 79871c0fd7f941f319bea767c969ed106d42d031 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 13:25:59 +0200 Subject: [PATCH 18/27] adding latest wget.exe to get-sys-utils-cm --- script/get-sys-utils-cm/_cm.json | 38 ------------------------------- script/get-sys-utils-cm/_cm.yaml | 3 ++- script/get-sys-utils-min/_cm.json | 2 +- 3 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 script/get-sys-utils-cm/_cm.json diff --git a/script/get-sys-utils-cm/_cm.json b/script/get-sys-utils-cm/_cm.json deleted file mode 100644 index a496f42c9..000000000 --- a/script/get-sys-utils-cm/_cm.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "alias": "get-sys-utils-cm", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "cache": true, - "category": "Detection or installation of tools and artifacts", - "deps": [ - { - "tags": "detect,os" - } - ], - "env": { - "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1", - "CM_SUDO": "sudo" - }, - "input_mapping": { - "skip": "CM_SKIP_SYS_UTILS" - }, - "new_env_keys": [ - "+PATH" - ], - "tags": [ - "get", - "sys-utils-cm" - ], - "uid": "bc90993277e84b8e", - "variations": { - "user": { - "env": { - "CM_PYTHON_PIP_USER": "--user" - } - } - }, - "warnings": [ - "This CM script will install extra OS system utils required for CM automation workflows!" - ] -} diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 555fe7c6a..1bbfac9ee 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -13,8 +13,9 @@ deps: env: CM_CLEAN_DIRS: bin - CM_PACKAGE_WIN_URL: https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip + CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 + ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip CM_SUDO: sudo input_mapping: diff --git a/script/get-sys-utils-min/_cm.json b/script/get-sys-utils-min/_cm.json index d033f8d1c..19ce3f21a 100644 --- a/script/get-sys-utils-min/_cm.json +++ b/script/get-sys-utils-min/_cm.json @@ -8,7 +8,7 @@ ], "env": { "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/10379926/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", + "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", "CM_SUDO": "sudo" }, "new_env_keys": [ From 5419871b768f63a89e2018cfb5f0f295408a674b Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 14:48:53 +0200 Subject: [PATCH 19/27] turn on tests on Windows --- .github/workflows/test-mlperf-inference-resnet50.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 797c6f2a0..69ce2722a 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -27,7 +27,7 @@ jobs: implementation: cpp - os: macos-latest backend: tf - - os: windows-latest +# - os: windows-latest steps: - uses: actions/checkout@v4 From 0942459e6a3b83892d0998c034fce32f7ec6688e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:08:21 +0530 Subject: [PATCH 20/27] fixed md5sum bug in windows --- script/download-file/customize.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 9918de7c3..64d9c9b64 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -6,6 +6,11 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + + # env to be passed to the subprocess + subprocess_env = {} + subprocess_env['PATH'] = ';'.join(env['+PATH']) + meta = i['meta'] automation = i['automation'] @@ -18,7 +23,7 @@ def preprocess(i): # xsep = '^&^&' if windows else '&&' xsep = '&&' - q = '"' if os_info['platform'] == 'windows' else "'" + q = '' if os_info['platform'] == 'windows' else "'" x='*' if os_info['platform'] == 'windows' else '' x_c='-s' if os_info['platform'] == 'darwin_off' else '' @@ -90,12 +95,12 @@ def preprocess(i): cmutil_require_download = 0 if env.get('CM_DOWNLOAD_CHECKSUM_FILE', '') != '': checksum_cmd = f"cd {q}{filepath}{q} {xsep} md5sum -c{x_c} {x}{q}{env['CM_DOWNLOAD_CHECKSUM_FILE']}{q}" - checksum_result = subprocess.run(checksum_cmd, cwd=f'{q}{filepath}{q}', capture_output=True, text=True, shell=True) + checksum_result = subprocess.run(checksum_cmd, cwd=f'{q}{filepath}{q}', capture_output=True, text=True, shell=True, env=subprocess_env) elif env.get('CM_DOWNLOAD_CHECKSUM', '') != '': checksum_cmd = f"echo {env.get('CM_DOWNLOAD_CHECKSUM')} {x}{q}{env['CM_DOWNLOAD_FILENAME']}{q} | md5sum -c{x_c} -" - checksum_result = subprocess.run(checksum_cmd, capture_output=True, text=True, shell=True) + checksum_result = subprocess.run(checksum_cmd, capture_output=True, text=True, shell=True, env=subprocess_env) if env.get('CM_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get('CM_DOWNLOAD_CHECKSUM', '') != '': - #print(checksum_result) #for debugging + # print(checksum_result) #for debugging if "checksum did not match" in checksum_result.stderr.lower(): computed_checksum = subprocess.run(f"md5sum {env['CM_DOWNLOAD_FILENAME']}", capture_output=True, text=True, shell=True).stdout.split(" ")[0] print(f"WARNING: File already present, mismatch between original checksum({env.get('CM_DOWNLOAD_CHECKSUM')}) and computed checksum({computed_checksum}). Deleting the already present file and downloading new.") From d459ef6b9cabf6214556f1e34172a7e22600fba6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:26:07 +0530 Subject: [PATCH 21/27] better handling of env variable --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 64d9c9b64..5b3d7e4fe 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -9,7 +9,7 @@ def preprocess(i): # env to be passed to the subprocess subprocess_env = {} - subprocess_env['PATH'] = ';'.join(env['+PATH']) + subprocess_env['PATH'] = ';'.join(env.get('+PATH', '')) meta = i['meta'] From 7dcaa616a32cb69276c39b0b4d3276f20d3b99ff Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 1 Oct 2024 19:03:12 +0530 Subject: [PATCH 22/27] improved env handling + remove escaping --- script/download-file/customize.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 5b3d7e4fe..3101f6c40 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -8,8 +8,8 @@ def preprocess(i): env = i['env'] # env to be passed to the subprocess - subprocess_env = {} - subprocess_env['PATH'] = ';'.join(env.get('+PATH', '')) + subprocess_env = os.environ.copy() + subprocess_env['PATH'] += os.pathsep + os.pathsep.join(env.get('+PATH', '')) meta = i['meta'] @@ -113,7 +113,7 @@ def preprocess(i): elif "no such file" in checksum_result.stderr.lower(): #print(f"No file {env['CM_DOWNLOAD_FILENAME']}. Downloading through cmutil.") cmutil_require_download = 1 - elif checksum_result.returncode == 1: + elif checksum_result.returncode > 0: return {"return":1, "error":f"Error while checking checksum: {checksum_result.stderr}"} else: print(f"File {env['CM_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..") @@ -212,7 +212,6 @@ def preprocess(i): env['CM_PRE_DOWNLOAD_CMD'] = '' if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD', '') != '': - env['CM_DOWNLOAD_CMD'] = env['CM_DOWNLOAD_CMD'].replace('&', '^&').replace('|', '^|').replace('(', '^(').replace(')', '^)') if pre_clean: env['CM_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %CM_DOWNLOAD_FILENAME%" # Check that if empty CMD, should add "" From a92f8d506b07ab3538f1e16864ee4c678d26509e Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 17:39:44 +0200 Subject: [PATCH 23/27] * removed windows test for MLPerf (requires interaction) * added windows test for image classification --- .github/workflows/test-image-classification-onnx.yml | 6 +++--- .github/workflows/test-mlperf-inference-resnet50.yml | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-image-classification-onnx.yml b/.github/workflows/test-image-classification-onnx.yml index 62284a2a9..44f27a616 100644 --- a/.github/workflows/test-image-classification-onnx.yml +++ b/.github/workflows/test-image-classification-onnx.yml @@ -13,12 +13,12 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [ "3.12"] + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.10", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 69ce2722a..acbe88b90 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -27,7 +27,9 @@ jobs: implementation: cpp - os: macos-latest backend: tf -# - os: windows-latest + - os: windows-latest +# MLPerf requires interaction when installing LLVM on Windows - that's why we excluded it here + steps: - uses: actions/checkout@v4 From e71477a15ace1221a334d893da39430398687fdc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:35:39 +0530 Subject: [PATCH 24/27] commit for windows download-file errorfix --- script/download-file/customize.py | 40 ++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 3101f6c40..3465df231 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -2,6 +2,21 @@ import os import subprocess +def escape_special_chars(text, tool=None): + special_chars = [ + '&', '|', '(', ')' + ] + + for char in special_chars: + text = text.replace(char, f'^{char}') + + #handle URL special cases + if tool != "rclone": + text = text.replace('%', "%%") + + print(text) + return text + def preprocess(i): os_info = i['os_info'] @@ -23,7 +38,7 @@ def preprocess(i): # xsep = '^&^&' if windows else '&&' xsep = '&&' - q = '' if os_info['platform'] == 'windows' else "'" + q = '"' if os_info['platform'] == 'windows' else "'" x='*' if os_info['platform'] == 'windows' else '' x_c='-s' if os_info['platform'] == 'darwin_off' else '' @@ -91,13 +106,18 @@ def preprocess(i): env['CM_DOWNLOAD_FILENAME'] = "index.html" if tool == "cmutil": - print ('') cmutil_require_download = 0 if env.get('CM_DOWNLOAD_CHECKSUM_FILE', '') != '': - checksum_cmd = f"cd {q}{filepath}{q} {xsep} md5sum -c{x_c} {x}{q}{env['CM_DOWNLOAD_CHECKSUM_FILE']}{q}" + if os_info['platform'] == 'windows': + checksum_cmd = f"cd {q}{filepath}{q} {xsep} md5sum -c{x_c} {x}{escape_special_chars(env['CM_DOWNLOAD_CHECKSUM_FILE'])}" + else: + checksum_cmd = f"cd {q}{filepath}{q} {xsep} md5sum -c{x_c} {x}{q}{env['CM_DOWNLOAD_CHECKSUM_FILE']}{q}" checksum_result = subprocess.run(checksum_cmd, cwd=f'{q}{filepath}{q}', capture_output=True, text=True, shell=True, env=subprocess_env) elif env.get('CM_DOWNLOAD_CHECKSUM', '') != '': - checksum_cmd = f"echo {env.get('CM_DOWNLOAD_CHECKSUM')} {x}{q}{env['CM_DOWNLOAD_FILENAME']}{q} | md5sum -c{x_c} -" + if os_info['platform'] == 'windows': + checksum_cmd = f"echo {env.get('CM_DOWNLOAD_CHECKSUM')} {x}{escape_special_chars(env['CM_DOWNLOAD_FILENAME'])} | md5sum -c{x_c} -" + else: + checksum_cmd = f"echo {env.get('CM_DOWNLOAD_CHECKSUM')} {x}{q}{env['CM_DOWNLOAD_FILENAME']}{q} | md5sum -c{x_c} -" checksum_result = subprocess.run(checksum_cmd, capture_output=True, text=True, shell=True, env=subprocess_env) if env.get('CM_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get('CM_DOWNLOAD_CHECKSUM', '') != '': # print(checksum_result) #for debugging @@ -200,11 +220,18 @@ def preprocess(i): if env.get('CM_DOWNLOAD_CHECKSUM_FILE', '') != '': env['CM_DOWNLOAD_CHECKSUM_CMD'] = f"cd {q}{filepath}{q} {xsep} md5sum -c {x_c} {x}{q}{env['CM_DOWNLOAD_CHECKSUM_FILE']}{q}" elif env.get('CM_DOWNLOAD_CHECKSUM', '') != '': - env['CM_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{}{}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'), x, q, env['CM_DOWNLOAD_FILENAME'], q, x_c) + if os_info['platform'] == 'windows': + env['CM_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'), x, escape_special_chars(env['CM_DOWNLOAD_FILENAME']), x_c) + else: + env['CM_DOWNLOAD_CHECKSUM_CMD'] = "echo {} {}{}{}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'), x, q, env['CM_DOWNLOAD_FILENAME'], q, x_c) for i in range(1,5): if env.get('CM_DOWNLOAD_CHECKSUM'+str(i),'') == '': break - env['CM_DOWNLOAD_CHECKSUM_CMD'] += " || echo {} {}{}{}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'+str(i)), x, q, env['CM_DOWNLOAD_FILENAME'], q, x_c) + if os_info['platform'] == 'windows': + env['CM_DOWNLOAD_CHECKSUM_CMD'] += " || echo {} {}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'+str(i)), x, escape_special_chars(env['CM_DOWNLOAD_FILENAME']), x_c) + else: + env['CM_DOWNLOAD_CHECKSUM_CMD'] += " || echo {} {}{}{}{} | md5sum {} -c -".format(env.get('CM_DOWNLOAD_CHECKSUM'+str(i)), x, q, env['CM_DOWNLOAD_FILENAME'].replace("%", "%%"), q, x_c) + # print(env['CM_DOWNLOAD_CHECKSUM_CMD']) else: env['CM_DOWNLOAD_CHECKSUM_CMD'] = "" @@ -212,6 +239,7 @@ def preprocess(i): env['CM_PRE_DOWNLOAD_CMD'] = '' if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD', '') != '': + env['CM_DOWNLOAD_CMD'] = escape_special_chars(env['CM_DOWNLOAD_CMD'], tool) if pre_clean: env['CM_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %CM_DOWNLOAD_FILENAME%" # Check that if empty CMD, should add "" From b522d7eb47556fc6973f5e8d1ae4c36dbdf0a04c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:55:14 +0530 Subject: [PATCH 25/27] code clean --- script/download-file/customize.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 3465df231..4f21ce9f8 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -13,8 +13,7 @@ def escape_special_chars(text, tool=None): #handle URL special cases if tool != "rclone": text = text.replace('%', "%%") - - print(text) + return text def preprocess(i): From b9cfe355c032aca44824f35c70d88b4fc05606e3 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 2 Oct 2024 13:54:51 +0200 Subject: [PATCH 26/27] fixed a few outdated URLs for Windows --- script/get-dataset-imagenet-val/_cm.json | 121 ----------------------- script/get-dataset-imagenet-val/_cm.yaml | 101 +++++++++++++++++++ script/get-sys-utils-cm/_cm.yaml | 4 +- script/get-sys-utils-min/_cm.json | 22 ----- script/get-sys-utils-min/_cm.yaml | 25 +++++ 5 files changed, 128 insertions(+), 145 deletions(-) delete mode 100644 script/get-dataset-imagenet-val/_cm.json create mode 100644 script/get-dataset-imagenet-val/_cm.yaml delete mode 100644 script/get-sys-utils-min/_cm.json create mode 100644 script/get-sys-utils-min/_cm.yaml diff --git a/script/get-dataset-imagenet-val/_cm.json b/script/get-dataset-imagenet-val/_cm.json deleted file mode 100644 index e41cf50cb..000000000 --- a/script/get-dataset-imagenet-val/_cm.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "alias": "get-dataset-imagenet-val", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "cache": true, - "category": "AI/ML datasets", - "env": { - "CM_DATASET": "IMAGENET" - }, - "new_env_keys": [ - "CM_DATASET_PATH", - "CM_DATASET_IMAGENET_PATH", - "CM_DATASET_IMAGENET_VAL_PATH", - "CM_DATASET_SIZE", - "CM_DATASET_VER" - ], - "tags": [ - "get", - "val", - "validation", - "dataset", - "imagenet", - "ILSVRC", - "image-classification", - "original" - ], - "input_mapping": { - "imagenet_path": "IMAGENET_PATH", - "torrent": "CM_DATASET_IMAGENET_VAL_TORRENT_PATH" - }, - "uid": "7afd58d287fe4f11", - "deps": [ - { - "tags": "detect,os" - } - ], - "prehook_deps": [ - { - "tags": "download-and-extract,file,_extract", - "update_tags_from_env_with_prefix": { - "_url.": [ "CM_DAE_URL" ] - }, - "update_tags_from_env": [ - "CM_DAE_EXTRA_TAGS" - ], - "env": { - "CM_EXTRACT_TO_FOLDER": "imagenet-2012-val" - }, - "enable_if_env": { - "CM_DATASET_IMAGENET_VAL_REQUIRE_DAE": ["yes", "True"] - } - }, - { - "tags": "file,extract,_no-remove-extracted", - "update_tags_from_env_with_prefix": { - "_path.": [ "CM_EXTRACT_PATH" ] - }, - "env": { - "CM_EXTRACT_TO_FOLDER": "imagenet-2012-val" - }, - "enable_if_env": { - "CM_DAE_ONLY_EXTRACT": ["yes", "True"] - } - } - ], - "variations": { - "2012": { - "group": "dataset-version", - "default": true, - "env": { - "CM_DATASET_VER": "2012" - } - }, - "2012-500": { - "base": [ - "size.500", - "2012" - ] - }, - "2012-full": { - "base": [ - "full", - "2012" - ] - }, - "full": { - "group": "count", - "env": { - "CM_DATASET_SIZE": "50000", - "CM_IMAGENET_FULL": "yes", - "CM_DAE_FILENAME": "ILSVRC2012_img_val.tar", - "CM_DOWNLOAD_CHECKSUM": "29b22e2961454d5413ddabcf34fc5622" - } - }, - "size.500": { - "group": "count", - "default": true, - "env": { - "CM_DATASET_SIZE": "500", - "CM_DAE_FILENAME": "ILSVRC2012_img_val_500.tar", - "CM_DOWNLOAD_FILENAME": "ILSVRC2012_img_val_500.tar", - "CM_DAE_URL": "http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar", - "CM_DOWNLOAD_CHECKSUM": "8627befdd8c2bcf305729020e9db354e" - } - }, - "size.#": { - "group": "count", - "env": { - "CM_DATASET_SIZE": "#" - } - }, - "run-during-docker-build": { - "docker": { - "run": true - } - } - }, - "docker": { - "run": false - } -} diff --git a/script/get-dataset-imagenet-val/_cm.yaml b/script/get-dataset-imagenet-val/_cm.yaml new file mode 100644 index 000000000..d21cdad2a --- /dev/null +++ b/script/get-dataset-imagenet-val/_cm.yaml @@ -0,0 +1,101 @@ +alias: get-dataset-imagenet-val +uid: 7afd58d287fe4f11 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: true +category: AI/ML datasets + +deps: +- tags: detect,os + +docker: + run: false + +env: + CM_DATASET: IMAGENET + +input_mapping: + imagenet_path: IMAGENET_PATH + torrent: CM_DATASET_IMAGENET_VAL_TORRENT_PATH + +new_env_keys: +- CM_DATASET_PATH +- CM_DATASET_IMAGENET_PATH +- CM_DATASET_IMAGENET_VAL_PATH +- CM_DATASET_SIZE +- CM_DATASET_VER + +prehook_deps: +- enable_if_env: + CM_DATASET_IMAGENET_VAL_REQUIRE_DAE: + - 'yes' + - 'True' + env: + CM_EXTRACT_TO_FOLDER: imagenet-2012-val + tags: download-and-extract,file,_extract + update_tags_from_env: + - CM_DAE_EXTRA_TAGS + update_tags_from_env_with_prefix: + _url.: + - CM_DAE_URL +- enable_if_env: + CM_DAE_ONLY_EXTRACT: + - 'yes' + - 'True' + env: + CM_EXTRACT_TO_FOLDER: imagenet-2012-val + tags: file,extract,_no-remove-extracted + update_tags_from_env_with_prefix: + _path.: + - CM_EXTRACT_PATH + +tags: +- get +- val +- validation +- dataset +- imagenet +- ILSVRC +- image-classification +- original + +variations: + '2012': + default: true + env: + CM_DATASET_VER: '2012' + group: dataset-version + 2012-500: + base: + - size.500 + - '2012' + 2012-full: + base: + - full + - '2012' + full: + env: + CM_DAE_FILENAME: ILSVRC2012_img_val.tar + CM_DATASET_SIZE: '50000' + CM_DOWNLOAD_CHECKSUM: 29b22e2961454d5413ddabcf34fc5622 + CM_IMAGENET_FULL: 'yes' + group: count + run-during-docker-build: + docker: + run: true + size.#: + env: + CM_DATASET_SIZE: '#' + group: count + size.500: + default: true + env: + CM_DAE_FILENAME: ILSVRC2012_img_val_500.tar +# CM_DAE_URL: http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar + CM_DAE_URL: https://www.dropbox.com/scl/fi/a7fhjnzxi6x3ceapxh5bm/ILSVRC2012_img_val_500.tar?rlkey=hz4rabo9ve43co3c303y9r6l7&st=ulcgb3av&dl=1 + CM_DATASET_SIZE: '500' + CM_DOWNLOAD_CHECKSUM: 8627befdd8c2bcf305729020e9db354e + CM_DOWNLOAD_FILENAME: ILSVRC2012_img_val_500.tar + group: count diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 1bbfac9ee..30939d556 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -14,8 +14,8 @@ deps: env: CM_CLEAN_DIRS: bin CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 - ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 - ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip + ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1 + ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1 CM_SUDO: sudo input_mapping: diff --git a/script/get-sys-utils-min/_cm.json b/script/get-sys-utils-min/_cm.json deleted file mode 100644 index 19ce3f21a..000000000 --- a/script/get-sys-utils-min/_cm.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "alias": "get-sys-utils-min", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "cache": true, - "category": "Detection or installation of tools and artifacts", - "deps": [ - ], - "env": { - "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", - "CM_SUDO": "sudo" - }, - "new_env_keys": [ - "+PATH" - ], - "tags": [ - "get", - "sys-utils-min" - ], - "uid": "a9af7714d3d94779" -} diff --git a/script/get-sys-utils-min/_cm.yaml b/script/get-sys-utils-min/_cm.yaml new file mode 100644 index 000000000..8f623e2c1 --- /dev/null +++ b/script/get-sys-utils-min/_cm.yaml @@ -0,0 +1,25 @@ +alias: get-sys-utils-min +uid: a9af7714d3d94779 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: true + +category: Detection or installation of tools and artifacts + +deps: [] + +env: + CM_CLEAN_DIRS: bin + CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 + ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1 + ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1 + CM_SUDO: sudo + +new_env_keys: +- +PATH + +tags: +- get +- sys-utils-min From 535d0d5d0df8342824c227b201b7912f153694b9 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 2 Oct 2024 14:02:17 +0200 Subject: [PATCH 27/27] fix stable version --- cmr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmr.yaml b/cmr.yaml index f84a5d345..226fd06b6 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -3,7 +3,7 @@ uid: 9e97bb72b0474657 git: true -version: 2.3.4 +version: 2.3.9 deps: - alias: mlcommons@ck