Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
wu-kan authored Mar 24, 2024
1 parent de9050e commit 8fe44ed
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 14 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
- name: Build Docker image
uses: docker/build-push-action@v3
with:
build-args: SCC_TAG=${{ steps.meta.outputs.version }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
load: true
Expand Down Expand Up @@ -83,6 +84,7 @@ jobs:
id: build-and-push
uses: docker/build-push-action@v3
with:
build-args: SCC_TAG=${{ steps.meta.outputs.version }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: ${{ github.event_name != 'pull_request' }}
17 changes: 14 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,22 @@

## How to use

### 使用预编译的环境

```shell
docker run \
--name sccenv \
wukan0621/sccenv
# 请提前将 sccenv 挂载到目标位置
docker cp sccenv:/sccenv /
docker rm sccenv
```

### 从零开始

最小化配置一个可以使用的 spack,需要的软件依赖可以参考 [Dockerfile](./Dockerfile)

```bash
```shell
python3 -c "from tarfile import open;from urllib.request import urlopen;open(mode='r|gz',fileobj=urlopen('https://github.com/SYSU-SCC/sysu-scc-spack-repo/archive/refs/tags/v0.21.2.12.3.0.12.20240311.tar.gz')).extractall()"

# 只依赖这一个环境变量,可以放进 ~/.bashrc
Expand All @@ -38,7 +49,7 @@ $(dirname $SCC_SETUP_ENV)/init-default-compiler.sh "[email protected] target=x8

### 集成进已有的 spack 环境

```bash
```shell
python3 -c "from tarfile import open;from urllib.request import urlopen;open(mode='r|gz',fileobj=urlopen('https://github.com/SYSU-SCC/sysu-scc-spack-repo/archive/refs/heads/latest.tar.gz')).extractall()"
spack repo add --scope=site sysu-scc-spack-repo-latest

Expand All @@ -51,7 +62,7 @@ spack env deactivate

### 测试是否能用

```bash
```shell
spack install sysu-scc.hpl-ai ^blaspp+openmp ^openblas threads=openmp ^mpich
spack load hpl-ai
cp $(spack location -i hpl-ai)/bin/HPL.dat HPL.dat
Expand Down
40 changes: 29 additions & 11 deletions lib/spack/spack/util/compression.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,9 @@ def _bunzip2(archive_file: str) -> str:
def _py_bunzip(archive_file: str) -> str:
"""Returns path to decompressed file.
Decompresses bz2 compressed archives/files via python's bz2 module"""
decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "bz2"))
decompressed_file = os.path.basename(
llnl.url.strip_compression_extension(archive_file, "bz2")
)
working_dir = os.getcwd()
archive_out = os.path.join(working_dir, decompressed_file)
f_bz = bz2.BZ2File(archive_file, mode="rb")
Expand All @@ -150,7 +152,9 @@ def _system_bunzip(archive_file: str) -> str:
"""Returns path to decompressed file.
Decompresses bz2 compressed archives/files via system bzip2 utility"""
compressed_file_name = os.path.basename(archive_file)
decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "bz2"))
decompressed_file = os.path.basename(
llnl.url.strip_compression_extension(archive_file, "bz2")
)
working_dir = os.getcwd()
archive_out = os.path.join(working_dir, decompressed_file)
copy_path = os.path.join(working_dir, compressed_file_name)
Expand All @@ -175,7 +179,9 @@ def _gunzip(archive_file: str) -> str:
def _py_gunzip(archive_file: str) -> str:
"""Returns path to gunzip'd file. Decompresses `.gz` compressed archvies via python gzip
module"""
decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "gz"))
decompressed_file = os.path.basename(
llnl.url.strip_compression_extension(archive_file, "gz")
)
working_dir = os.getcwd()
destination_abspath = os.path.join(working_dir, decompressed_file)
f_in = gzip.open(archive_file, "rb")
Expand Down Expand Up @@ -235,11 +241,14 @@ def _system_unZ(archive_file: str) -> str:

def _lzma_decomp(archive_file):
"""Returns path to decompressed xz file. Decompress lzma compressed files. Prefer Python native
lzma module, but fall back on command line xz tooling to find available Python support."""
lzma module, but fall back on command line xz tooling to find available Python support.
"""
return _py_lzma(archive_file) if LZMA_SUPPORTED else _xz(archive_file)


def _win_compressed_tarball_handler(decompressor: Callable[[str], str]) -> Callable[[str], str]:
def _win_compressed_tarball_handler(
decompressor: Callable[[str], str]
) -> Callable[[str], str]:
"""Returns function pointer to two stage decompression
and extraction method
Decompress and extract compressed tarballs on Windows.
Expand Down Expand Up @@ -267,7 +276,9 @@ def unarchive(archive_file: str):
def _py_lzma(archive_file: str) -> str:
"""Returns path to decompressed .xz files. Decompress lzma compressed .xz files via Python
lzma module."""
decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "xz"))
decompressed_file = os.path.basename(
llnl.url.strip_compression_extension(archive_file, "xz")
)
archive_out = os.path.join(os.getcwd(), decompressed_file)
with open(archive_out, "wb") as ar:
with lzma.open(archive_file) as lar:
Expand All @@ -278,7 +289,9 @@ def _py_lzma(archive_file: str) -> str:
def _xz(archive_file):
"""Returns path to decompressed xz files. Decompress lzma compressed .xz files via xz command
line tool."""
decompressed_file = os.path.basename(llnl.url.strip_extension(archive_file, extension="xz"))
decompressed_file = os.path.basename(
llnl.url.strip_extension(archive_file, extension="xz")
)
working_dir = os.getcwd()
destination_abspath = os.path.join(working_dir, decompressed_file)
compressed_file = os.path.basename(archive_file)
Expand Down Expand Up @@ -354,7 +367,9 @@ def decompressor_for_nix(extension: str) -> Callable[[str], Any]:
return extension_to_decompressor.get(extension, _untar)


def _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[str], Any]]:
def _determine_py_decomp_archive_strategy(
extension: str,
) -> Optional[Callable[[str], Any]]:
"""Returns appropriate python based decompression strategy
based on extension type"""
extension_to_decompressor: Dict[str, Callable[[str], str]] = {
Expand Down Expand Up @@ -432,7 +447,9 @@ class FileTypeInterface:
def magic_numbers(cls) -> List[bytes]:
"""Return a list of all potential magic numbers for a filetype"""
return [
value for name, value in inspect.getmembers(cls) if name.startswith("_MAGIC_NUMBER")
value
for name, value in inspect.getmembers(cls)
if name.startswith("_MAGIC_NUMBER")
]

@classmethod
Expand Down Expand Up @@ -589,7 +606,8 @@ def extension_from_magic_numbers_by_stream(

def _maybe_abbreviate_extension(path: str, extension: str) -> str:
"""If the file is a compressed tar archive, return the abbreviated extension t[xz|gz|bz2|bz]
instead of tar.[xz|gz|bz2|bz] if the file's original name also has an abbreviated extension."""
instead of tar.[xz|gz|bz2|bz] if the file's original name also has an abbreviated extension.
"""
if not extension.startswith("tar."):
return extension
abbr = f"t{extension[4:]}"
Expand Down Expand Up @@ -626,4 +644,4 @@ def extension_from_magic_numbers(path: str, decompress: bool = False) -> Optiona
return _maybe_abbreviate_extension(path, ext)

# Otherwise, use the extension from the file name.
return llnl.url.extension_from_path(path)
return llnl.url.extension_from_path(path)

0 comments on commit 8fe44ed

Please sign in to comment.