From 1b53fcac29f7b6513c9415c8de282b999579ca70 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 13:37:32 +0800 Subject: [PATCH 01/23] prepare docsity files --- convert_to_docsify.sh | 38 ++++++++++++++++++++++++ docs/Dockerfile | 6 ++++ docs/_coverpage.md | 4 +++ docs/_navbar.md | 3 ++ docs/_sidebar.md | 31 +++++++++++++++++++ docs/index.html | 69 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100644 convert_to_docsify.sh create mode 100644 docs/Dockerfile create mode 100644 docs/_coverpage.md create mode 100644 docs/_navbar.md create mode 100644 docs/_sidebar.md create mode 100644 docs/index.html diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh new file mode 100644 index 0000000..0eacbec --- /dev/null +++ b/convert_to_docsify.sh @@ -0,0 +1,38 @@ +#!/bin/bash +################### convert_to_docsify.sh ################### +# This script is used to convert the original markdown files +# to docsify markdown files. +############################################################# +set -e + +# Check if the script is running inside a Git repository +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "This script must be run inside a Git repository!!!" + exit 1 +fi + +source_file="docs/index.md" +target_file="docs/README.md" + +# Check if source file exists +if [ ! -f "$source_file" ]; then + echo "source file $source_file does not exist" +else + # Rename file + mv "$source_file" "$target_file" +fi + +# Delete start part of the file using awk +find . -type f -name "*.md" -exec awk '/^---$/ { p = !p; next } !p' {} > {}.tmp && mv {}.tmp {} \; + +# # Convert images path using sed +# find . -type f -name "*.md" -exec sed -i 's|\.\./images/\(.*\.png\)|./images/\1|g' {} + + +# Build Docker image of docsify +if [ -d "docs" ] && [ "$(id -u)" = "0" ]; then + cd docs + docker build -f Dockerfile -t docsify/demo . + docker restart docsify +else + echo "docs directory does not exist or you are not root" +fi \ No newline at end of file diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 0000000..d51f6df --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,6 @@ + FROM node:latest + LABEL description="A demo Dockerfile for build Docsify." + WORKDIR /docs + RUN npm install -g docsify-cli@latest + EXPOSE 3000/tcp + ENTRYPOINT docsify serve . \ No newline at end of file diff --git a/docs/_coverpage.md b/docs/_coverpage.md new file mode 100644 index 0000000..30f009a --- /dev/null +++ b/docs/_coverpage.md @@ -0,0 +1,4 @@ + + +[GitHub](https://github.com/gravity-doc/gravity-doc.github.io) +[Get started](/README.md) diff --git a/docs/_navbar.md b/docs/_navbar.md new file mode 100644 index 0000000..a6cbb16 --- /dev/null +++ b/docs/_navbar.md @@ -0,0 +1,3 @@ +- [Cheatsheet](/QuickStart?id=cheatsheet-📜) +- [Gravity Homepage](https://jupyter.gravity.sjtu.edu.cn/) +- [Department of Astronomy (SJTU)](http://astro.sjtu.edu.cn/) \ No newline at end of file diff --git a/docs/_sidebar.md b/docs/_sidebar.md new file mode 100644 index 0000000..0a27f98 --- /dev/null +++ b/docs/_sidebar.md @@ -0,0 +1,31 @@ +- [Quick start](/QuickStart.md) + +- Basic + + - [Account](/Basic/Account.md) + - [Login](/Basic/Login.md) + - [Job](/Basic/Job.md) + - [Quota usage](/Basic/Resource_Monitor.md) + - [Data transfer](/Basic/Data_Transfer.md) + - [Coding](/Basic/Coding.md) + +- Advanced + + - [X11 tunnel](/Advanced/X11_Tunnel.md) + - [VNC](/Advanced/VNC.md) + - [Jupyter](/Advanced/jupyter_notebook.md) + - [Status/Statistics website](/Advanced/status_website.md) + - [Globus](/Advanced/Globus.md) + - [Gitlab](/Advanced/Gitlab.md) + +- Tools + + - [Load installed module](/Software/Software_Installed.md) + - [Custom installation](/Software/User_Software_Installation.md) + - [Proxy](/Software/Proxy.md) + +- [FAQ](/MISC/FAQ.md) +- [Issues](/MISC/Issues.md) +- [Cluster info](/About.md) +- [Policy](/Policy.md) +- [Changelog](/Updates.md) diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..295dd79 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,69 @@ + + + + + Gravity Documentation + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + From 1b99e3837e0e0e2527167056408d9323da791556 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 14:04:11 +0800 Subject: [PATCH 02/23] convert HTML format figure to Markdown format --- convert_to_docsify.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index 0eacbec..3405d8b 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -22,11 +22,17 @@ else mv "$source_file" "$target_file" fi -# Delete start part of the file using awk -find . -type f -name "*.md" -exec awk '/^---$/ { p = !p; next } !p' {} > {}.tmp && mv {}.tmp {} \; +# Function to process markdown files +process_markdown() { + echo "Processing $1" + # remove part of "--- ... ---" + sed -i '/^---$/,/^---$/d' "$1" + # convert HTML image to markdown image + sed -i 's||![\1](\0)|g' "$1" +} -# # Convert images path using sed -# find . -type f -name "*.md" -exec sed -i 's|\.\./images/\(.*\.png\)|./images/\1|g' {} + +# Process markdown files using find and process_markdown function +find . -type f -name "*.md" -exec bash -c 'process_markdown "$0"' {} \; # Build Docker image of docsify if [ -d "docs" ] && [ "$(id -u)" = "0" ]; then @@ -35,4 +41,6 @@ if [ -d "docs" ] && [ "$(id -u)" = "0" ]; then docker restart docsify else echo "docs directory does not exist or you are not root" -fi \ No newline at end of file +fi + +echo "Done" \ No newline at end of file From 99a0c9b547f6f519adafb7cb0692a51641df3e5c Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 14:09:37 +0800 Subject: [PATCH 03/23] rename index.md to README.md --- docs/{index.md => README.md} | 82 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 41 deletions(-) rename docs/{index.md => README.md} (98%) diff --git a/docs/index.md b/docs/README.md similarity index 98% rename from docs/index.md rename to docs/README.md index f726098..f6bfc81 100644 --- a/docs/index.md +++ b/docs/README.md @@ -1,41 +1,41 @@ -## [Gravity Documentation](https://gravity-doc.github.io) - This is the documentation for the **Gravity Cluster** of [Department of Astronomy (DOA)](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). - -```tip -### Welcome to contribute -This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. Contributing is pretty simple as following: -1. **Click**. To contribute, click the **[edit](https://github.com/gravity-doc/gravity-doc.github.io/edit/master/docs/index.md)** :pencil: on the upper right of the page you feel like editing. Every document page is **editable**. It will bring you to the source files on github. You may will need to login to github to proceed. Once logged-in, a fork of the document repository will be created under your own account, so that you can edit it in whatever way you like. -2. **Edit**. Edit the page to contribute useful stuff. Check [here](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github/basic-writing-and-formatting-syntax) or the existing source pages for examples in the syntax. [This page](https://rundocs.io/writng/) contains theme-related syntax. -3. **Commit**. Finish the editing by leaving a commit message, describing what you have done. -4. **Submit**. Submit a **pull request** so that your changes will be submitted to the upstream branch (a.k.a., this website). The administrators will review your changes and absorb them into the website. - -Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. -### **中文内容也欢迎！🥳** -``` -## Contact - -You can contact us via - -- [gravity-hpc@sjtu.edu.cn ](mailto:gravity-hpc@sjtu.edu.cn) 📧 -- **WeChat** group 💬 -- Administrators: Yihe Wang, Yu Yu, Jiaxin Han - -if you need - -1. create an **account** 🙋‍♂️🙋‍♀️ -2. change **password** 🔐 -3. **extend time** of your job ⌛ -4. other problems ❔ - -## Acknowledging Gravity Cluster - -We would appreciate it if all publications that include results generated using *Gravity* cluster include a short statement in the Acknowledgment section. For example: - -若您在论文中使用了Gravity得到的结果，希望您能够在致谢中提到**Gravity**。例如： - -- **本工作的计算使用了上海交通大学天文系Gravity高性能计算集群** -- **This work made use of the Gravity Supercomputer at the Department of Astronomy, Shanghai Jiao Tong University.** - -- [List of Publications powered by Gravity](https://ui.adsabs.harvard.edu/search/q=%3D%20full%3A%22gravity%20supercomputer%22&sort=date%20desc%2C%20bibcode%20desc&p_=0) - - +## [Gravity Documentation](https://gravity-doc.github.io) + This is the documentation for the **Gravity Cluster** of [Department of Astronomy (DOA)](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). + +```tip +### Welcome to contribute +This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. Contributing is pretty simple as following: +1. **Click**. To contribute, click the **[edit](https://github.com/gravity-doc/gravity-doc.github.io/edit/master/docs/index.md)** :pencil: on the upper right of the page you feel like editing. Every document page is **editable**. It will bring you to the source files on github. You may will need to login to github to proceed. Once logged-in, a fork of the document repository will be created under your own account, so that you can edit it in whatever way you like. +2. **Edit**. Edit the page to contribute useful stuff. Check [here](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github/basic-writing-and-formatting-syntax) or the existing source pages for examples in the syntax. [This page](https://rundocs.io/writng/) contains theme-related syntax. +3. **Commit**. Finish the editing by leaving a commit message, describing what you have done. +4. **Submit**. Submit a **pull request** so that your changes will be submitted to the upstream branch (a.k.a., this website). The administrators will review your changes and absorb them into the website. + +Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. +### **中文内容也欢迎！🥳** +``` +## Contact + +You can contact us via + +- [gravity-hpc@sjtu.edu.cn ](mailto:gravity-hpc@sjtu.edu.cn) 📧 +- **WeChat** group 💬 +- Administrators: Yihe Wang, Yu Yu, Jiaxin Han + +if you need + +1. create an **account** 🙋‍♂️🙋‍♀️ +2. change **password** 🔐 +3. **extend time** of your job ⌛ +4. other problems ❔ + +## Acknowledging Gravity Cluster + +We would appreciate it if all publications that include results generated using *Gravity* cluster include a short statement in the Acknowledgment section. For example: + +若您在论文中使用了Gravity得到的结果，希望您能够在致谢中提到**Gravity**。例如： + +- **本工作的计算使用了上海交通大学天文系Gravity高性能计算集群** +- **This work made use of the Gravity Supercomputer at the Department of Astronomy, Shanghai Jiao Tong University.** + +- [List of Publications powered by Gravity](https://ui.adsabs.harvard.edu/search/q=%3D%20full%3A%22gravity%20supercomputer%22&sort=date%20desc%2C%20bibcode%20desc&p_=0) + + From 735d299e39f590be66c1bf5439c7b0530e5ea595 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 14:12:27 +0800 Subject: [PATCH 04/23] fix "process_markdown: command not found" --- convert_to_docsify.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index 3405d8b..5921490 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -31,6 +31,8 @@ process_markdown() { sed -i 's||![\1](\0)|g' "$1" } +export -f process_markdown + # Process markdown files using find and process_markdown function find . -type f -name "*.md" -exec bash -c 'process_markdown "$0"' {} \; From 2084e87296f3048b2bdb940ff9d5b8094358657f Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 15:13:13 +0800 Subject: [PATCH 05/23] fix convert script, add convert danger/tip/note --- convert_to_docsify.sh | 40 +++++++++++++++++++++++++++++++++++++++- docs/index.html | 2 +- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index 5921490..cd1796c 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -26,9 +26,47 @@ fi process_markdown() { echo "Processing $1" # remove part of "--- ... ---" - sed -i '/^---$/,/^---$/d' "$1" + sed -i '/^---[[:space:]]*$/,/^---[[:space:]]*$/d' "$1" # convert HTML image to markdown image sed -i 's||![\1](\0)|g' "$1" + # Convert danger, warning, note, and tip blocks + awk '/^```danger$/,/^```$/ { + if ($0 == "```danger```") { + print " ?> **Important**" + next + } + if ($0 != "```") { + print " >", substr($0, 5) + } + } + /^```warning$/,/^```$/ { + if ($0 == "```warning```") { + print " ?> **Important**" + next + } + if ($0 != "```") { + print " >", substr($0, 5) + } + } + /^```note$/,/^```$/ { + if ($0 == "```note```") { + print " ?> **Important**" + next + } + if ($0 != "```") { + print " >", substr($0, 5) + } + } + /^```tip$/,/^```$/ { + if ($0 == "```tip```") { + print " !> **Tip**" + next + } + if ($0 != "```") { + print " >", substr($0, 5) + } + } + { print }' "$1" > "$1.tmp" && mv "$1.tmp" "$1" } export -f process_markdown diff --git a/docs/index.html b/docs/index.html index 295dd79..d0ebdbd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -25,7 +25,7 @@ subMaxLevel: 2, name: 'Gravity Documentation', repo: 'https://github.com/gravity-doc/gravity-doc.github.io/', - logo: './images/logo_DOA_mini.png', + logo: 'https://git.gravity.sjtu.edu.cn/gravity-hpc/gravity-doc/-/raw/master/docs/images/logo_DOA_mini.png', search: { paths: 'auto', placeholder: 'Search / 搜索关键词', From 68a28e8131ec7860f7574967d7341965a6cf4b49 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 15:53:44 +0800 Subject: [PATCH 06/23] fix image convert HTML to markdown --- convert_to_docsify.sh | 44 ++++++------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index cd1796c..3847475 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -28,45 +28,13 @@ process_markdown() { # remove part of "--- ... ---" sed -i '/^---[[:space:]]*$/,/^---[[:space:]]*$/d' "$1" # convert HTML image to markdown image - sed -i 's||![\1](\0)|g' "$1" + sed -i 's|]*>|![\2](\1/\2)|g' "$1" + # Convert danger, warning, note, and tip blocks - awk '/^```danger$/,/^```$/ { - if ($0 == "```danger```") { - print " ?> **Important**" - next - } - if ($0 != "```") { - print " >", substr($0, 5) - } - } - /^```warning$/,/^```$/ { - if ($0 == "```warning```") { - print " ?> **Important**" - next - } - if ($0 != "```") { - print " >", substr($0, 5) - } - } - /^```note$/,/^```$/ { - if ($0 == "```note```") { - print " ?> **Important**" - next - } - if ($0 != "```") { - print " >", substr($0, 5) - } - } - /^```tip$/,/^```$/ { - if ($0 == "```tip```") { - print " !> **Tip**" - next - } - if ($0 != "```") { - print " >", substr($0, 5) - } - } - { print }' "$1" > "$1.tmp" && mv "$1.tmp" "$1" + sed -i -e '/^```[[:space:]]*danger[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*danger[[:space:]]*$/!>\*\*Important\*\*\n/; t; }' \ + -e '/^```[[:space:]]*warning[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*warning[[:space:]]*$/!>\*\*Important\*\*\n/; t; }' \ + -e '/^```[[:space:]]*note[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*note[[:space:]]*$/?>\*\*Important\*\*\n/; t; }' \ + -e '/^```[[:space:]]*tip[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*tip[[:space:]]*$/?>\*\*Tip\*\*\n/; t; }' "$1" } export -f process_markdown From da9e09b4475e92c88567288511bcb2da54d0d863 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 16:45:24 +0800 Subject: [PATCH 07/23] try to fix convert helper message --- convert_to_docsify.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index 3847475..b195c77 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -25,16 +25,24 @@ fi # Function to process markdown files process_markdown() { echo "Processing $1" - # remove part of "--- ... ---" + # Remove part of "--- ... ---" sed -i '/^---[[:space:]]*$/,/^---[[:space:]]*$/d' "$1" - # convert HTML image to markdown image + + # Convert HTML image to markdown image sed -i 's|]*>|![\2](\1/\2)|g' "$1" - + # Convert danger, warning, note, and tip blocks - sed -i -e '/^```[[:space:]]*danger[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*danger[[:space:]]*$/!>\*\*Important\*\*\n/; t; }' \ - -e '/^```[[:space:]]*warning[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*warning[[:space:]]*$/!>\*\*Important\*\*\n/; t; }' \ - -e '/^```[[:space:]]*note[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*note[[:space:]]*$/?>\*\*Important\*\*\n/; t; }' \ - -e '/^```[[:space:]]*tip[[:space:]]*$/,/^```[[:space:]]*$/ { s/^```[[:space:]]*tip[[:space:]]*$/?>\*\*Tip\*\*\n/; t; }' "$1" + sed -i -E -e '/^```(danger|warning|note|tip)[[:space:]]*$/,/^```[[:space:]]*$/ { + /^```(danger|warning|note|tip)[[:space:]]*$/! { + s/^[>-][[:space:]]*// + s/^[[:digit:]]+\.*// + /^$/d + } + s/^```(danger|warning)[[:space:]]*$/!> \*\*Important\*\*⚠️/ + s/^```(note|tip)[[:space:]]*$/?> \*\*Tip\*\*💡/ + s/```[[:space:]]*$// + t + }' "$1" } export -f process_markdown From ccb08ed11ae332929865cc40bccc9d811eb1512d Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 17:06:41 +0800 Subject: [PATCH 08/23] change cover menu, add copy code/edit on github --- docs/_coverpage.md | 5 +++-- docs/_navbar.md | 1 - docs/index.html | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 30f009a..4d3119c 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,4 +1,5 @@ - + + +[Gravity Home](https://jupyter.gravity.sjtu.edu.cn/) -[GitHub](https://github.com/gravity-doc/gravity-doc.github.io) [Get started](/README.md) diff --git a/docs/_navbar.md b/docs/_navbar.md index a6cbb16..f85dafa 100644 --- a/docs/_navbar.md +++ b/docs/_navbar.md @@ -1,3 +1,2 @@ - [Cheatsheet](/QuickStart?id=cheatsheet-📜) -- [Gravity Homepage](https://jupyter.gravity.sjtu.edu.cn/) - [Department of Astronomy (SJTU)](http://astro.sjtu.edu.cn/) \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index d0ebdbd..1b61308 100644 --- a/docs/index.html +++ b/docs/index.html @@ -38,7 +38,10 @@ fontsize:'0.9em', color:'rgb(90,90,90)', language:'chinese' - } + }, + plugins: [ + EditOnGithubPlugin.create(docBase, docEditBase, title) + ] } @@ -48,6 +51,12 @@ + + + + + + From 396eb5c53844d3d050017a4c48fe26c249fdb6f6 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 17:20:16 +0800 Subject: [PATCH 09/23] Revert "change cover menu, add copy code/edit on github" This reverts commit ccb08ed11ae332929865cc40bccc9d811eb1512d. --- docs/_coverpage.md | 5 ++--- docs/_navbar.md | 1 + docs/index.html | 11 +---------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 4d3119c..30f009a 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,5 +1,4 @@ - - -[Gravity Home](https://jupyter.gravity.sjtu.edu.cn/) + +[GitHub](https://github.com/gravity-doc/gravity-doc.github.io) [Get started](/README.md) diff --git a/docs/_navbar.md b/docs/_navbar.md index f85dafa..a6cbb16 100644 --- a/docs/_navbar.md +++ b/docs/_navbar.md @@ -1,2 +1,3 @@ - [Cheatsheet](/QuickStart?id=cheatsheet-📜) +- [Gravity Homepage](https://jupyter.gravity.sjtu.edu.cn/) - [Department of Astronomy (SJTU)](http://astro.sjtu.edu.cn/) \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 1b61308..d0ebdbd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -38,10 +38,7 @@ fontsize:'0.9em', color:'rgb(90,90,90)', language:'chinese' - }, - plugins: [ - EditOnGithubPlugin.create(docBase, docEditBase, title) - ] + } } @@ -51,12 +48,6 @@ - - - - - - From 6892d8cbe5e218f8280fb55881b92e6e147b53f3 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 17:23:07 +0800 Subject: [PATCH 10/23] remove github address --- docs/_coverpage.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 30f009a..243b9f0 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,4 +1,3 @@ -[GitHub](https://github.com/gravity-doc/gravity-doc.github.io) [Get started](/README.md) From 2944b4d61175c07c2ee96b03dd38ce296a296943 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 17:59:57 +0800 Subject: [PATCH 11/23] fix: convert script, delete blank line --- convert_to_docsify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index b195c77..916d9e5 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -36,7 +36,7 @@ process_markdown() { /^```(danger|warning|note|tip)[[:space:]]*$/! { s/^[>-][[:space:]]*// s/^[[:digit:]]+\.*// - /^$/d + /^[[:space:]]*$/d } s/^```(danger|warning)[[:space:]]*$/!> \*\*Important\*\*⚠️/ s/^```(note|tip)[[:space:]]*$/?> \*\*Tip\*\*💡/ From 4fd94a45cdd9c725a3c76f9072fc9fe94beb23d9 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 18:00:44 +0800 Subject: [PATCH 12/23] update link --- README.md | 76 +- docs/Advanced/X11_Tunnel.md | 114 +- docs/Advanced/jupyter_notebook.md | 2 +- docs/Basic/Account.md | 162 +- docs/Basic/Coding.md | 4 +- docs/Basic/Data_Transfer.md | 238 +-- docs/Basic/Job.md | 1548 +++++++++---------- docs/Basic/Login.md | 4 +- docs/Basic/Resource_Monitor.md | 351 ++--- docs/MISC/FAQ.md | 226 +-- docs/MISC/Issues.md | 196 +-- docs/Policy.md | 3 +- docs/QuickStart.md | 282 ++-- docs/README.md | 2 +- docs/Software/User_Software_Installation.md | 174 +-- docs/Updates.md | 133 +- 16 files changed, 1772 insertions(+), 1743 deletions(-) diff --git a/README.md b/README.md index be173f7..8dcf58f 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,39 @@ -## Gravity Documentation -[上海交通大学天文系Gravity集群文档](https://gravity-doc.github.io) - -This is the documentation for the **Gravity Cluster** of [Department of Astronomy](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). - -### Welcome to contribute - This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. - - Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. - - **中文内容也欢迎！🥳** - - -### Contact - -You can contact us via - -- [gravity-hpc@sjtu.edu.cn](mailto:gravity-hpc@sjtu.edu.cn) -- **WeChat group** - -if you need - -1. create an **account** 🙋‍♂️🙋‍♀️ -2. change **password** 🔐 -3. **extend time** of your job ⌛ -4. other problems ❔ - -### Acknowledging the Gravity Cluster - -We would appreciate it if all publications that include results generated using **Gravity** cluster include a short statement in the Acknowledgment section. For example: - -若您在论文中使用了Gravity得到的结果，希望您能够在致谢中提到**Gravity**。例如： - -- **本工作的计算使用了上海交通大学天文系Gravity高性能计算集群** -- **This work made use of the Gravity Supercomputer at the Department of Astronomy, Shanghai Jiao Tong University.** - - +## Gravity Documentation + +[上海交通大学天文系Gravity集群文档](https://gravity-doc.github.io) + +This is the documentation for the **Gravity Cluster** of [Department of Astronomy](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). + +### Welcome to contribute + + This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. + + Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. + + **中文内容也欢迎！🥳** + + +### Contact + +You can contact us via + +- [gravity-hpc@sjtu.edu.cn](mailto:gravity-hpc@sjtu.edu.cn) +- **WeChat group** + +if you need + +1. create an **account** 🙋‍♂️🙋‍♀️ +2. change **password** 🔐 +3. **extend time** of your job ⌛ +4. other problems ❔ + +### Acknowledging the Gravity Cluster + +We would appreciate it if all publications that include results generated using **Gravity** cluster include a short statement in the Acknowledgment section. For example: + +若您在论文中使用了Gravity得到的结果，希望您能够在致谢中提到**Gravity**。例如： + +- **本工作的计算使用了上海交通大学天文系Gravity高性能计算集群** +- **This work made use of the Gravity Supercomputer at the Department of Astronomy, Shanghai Jiao Tong University.** + + diff --git a/docs/Advanced/X11_Tunnel.md b/docs/Advanced/X11_Tunnel.md index 0a291bb..a144a65 100644 --- a/docs/Advanced/X11_Tunnel.md +++ b/docs/Advanced/X11_Tunnel.md @@ -1,57 +1,57 @@ ---- -sort: 1 -title: X11 Tunnel ---- - -# X11 Tunnel -```note -Running graphical software on Gravity with [X11 tunneling](https://en.wikipedia.org/wiki/X_Window_System) -Actually, sometimes using **[VNC](https://gravity-doc.github.io/Advanced/VNC.html)** is better 😜 -X11 allows you to run a **graphical application** on the *login* or *computing node*, and to control it from your local computer. For example, you can **view images** in the *Gravity* using -- `eog` -- `display` - -Or, you can even run a browser -- `firefox` -``` - -## Windows - -For Windows users, we recommend you to use **[MobaXTerm](https://mobaxterm.mobatek.net/download.html)**. It comes with built-in X11 support, so you don't need to do any extra steps. Just [login Gravity](https://gravity-doc.github.io/Basic/Login.html). Once you are connected, you can connect to an interactive compute node using the -X switch, for example 🌰 - -```bash -qsub -I -X -v DISPLAY -l nodes=1:ppn=72,mem=1gb,walltime=12:00:00 -q normal -``` - -Then, you can run your graphical application from the computing node 🎉. - -## Mac - -Mac users would need to install the program called [XQuartz](https://www.xquartz.org). After installation, double-click on the XQuartz icon to start it (it usually installs into `Applications` -> `Utilities`). Then, in a terminal window - -```bash -ssh -Y -i @gravity.sjtu.edu.cn -``` - -Once you are logged in, you can connect to an interactive compute node using the -X switch, for example - -```bash -qsub -I -X -v DISPLAY -l nodes=1:ppn=36,mem=100gb,walltime=12:00:00 -q normal -``` - -Then, you can run your graphical application from the computing node 🎉. - -## Linux -Just use `ssh` to log in! - -```bash -ssh -X -i @gravity.sjtu.edu.cn -``` - -Once you are connected, you can connect to an interactive compute node using the -X switch, for example - -```bash -qsub -I -X -v DISPLAY -l nodes=1:ppn=36,mem=100gb,walltime=12:00:00 -q normal -``` - -Then, you can run your graphical application from the computing node 🎉. +--- +sort: 1 +title: X11 Tunnel +--- + +# X11 Tunnel +```note +Running graphical software on Gravity with [X11 tunneling](https://en.wikipedia.org/wiki/X_Window_System) +Actually, sometimes using **[VNC](/Advanced/VNC)** is better 😜 +X11 allows you to run a **graphical application** on the *login* or *computing node*, and to control it from your local computer. For example, you can **view images** in the *Gravity* using +- `eog` +- `display` + +Or, you can even run a browser +- `firefox` +``` + +## Windows + +For Windows users, we recommend you to use **[MobaXTerm](https://mobaxterm.mobatek.net/download.html)**. It comes with built-in X11 support, so you don't need to do any extra steps. Just [login Gravity](/Basic/Login). Once you are connected, you can connect to an interactive compute node using the -X switch, for example 🌰 + +```bash +qsub -I -X -v DISPLAY -l nodes=1:ppn=72,mem=1gb,walltime=12:00:00 -q normal +``` + +Then, you can run your graphical application from the computing node 🎉. + +## Mac + +Mac users would need to install the program called [XQuartz](https://www.xquartz.org). After installation, double-click on the XQuartz icon to start it (it usually installs into `Applications` -> `Utilities`). Then, in a terminal window + +```bash +ssh -Y -i @gravity.sjtu.edu.cn +``` + +Once you are logged in, you can connect to an interactive compute node using the -X switch, for example + +```bash +qsub -I -X -v DISPLAY -l nodes=1:ppn=36,mem=100gb,walltime=12:00:00 -q normal +``` + +Then, you can run your graphical application from the computing node 🎉. + +## Linux +Just use `ssh` to log in! + +```bash +ssh -X -i @gravity.sjtu.edu.cn +``` + +Once you are connected, you can connect to an interactive compute node using the -X switch, for example + +```bash +qsub -I -X -v DISPLAY -l nodes=1:ppn=36,mem=100gb,walltime=12:00:00 -q normal +``` + +Then, you can run your graphical application from the computing node 🎉. diff --git a/docs/Advanced/jupyter_notebook.md b/docs/Advanced/jupyter_notebook.md index 83c89bc..f99fc2a 100644 --- a/docs/Advanced/jupyter_notebook.md +++ b/docs/Advanced/jupyter_notebook.md @@ -5,7 +5,7 @@ title: Jupyter Lab/Notebook ```tip We recommend you to use [*JupyterHub*](/Basic/Coding.html), it's more easier and convenient 😜 -You can also **use your own environment** in [*JupyterHub*](https://gravity-doc.github.io/Basic/Coding.html) +You can also **use your own environment** in [*JupyterHub*](/Basic/Coding?id=create-new-environmentkernel) Here, we introduce how to connect *jupyter-notebook/lab* on *Gravity* (login or computing nodes) 👇 ``` diff --git a/docs/Basic/Account.md b/docs/Basic/Account.md index 5af55ff..d445bff 100644 --- a/docs/Basic/Account.md +++ b/docs/Basic/Account.md @@ -1,81 +1,83 @@ ---- -sort: 1 -title: Account ---- - -## Apply for a NEW Account -```note -请仔细阅读[用户协议](https://gravity-doc.github.io/Policy.html)，并征求导师同意。 - -Please make sure you have read the [user policy](https://gravity-doc.github.io/Policy.html), and get consent of your sponsor/supervisor before application. -``` - -Use [👉**this link**👈](https://forms.office.com/Pages/ResponsePage.aspx?id=-f5HFYhWBkCG2kSQ-Sc_lW_CRAlVS3tEtz1OEMF6VRNUMUNLOUVOSFhSMTJSTzJSUVozQldJVlRDUy4u) to fill out the form, we will send an e-mail📧 to you after a while. - -## Change password - -You will receive an email once your account is created. -The initial password is included. -1. In terminal, you can use `yppasswd` or `passwd` to change your password. -2. In web, you can change password by clicking *Reset password?* to change your password. More details are here 👉 [**2FA Authentication**](https://gravity-doc.github.io/Basic/Login.html#web-login) - - -```tip -- Note that it may take a few minutes before the password change is synchronized between the terminal and the web login. - -- 更改密码后，新的密码将会需要几分钟在terminal和web之间得到同步。 -``` - -```warning -- Please change your password immediately the first time you login to your account. -- Never write your password explicitly to others, in emails or chats. The administrators will never ask you for your password and never give your password to anyone. - -- 请在第一次登陆后立即修改初始密码。 -- 请不要在邮件、微信等通信里向任何人提供密码。管理员维护账号也从**不会**需要你的密码。 -``` - -## Forgot password - -If you forget your password, don't worry! - -1. click *Reset password?* on [authentication webpage](https://gravity.sjtu.edu.cn/auth/), you will receive an email📧, so you can reset your password instantly⚡! -2. [**send email📧 to us**](mailto:gravity-hpc@sjtu.edu.cn), provide your username, we will help you to reset the password, maybe it will take several hours🐢. - -若忘记了密码，不要担心！ -1. 点击[认证页面](https://gravity.sjtu.edu.cn/auth/)的*重置密码*，你会收到一封邮件，即刻重置密码⚡! -2. 请发送邮件至[hpc邮箱](mailto:gravity-hpc@sjtu.edu.cn)，在邮件中注明您要重置密码的账号。如果您不是账号申请人，为了确认您的身份，请务必抄送账号申请人的邮箱。我们会在确保本次重置并非恶意重置之后完成密码重置工作，并发送邮件通知您新的密码，这可能会需要若干个小时🐢。 - -## Password specification - -如果您要自行更改密码，我们建议您遵循如下规则制定新的密码： - -1. 不要使用固定的常用密码 -2. 不要使用有意义的字符串 -3. 密码长度至少8位 -4. 密码中至少包含字母和数字 - -## Change Email📧 -If you want to change your email📧, please [contact us](https://gravity-doc.github.io/#contact), provide your **username** and **NEW email**. - -## SGI Account - -```tip -1. **SGI** is a **standalone** machine, and it has a different OS and disk system. -2. **SGI** use *slurm* rather than *pbs* as the job scheduler. - -**NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. -``` - -You cannot access **SGI** directly, you need to log in **Gravity** first, then on login nodes, use: - -```bash -ssh sgi -``` - -Each account has a default quota of -- **10GB** at `/home/username` -- **1TB** at `/mnt/ddnfs/data_users/username` - -Gravity **Home Directory** is mounted at `/gravity/home` - +--- +sort: 1 +title: Account +--- + +## Apply for a NEW Account +```note +请仔细阅读[用户协议](/Policy)，并征求导师同意。 + +Please make sure you have read the [user policy](/Policy), and get consent of your sponsor/supervisor before application. +``` + +Use [👉**this link**👈](https://forms.office.com/Pages/ResponsePage.aspx?id=-f5HFYhWBkCG2kSQ-Sc_lW_CRAlVS3tEtz1OEMF6VRNUMUNLOUVOSFhSMTJSTzJSUVozQldJVlRDUy4u) to fill out the form, we will send an e-mail📧 to you after a while. + +## Change password + +You will receive an email once your account is created. +The initial password is included. + +1. In terminal, you can use `yppasswd` or `passwd` to change your password. +2. In web, you can change password by clicking *Reset password?* to change your password. More details are here 👉 [**2FA Authentication**](/Basic/Login?id=web-login) + + +```tip +- Note that it may take a few minutes before the password change is synchronized between the terminal and the web login. + +- 更改密码后，新的密码将会需要几分钟在terminal和web之间得到同步。 +``` + +```warning +- Please change your password immediately the first time you login to your account. +- Never write your password explicitly to others, in emails or chats. The administrators will never ask you for your password and never give your password to anyone. + +- 请在第一次登陆后立即修改初始密码。 +- 请不要在邮件、微信等通信里向任何人提供密码。管理员维护账号也从**不会**需要你的密码。 +``` + +## Forgot password + +If you forget your password, don't worry! + +1. click *Reset password?* on [authentication webpage](https://gravity.sjtu.edu.cn/auth/), you will receive an email📧, so you can reset your password instantly⚡! +2. [**send email📧 to us**](mailto:gravity-hpc@sjtu.edu.cn), provide your username, we will help you to reset the password, maybe it will take several hours🐢. + +若忘记了密码，不要担心！ +1. 点击[认证页面](https://gravity.sjtu.edu.cn/auth/)的*重置密码*，你会收到一封邮件，即刻重置密码⚡! +2. 请发送邮件至[hpc邮箱](mailto:gravity-hpc@sjtu.edu.cn)，在邮件中注明您要重置密码的账号。如果您不是账号申请人，为了确认您的身份，请务必抄送账号申请人的邮箱。我们会在确保本次重置并非恶意重置之后完成密码重置工作，并发送邮件通知您新的密码，这可能会需要若干个小时🐢。 + +## Password specification + +如果您要自行更改密码，我们建议您遵循如下规则制定新的密码： + +1. 不要使用固定的常用密码 +2. 不要使用有意义的字符串 +3. 密码长度至少8位 +4. 密码中至少包含字母和数字 + +## Change Email📧 +If you want to change your email📧, please [contact us](/?id=contact), provide your **username** and **NEW email**. + +## SGI Account + +```tip +1. **SGI** is a **standalone** machine, and it has a different OS and disk system. +2. **SGI** use *slurm* rather than *pbs* as the job scheduler. + +**NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. +``` + +You cannot access **SGI** directly, you need to log in **Gravity** first, then on login nodes, use: + +```bash +ssh sgi +``` + +Each account has a default quota of + +- **10GB** at `/home/username` +- **1TB** at `/mnt/ddnfs/data_users/username` + +Gravity **Home Directory** is mounted at `/gravity/home` + You can use `quota -ls` to check your disk quota and usage. \ No newline at end of file diff --git a/docs/Basic/Coding.md b/docs/Basic/Coding.md index 37215d5..8eb8dd3 100644 --- a/docs/Basic/Coding.md +++ b/docs/Basic/Coding.md @@ -12,7 +12,7 @@ title: Coding You can access *Jupyter* on our website: - [Jupyter](https://jupyter.gravity.sjtu.edu.cn) (needs **2FA**) -- PS. We have enabled **Two-Factor Authentication (2FA)**🎉🎉🎉 More details are [Gravity Authentication](https://gravity-doc.github.io/Basic/Login.html#web-login) +- PS. We have enabled **Two-Factor Authentication (2FA)**🎉🎉🎉 More details are [Gravity Authentication](/Basic/Login?id=web-login) ``` ```warning @@ -36,7 +36,7 @@ click *Jupyter* button ![login](../images/Basic/index.png) input *username* + *password* + *TOTP* -[more details about authentication](https://gravity-doc.github.io/Basic/Login.html#web-login) +[more details about authentication](/Basic/Login?id=web-login) ![login](../images/Basic/auth_login_2fa.png) diff --git a/docs/Basic/Data_Transfer.md b/docs/Basic/Data_Transfer.md index 6809065..ea7d7d0 100644 --- a/docs/Basic/Data_Transfer.md +++ b/docs/Basic/Data_Transfer.md @@ -1,119 +1,119 @@ ---- -sort: 5 -title: Data Transfer ---- - -# Data Transfer - -It is a common task to transfer files between different servers or your own desktop. -For transmission of large files, command lines like `scp` or `rsync` are usually recommended. -When one is making plots or editing codes, it might be more convenient to use -techniques with graphical user interface support. - -## FileServer -```note -You need to `copy`/`move` your files into `~/.fileServer` to use *FileServer*. - -If you need to **share massive data**, feel free to [contact admin🧙‍](https://gravity-doc.github.io/#contact) - -This service is provided for sharing data on gravity to your collaborators. Please do not abuse it to share your personal/non-academic files. -``` - -### access -We have enabled a **File Server** on Gravity 🎉🎉🎉 -You can access *fileserver* via: -- https://jupyter.gravity.sjtu.edu.cn/file/ -- https://gravity.sjtu.edu.cn/file/ - -Cause we enable **2FA**, you need to install *Authenticator App* at the first time. See [more details here](https://gravity-doc.github.io/Basic/Login.html#web-login). - -### properties -1. **share file** 🎉🎉🎉 (you can set **password/limiting time** of your sharing files) - -![share file](../images/Basic/data-share.png) - -2. download file (you can use download tools, such as **multi-thread** downloader) - -![download](../images/Basic/data-download.png) - -3. view files (picture/video/txt) - -PS. If you need to create NEW file, please use *terminal* to `remove/copy` to `~/.fileServer`, symlink `ln -s` is not allowed here❌! -## Jupyter-lab/notebook -Because of the GUI of jupyter, you can easily transfer your data, as long as your data is not too large (eg. <10G) -> 1. login [JupyterHub](https://gravity.sjtu.edu.cn/) -> 2. start a server -> 3. **drag** your files (download or upload) - -![image-20210801101248053](../images/Basic/DataTransfer-jupyterlab.png) - -## Filezilla -[安装 Filezilla 软件](https://filezilla-project.org/)后，可以双击打开该软件，按照下图进行设置，完成后单击快速链接或者回车可以登录ftp 站点。 -输入数据集群IP 用户名 密码和端口号，如下图所示： - - - -## scp and rsync -Linux/Unix/Mac 用户可以使用命令行工具在集群和您自己的计算机之间传输数据。下列语句会将文件夹`data/`上传到主文件夹的`tmp/`。 - -```bash -$ scp -r data/ YOUR_USERNAME@TARGET_IP:tmp/ -``` - -下列语句会将主文件夹的data.out下载到本地当前工作目录中。 - -```bash -$ scp YOUR_USERNAME@TARGET_IP:data.out ./ -``` - -In a word 👇 - -```bash -# copy local file to Gravity -scp -i your_ssh_key ~/local_file username@gravity.sjtu.edu.cn:/home/username/remote_file -rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file -# copy Gravity file to local -scp -i your_ssh_key username@gravity.sjtu.edu.cn:remote_file local_file -rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file -``` - -## SFTP - -Gravity provides file access via SFTP. -One can use any file manager or FTP client that supports SFTP. - -For example, one can open "dolphin" (file manager) on Linux and enter the following path in the location bar - -```bash -sftp://username@gravity.sjtu.edu.cn/home/username -# replace `username` with your own user name -``` - -The system will mount it as a remote directory. -One can use this directory just as local directories. - -If you are too lazy to enter password at every mounting, you can copy his SSH key to the server - -```bash -ssh-copy-id username@gravity.sjtu.edu.cn -``` - - -## SSHFS - -Another popular way to mount a remote directory is to use SSHFS. -One can create a directory in his computer, e.g., - -```bash -mkdir ~/mnt/gravity -``` - -and mount the server as following, - -```bash -sshfs username@gravity.sjtu.edu.cn/home/username ~/mnt/gravity -o auto_cache,reconnect,follow_symlinks -o Compression=no -o Ciphers=aes128-ctr -``` - -One may change these options for his own purpose. - -It seems SFTP has better performance than SSHFS. +--- +sort: 5 +title: Data Transfer +--- + +# Data Transfer + +It is a common task to transfer files between different servers or your own desktop. +For transmission of large files, command lines like `scp` or `rsync` are usually recommended. +When one is making plots or editing codes, it might be more convenient to use +techniques with graphical user interface support. + +## FileServer +```note +You need to `copy`/`move` your files into `~/.fileServer` to use *FileServer*. + +If you need to **share massive data**, feel free to [contact admin🧙‍](/?id=contact) + +This service is provided for sharing data on gravity to your collaborators. Please do not abuse it to share your personal/non-academic files. +``` + +### access +We have enabled a **File Server** on Gravity 🎉🎉🎉 +You can access *fileserver* via: +- https://jupyter.gravity.sjtu.edu.cn/file/ +- https://gravity.sjtu.edu.cn/file/ + +Cause we enable **2FA**, you need to install *Authenticator App* at the first time. See [more details here](/Basic/Login?id=web-login). + +### properties +1. **share file** 🎉🎉🎉 (you can set **password/limiting time** of your sharing files) + +![share file](../images/Basic/data-share.png) + +2. download file (you can use download tools, such as **multi-thread** downloader) + +![download](../images/Basic/data-download.png) + +3. view files (picture/video/txt) + +PS. If you need to create NEW file, please use *terminal* to `remove/copy` to `~/.fileServer`, symlink `ln -s` is not allowed here❌! +## Jupyter-lab/notebook +Because of the GUI of jupyter, you can easily transfer your data, as long as your data is not too large (eg. <10G) +> 1. login [JupyterHub](https://gravity.sjtu.edu.cn/) +> 2. start a server +> 3. **drag** your files (download or upload) + +![image-20210801101248053](../images/Basic/DataTransfer-jupyterlab.png) + +## Filezilla +[安装 Filezilla 软件](https://filezilla-project.org/)后，可以双击打开该软件，按照下图进行设置，完成后单击快速链接或者回车可以登录ftp 站点。 +输入数据集群IP 用户名 密码和端口号，如下图所示： + + + +## scp and rsync +Linux/Unix/Mac 用户可以使用命令行工具在集群和您自己的计算机之间传输数据。下列语句会将文件夹`data/`上传到主文件夹的`tmp/`。 + +```bash +$ scp -r data/ YOUR_USERNAME@TARGET_IP:tmp/ +``` + +下列语句会将主文件夹的data.out下载到本地当前工作目录中。 + +```bash +$ scp YOUR_USERNAME@TARGET_IP:data.out ./ +``` + +In a word 👇 + +```bash +# copy local file to Gravity +scp -i your_ssh_key ~/local_file username@gravity.sjtu.edu.cn:/home/username/remote_file +rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file +# copy Gravity file to local +scp -i your_ssh_key username@gravity.sjtu.edu.cn:remote_file local_file +rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file +``` + +## SFTP + +Gravity provides file access via SFTP. +One can use any file manager or FTP client that supports SFTP. + +For example, one can open "dolphin" (file manager) on Linux and enter the following path in the location bar + +```bash +sftp://username@gravity.sjtu.edu.cn/home/username +# replace `username` with your own user name +``` + +The system will mount it as a remote directory. +One can use this directory just as local directories. + +If you are too lazy to enter password at every mounting, you can copy his SSH key to the server + +```bash +ssh-copy-id username@gravity.sjtu.edu.cn +``` + + +## SSHFS + +Another popular way to mount a remote directory is to use SSHFS. +One can create a directory in his computer, e.g., + +```bash +mkdir ~/mnt/gravity +``` + +and mount the server as following, + +```bash +sshfs username@gravity.sjtu.edu.cn/home/username ~/mnt/gravity -o auto_cache,reconnect,follow_symlinks -o Compression=no -o Ciphers=aes128-ctr +``` + +One may change these options for his own purpose. + +It seems SFTP has better performance than SSHFS. diff --git a/docs/Basic/Job.md b/docs/Basic/Job.md index 27dd85e..6f22abf 100644 --- a/docs/Basic/Job.md +++ b/docs/Basic/Job.md @@ -1,774 +1,774 @@ ---- -sort: 3 -title: Job ---- - -```warning -You **should** run your job via PBS (Torque). -You can **ONLY** do some **simple** jobs on the **login nodes**, otherwise, you might crash the system and be *banned*. - -Please only request resources that you actually need. **Do not** use more than or less than the number of cores you requested. -``` - -## Queues - -`qstat -q ` shows a summary of available queues. - -`qstat -f -Q` shows the detailed queue configurations, which are also listed below: - -- **Ordinary queues** - -| Queue name | Max nodes | Max time | Max memory per node | Priority | -| :--------: | :--------: | :------: | :-----------------: | :---------------------: | -| *debug* | 1 | 3 hours | 376 GB | High | -| *small* | 3 | 72 hours | 376 GB | Medium | -| *normal* | 18 | 48 hours | 376 GB | Low (**default queue**) | - -- **Special queues** - -| Queue name | Node | Max time | Max memory per node | Note | -| :--------: | :--------------------------: | :------: | :------------------: | :----------------------------------------------------------------------------------------------------------------------- | -| *gpu* | gr01
gr02
gr03
gr04 | 72 hours | 376 GB | **NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla A100 (80G)** | -| *fat* | gr35
gr36
fat01 | 72 hours | 754GB
3TB
6 TB | **large shared-memory** jobs (**mem > 360GB**) | - -## Basic example - - -### Serial job - -Suppose you have a program `test.f90` which is compiled as follows: - -```bash -module load gcc -gfortran test.f90 -o test.exe -``` - -A simply job submission script (`example.qsub`) is as follow. - -```bash -#!/bin/sh -#PBS -N testjob -#PBS -l nodes=1:ppn=1 -#PBS -q normal - -cd $PBS_O_WORKDIR - -### load the modules ### -module load gcc && echo $_ "LOADED" - -### RUN ### -./test.exe -``` - -To submit the job, - -```bash -qsub example.qsub -``` - -### parallel job -The content of an example *parallel* PBS script (`example.qsub`) is provided below: - -```bash -#!/bin/bash -#PBS -N My_job -#PBS -l nodes=1:ppn=72 -#PBS -q normal - -# run your own program!!! -cd $PBS_O_WORKDIR -mpirun -np 72 exec -``` - - -To submit the job, - -```bash -qsub example.qsub -``` - -```tip -Please only request resources that you actually need. In the above example, a full node (72 cores) is requested. However, if your job uses fewer cores, you should only request that many cores. For example -`#PBS -l nodes=1:ppn=8` -to request 8 cores on a single node. - -If your job uses OpenMP, you should set the number of threads to match your requested cores. For example, when requesting 8 cores for an OpenMP job, remember to also put the following in your job script before your final program line: -`export OMP_NUM_THREADS=8` -. - -If you are running a serial job, please only request **one** core: -`#PBS -l nodes=1:ppn=1` -``` - -## Complete example - -> job name is `My_job`. -> E-mail address is `lalala@sjtu.edu.cn`. -> need 1 node 72 cores per node -> need 360GB memory `mem=360gb ` -> need at most 48 hours -> make all of the output including error to `/home/lalala/code/my_output.log` - -```bash -#!/bin/bash -#PBS -N My_job -#PBS -m abe -#PBS -M lalala@sjtu.edu.cn -#PBS -l nodes=1:ppn=72 -#PBS -l mem=360gb -#PBS -l walltime=48:00:00 -#PBS -q normal -#PBS -j oe -#PBS -o /home/lalala/code/my_output.log - -echo running with ${PBS_NP} processes on ${PBS_NUM_NODES} nodes - -# run your own program!!! -cd $PBS_O_WORKDIR -mpirun -np 72 exec - -echo Done! -``` - -## More Examples -### Interactive job - -To request resources for doing something interactively, you can submit an interactive job. - -For example: - -```bash -qsub -l nodes=1:ppn=4,mem=8gb,walltime=12:00:00 -q normal -I -``` - -You will be logged onto the assigned node once the above job get running, and you can start to run your actual computation interactively. - -```warning -- Please use the system responsibly and only start an interactive job when necessary. - -- Do not use more resources than requested, otherwise you might affect other jobs or crash the node. - -- CPU time will be charged in the same way as normal jobs. Remember to log out of the computing node to end the interactive job. - -``` - -Reference: [https://www.msi.umn.edu/content/interactive-queue-use-qsub](https://www.msi.umn.edu/content/interactive-queue-use-qsub) - -```tip -The following PBS environment variable list is output by `env | grep PBS`, which maybe useful in your scripts. -``` - -```bash -PBS_VERSION=TORQUE-6.0.2 -PBS_JOBNAME=aaa -PBS_ENVIRONMENT=PBS_BATCH -PBS_O_WORKDIR=/home/user1 -PBS_TASKNUM=1 -PBS_O_HOME=/home/user1 -PBS_WALLTIME=25920000 -PBS_MOMPORT=15003 -PBS_GPUFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01gpu -PBS_O_QUEUE=normal -PBS_O_LOGNAME=user1 -PBS_O_LANG=en_US.utf8 -PBS_JOBCOOKIE=24CA13DFB269BB24B9E687EE94037901 -PBS_NODENUM=0 -PBS_NUM_NODES=1 -PBS_O_SHELL=/bin/bash -PBS_JOBID=1399.login01 -PBS_O_HOST=login02 -PBS_VNODENUM=0 -PBS_QUEUE=normal -PBS_O_MAIL=/var/spool/mail/user1 -PBS_O_SUBMIT_FILTER=/opt/tsce4/torque6/bin/submitfilter.sh -PBS_MICFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01mic -PBS_NP=1 -PBS_O_SERVER=login01 -PBS_NUM_PPN=1 -PBS_NODEFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01 -PBS_O_PATH=/usr/java/jre1.8.0_151/bin:/usr/java/jre1.8.0_151/bin:/opt/tsce4/maui/sbin:/opt/tsce4/maui/bin:/opt/tsce4/torque6/bin:/opt/tsce4/torque6/sbin:/usr/local/bin:/usr/local/bin:/usr/lib64/qt-3.3/bin:/home/user1/perl5/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/opt/pdsh-2.29/bin:/opt/pdsh-2.29/sbin:.:/home/user1/.local/bin:/home/user1/bin -``` - -### Job array -When you need to run a series of jobs with different arguments, it is pretty useful. Because you do not need a loop any more. You can find more arguments [here](http://docs.adaptivecomputing.com/torque/6-1-1/adminGuide/Content/topics/torque/commands/qsub.htm). -1. You can use `-t` in the shell - - *example.pbs* script 👇 - - ```bash - #!/bin/bash - #PBS -N My_job - #PBS -l nodes=1:ppn=1 - #PBS -q normal - - # run your own program!!! - python test.py $PBS_ARRAYID - ``` - - run it 👇 - - ```bash - qsub example.pbs -t 1-666 - ``` - -2. Another way is writing `-t` in the script - - *example.pbs* script 👇 - - ```bash - #!/bin/bash - #PBS -N My_job - #PBS -l nodes=1:ppn=1 - #PBS -q normal - #PBS -t 1-666%5 - - # run your own program!!! - python test.py $PBS_ARRAYID - ``` - - run it 👇 - - ```bash - qsub example.pbs - ``` - The optional `%5` in the `-t` parameter specifies a maximum of 5 jobs to be allowed running simultaneously. - -**`PBS_ARRAYID`** here represents the range `1-666`, which means your job will run as - -```bash -python test.py 1 -python test.py 2 -python test.py 3 -... -... -python test.py 666 -``` - -### Openmp job -This is a sample *Fortran90* program using *openmp* to parallelize a *do* loop. - -```fortran -program testopenmp -use omp_lib -implicit none - -integer(4)::num_threads -integer(4)::thread -integer(4)::i - -num_threads=omp_get_max_threads() -write(*,*) 'number of threads:',num_threads - -!$omp parallel do schedule(static) default(private) -do i=1,16 - thread=omp_get_thread_num() - write(*,*) 'loop',i,'is processed by thread',thread -enddo -!$omp end parallel do - -endprogram testopenmp -``` - -Use *Intel compiler* to compile. - -```bash -module load compiler/intel-2018 -ifort testopenmp.f90 -fopenmp -``` - -This sample job submission script is applying one node. -The number of threads is set to 8. - -```bash -#!/bin/sh -#PBS -N testopenmp -#PBS -l nodes=1:ppn=8 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} -module load compiler/intel-2018 && echo $_ "LOADED" -export OMP_NUM_THREADS=8 - -### RUN ### -echo "###############################################" -./a.out > log - -``` - -The result shows that the *do* loop is been parallelized by 8 threads. - -```bash - number of threads: 8 - loop 1 is processed by thread 0 - loop 3 is processed by thread 1 - loop 5 is processed by thread 2 - loop 7 is processed by thread 3 - loop 2 is processed by thread 0 - loop 9 is processed by thread 4 - loop 4 is processed by thread 1 - loop 6 is processed by thread 2 - loop 8 is processed by thread 3 - loop 11 is processed by thread 5 - loop 13 is processed by thread 6 - loop 10 is processed by thread 4 - loop 12 is processed by thread 5 - loop 15 is processed by thread 7 - loop 14 is processed by thread 6 - loop 16 is processed by thread 7 -``` - -### MPI job -This sample shows a *MPI* parallel program in *Fortran90*. - -```fortran -program testmpi -use mpi -implicit none - -integer(4)::rank,nrank -integer(4)::ierr -character(len=MPI_MAX_PROCESSOR_NAME)::nodename -integer(4)::namelen - -call mpi_init(ierr) -call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr) -call mpi_comm_size(MPI_COMM_WORLD,nrank,ierr) -call mpi_get_processor_name(nodename,namelen,ierr) - -write(*,'(a6,i4,a18,a6)') 'rank',rank,'is run on node',trim(nodename) - -call mpi_finalize(ierr) - -endprogram testmpi -``` - -Use *Intel compiler* and *IntelMPI* to compile. - -```bash -module load compiler/intel-2018 -module load mpi/intel-2018 -mpif90 testmpi.f90 -``` - -This sample script is applying 4 nodes and setting 2 processes per node. - -```bash -#!/bin/sh -#PBS -N testmpi -#PBS -l nodes=4:ppn=2 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} -module load compiler/intel-2018 && echo $_ "LOADED" -module load mpi/intel-2018 && echo $_ "LOADED" - -### RUN ### -echo "###############################################" -mpirun -np ${PBS_NP} ./a.out > log - -``` - -The result shows that *gr25*-*gr28* is used to finish the job, with 2 processes per node. - -```bash - rank 2 is run on node gr27 - rank 6 is run on node gr25 - rank 3 is run on node gr27 - rank 7 is run on node gr25 - rank 0 is run on node gr28 - rank 4 is run on node gr26 - rank 1 is run on node gr28 - rank 5 is run on node gr26 -``` - -### Hybrid job -This sample *Fortran90* program use both *MPI* and *openmp*. - -```fortran -program checkhybrid -use omp_lib -use mpi -implicit none - -integer(4)::rank,nrank -integer(4)::ierr -integer(4)::NUM_THREADS -character(len=MPI_MAX_PROCESSOR_NAME)::nodename -integer(4)::namelen - -call mpi_init(ierr) -call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr) -call mpi_comm_size(MPI_COMM_WORLD,nrank,ierr) -call mpi_get_processor_name(nodename,namelen,ierr) -NUM_THREADS=omp_get_max_threads() - -write(*,'(a6,i4,a6,i4)') 'rank',rank,trim(nodename),NUM_THREADS - -call mpi_finalize(ierr) - -endprogram checkhybrid -``` - -Use *Intel compiler* and *IntelMPI* to compile the code. - -```bash -module load compiler/intel-2018 -module load mpi/intel-2018 -mpif90 testhybrid.f90 -fopenmp -``` - -The first script is applying 4 nodes and setting 4 processes per node. -The number of openmp thread is not set in the environment. - -```bash -#!/bin/sh -#PBS -N testhybrid -#PBS -l nodes=4:ppn=4 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} -module load compiler/intel-2018 && echo $_ "LOADED" -module load mpi/intel-2018 && echo $_ "LOADED" - -### RUN ### -echo "###############################################" -mpirun -np ${PBS_NP} ./a.out > log.n4ppn4ompnull - -``` - -The result shows that 4 processes per node and 4 nodes in total is used to complete the job. -Each process has 18 threads automatically, which equals the number of cores per node divided by the number of processes per node, i.e. 18=72/4. - -```bash - rank 0 gr28 18 - rank 12 gr25 18 - rank 4 gr27 18 - rank 8 gr26 18 - rank 1 gr28 18 - rank 13 gr25 18 - rank 5 gr27 18 - rank 9 gr26 18 - rank 2 gr28 18 - rank 14 gr25 18 - rank 6 gr27 18 - rank 10 gr26 18 - rank 3 gr28 18 - rank 15 gr25 18 - rank 7 gr27 18 - rank 11 gr26 18 -```` - - -The second script is similar to the first one but setting 2 processes per node. - -```bash -#!/bin/sh -#PBS -N testhybrid -#PBS -l nodes=4:ppn=2 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} -module load compiler/intel-2018 && echo $_ "LOADED" -module load mpi/intel-2018 && echo $_ "LOADED" - -### RUN ### -echo "###############################################" -mpirun -np ${PBS_NP} ./a.out > log.n4ppn2ompnull - -``` - -One can find that each process has more number of threads. - -```bash - rank 0 gr28 36 - rank 2 gr27 36 - rank 4 gr26 36 - rank 1 gr28 36 - rank 3 gr27 36 - rank 5 gr26 36 - rank 6 gr25 36 - rank 7 gr25 36 -``` - -The third sample is specifying the number of `openmp` threads by `OMP_NUM_THREADS` environmental variable. - -```bash -#!/bin/sh -#PBS -N testhybrid -#PBS -l nodes=4:ppn=4 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} -module load compiler/intel-2018 && echo $_ "LOADED" -module load mpi/intel-2018 && echo $_ "LOADED" -export OMP_NUM_THREADS=12 - -### RUN ### -echo "###############################################" -mpirun -np ${PBS_NP} ./a.out > log.n4ppn4omp12 - -``` - -The result is same as the first one expect that the number of threads per process is limited. - -```bash - rank 4 gr27 12 - rank 8 gr26 12 - rank 12 gr25 12 - rank 0 gr28 12 - rank 5 gr27 12 - rank 9 gr26 12 - rank 13 gr25 12 - rank 1 gr28 12 - rank 6 gr27 12 - rank 10 gr26 12 - rank 14 gr25 12 - rank 2 gr28 12 - rank 7 gr27 12 - rank 11 gr26 12 - rank 15 gr25 12 - rank 3 gr28 12 -``` - -### GPU job -This sample use `nvcc` to compile a example *cuda* code - -```cuda -/* --------------------------------------------------- - My Hello world for CUDA programming - --------------------------------------------------- */ - -#include // C programming header file -#include // C programming header file - // cude.h is automatically included by nvcc... - -/* ------------------------------------ - Your first kernel (= GPU function) - ------------------------------------ */ -__global__ void hello( ) -{ - printf("Hello World From GPU!\n"); -} - -int main() -{ - /* ------------------------------------ - Call the hello( ) kernel function - ------------------------------------ */ - hello<<< 1, 4 >>>( ); - - printf("I am the CPU: Hello World ! \n"); - - sleep(1); // Necessary to give time to let GPU threads run !!! - - return 0; -} -``` - -```bash -nvcc hello.cu -o hello -``` - -A simply job submission script is as follow. - -```bash -#!/bin/sh -#PBS -N testgpu -#PBS -q gpu -#PBS -l nodes=1:ppn=72 -#PBS -W x=GRES:gpu at 1 -#PBS -l walltime=1:00:00 - -cd $PBS_O_WORKDIR - -### Set environment### -echo "###############################################" -echo "JOBID: " ${PBS_JOBID} -echo "JOBNAME: " ${PBS_JOBNAME} - -### RUN ### -echo "###############################################" -./hello > log -``` - -You will see the following output - -``` -I am the CPU: Hello World ! -Hello World From GPU! -Hello World From GPU! -Hello World From GPU! -Hello World From GPU! -``` - - -## PBS commands - -### submit a new job - -One can submit a job via the `qsub` command. - -`qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` -`[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` -`[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` -`[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` -`[-W additional_attributes]` `[-z]` `[script]` - -We recommend submit jobs via the job scripts 👉 `qsub JOB_SCRIPT`. -For job script sample, check the [samples](https://gravity-doc.github.io/Basic/Job.html#Submission). - -### check the job status - -`qstat` can show the status of all the jobs in **S (status)** column. - -```bash -[testuser@GRAVITY:~]:qstat -Job ID Name User Time Use S Queue -------------------------- ---------------- --------------- -------- - ----- -1276.login01 athena++ user1 3342:50: R small -1277.login01 athena++ user1 0 Q small -1352.login01 run_velprof user3 1257:36: R fat -``` - -Use `qstat -u ` to show the jobs submitted by one user. - -### cancel job - -| Command | Description | -| ----------------------- | ------------------------------------------------------------------------ | -| `qdel [-W time] JOB_ID` | `-W time` can specify the delay of the job cancel. The unit is seconds. | -| `qselect -u $USER | xargs qdel` | Delete all of your jobs | -| `qselect -u $USER -s Q | xargs qdel` | Delete all of your **Queueing** jobs | -| `qselect -u $USER -s R | xargs qdel` | Delete all of your **Running** jobs | - - - -### check pending status - -`checkjob JOB_ID` can provide the information of a pending job. - -### References - -```note -`qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` -`[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` -`[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` -`[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` -`[-W additional_attributes]` `[-z]` `[script]` --a date_time ： delay the job with time [[[[CC]YY]MM]DD]hhmm[.SS] --c interval ： 定义作业的检查点间隔，如果机器不支持检查点，则忽略此选项。 --C directive_prefix ：treat all lines begin with directive_prefix as qsub option. Otherwise, qsub option begins with '#PBS' --e path ：specify error file --I ：interactive --j join ：join output and error file. --l resource_list ： define resource list as follow -walltime=N ： wall time in unit of second, or in the form of hh:mm:ss -mem=N[K|M|G][B|W]：define memory usage -nodes=N:ppn=M ：define number of nodes N and processes per node M. --m mail_options ：mail_option为a：作业abort时给用户发信；为b：作业开始运行发信；为e：作业结束运行时发信。若无此选项，默认为a。 --M user_list ： mail addresss --N name ： jobname, less than 15 characters. --o path ： specify output file --p priority ： adjust priority, [-1024，1023]，default 0. --q destination ： specify which queue to use --r y|n ： 指明作业是否可运行，y为可运行，n为不可运行。 --S shell ： specify the SHELL to use, full path. --u user_list ： specify the user to run the job --v variable_list ： specify the environment variable list to export to this job. --V ： export all environment variable to this job. --W additional_attributes ： --z ： do not show JOB_ID information after submission -``` - -```note -`qstat` `[-f JOB_ID]` `[-a]` `[-i]` `[-n]` `[-s]` `[-R]` `[-Q [QUEUE]]` `[-q]` `[-B]` `[-u USER]` --f JOB_ID : specify the job --a : list all the jobs --i : list all non-running jobs --r : list all the running jobs --n : list the nodes assigned to jobs --s : 列出队列管理员与scheduler所提供的建议 --R : 列出磁盘预留信息 --Q [QUEUE] : list queue status --q [QUEUE] : list queue status --B : list PBS Server information --u USER : list the jobs for USER -``` - -- [qsub doc](http://docs.adaptivecomputing.com/torque/4-1-3/help.htm#topics/commands/qsub.htm) -- `man qsub` -- `man pbs_resources` - -## Slurm -```tip -*Slurm* is only available on **SGI** server! -``` - -### examples -create a job script `job.slurm` with the following content: - -```bash -#!/bin/bash - -#SBATCH --job-name=test -#SBATCH -n 16 # CPU cores -#SBATCH --output=%j.out -#SBATCH --error=%j.err - -echo "Start running ... ..." - -# run your own program!!! -sleep 10s - -echo "Job is done!" -``` - -Then, we can **submit** the job: - -```bash -sbatch job.slurm -``` - -### commands - -```bash -# submit a job -sbatch job.slurm - -# check job status -squeue - -# cancel a job -scancel JOB_ID - -# attach to a job (connect to the input/output/error stream) -sattach JOB_ID - -# view the queue info -sinfo -``` +--- +sort: 3 +title: Job +--- + +```warning +You **should** run your job via PBS (Torque). +You can **ONLY** do some **simple** jobs on the **login nodes**, otherwise, you might crash the system and be *banned*. + +Please only request resources that you actually need. **Do not** use more than or less than the number of cores you requested. +``` + +## Queues + +`qstat -q ` shows a summary of available queues. + +`qstat -f -Q` shows the detailed queue configurations, which are also listed below: + +- **Ordinary queues** + +| Queue name | Max nodes | Max time | Max memory per node | Priority | +| :--------: | :--------: | :------: | :-----------------: | :---------------------: | +| *debug* | 1 | 3 hours | 376 GB | High | +| *small* | 3 | 72 hours | 376 GB | Medium | +| *normal* | 18 | 48 hours | 376 GB | Low (**default queue**) | + +- **Special queues** + +| Queue name | Node | Max time | Max memory per node | Note | +| :--------: | :--------------------------: | :------: | :------------------: | :----------------------------------------------------------------------------------------------------------------------- | +| *gpu* | gr01
gr02
gr03
gr04 | 72 hours | 376 GB | **NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla V100 (32G)**
**NVIDIA Tesla A100 (80G)** | +| *fat* | gr35
gr36
fat01 | 72 hours | 754GB
3TB
6 TB | **large shared-memory** jobs (**mem > 360GB**) | + +## Basic example + + +### Serial job + +Suppose you have a program `test.f90` which is compiled as follows: + +```bash +module load gcc +gfortran test.f90 -o test.exe +``` + +A simply job submission script (`example.qsub`) is as follow. + +```bash +#!/bin/sh +#PBS -N testjob +#PBS -l nodes=1:ppn=1 +#PBS -q normal + +cd $PBS_O_WORKDIR + +### load the modules ### +module load gcc && echo $_ "LOADED" + +### RUN ### +./test.exe +``` + +To submit the job, + +```bash +qsub example.qsub +``` + +### parallel job +The content of an example *parallel* PBS script (`example.qsub`) is provided below: + +```bash +#!/bin/bash +#PBS -N My_job +#PBS -l nodes=1:ppn=72 +#PBS -q normal + +# run your own program!!! +cd $PBS_O_WORKDIR +mpirun -np 72 exec +``` + + +To submit the job, + +```bash +qsub example.qsub +``` + +```tip +Please only request resources that you actually need. In the above example, a full node (72 cores) is requested. However, if your job uses fewer cores, you should only request that many cores. For example +`#PBS -l nodes=1:ppn=8` +to request 8 cores on a single node. + +If your job uses OpenMP, you should set the number of threads to match your requested cores. For example, when requesting 8 cores for an OpenMP job, remember to also put the following in your job script before your final program line: +`export OMP_NUM_THREADS=8` +. + +If you are running a serial job, please only request **one** core: +`#PBS -l nodes=1:ppn=1` +``` + +## Complete example + +> job name is `My_job`. +> E-mail address is `lalala@sjtu.edu.cn`. +> need 1 node 72 cores per node +> need 360GB memory `mem=360gb ` +> need at most 48 hours +> make all of the output including error to `/home/lalala/code/my_output.log` + +```bash +#!/bin/bash +#PBS -N My_job +#PBS -m abe +#PBS -M lalala@sjtu.edu.cn +#PBS -l nodes=1:ppn=72 +#PBS -l mem=360gb +#PBS -l walltime=48:00:00 +#PBS -q normal +#PBS -j oe +#PBS -o /home/lalala/code/my_output.log + +echo running with ${PBS_NP} processes on ${PBS_NUM_NODES} nodes + +# run your own program!!! +cd $PBS_O_WORKDIR +mpirun -np 72 exec + +echo Done! +``` + +## More Examples +### Interactive job + +To request resources for doing something interactively, you can submit an interactive job. + +For example: + +```bash +qsub -l nodes=1:ppn=4,mem=8gb,walltime=12:00:00 -q normal -I +``` + +You will be logged onto the assigned node once the above job get running, and you can start to run your actual computation interactively. + +```warning +- Please use the system responsibly and only start an interactive job when necessary. + +- Do not use more resources than requested, otherwise you might affect other jobs or crash the node. + +- CPU time will be charged in the same way as normal jobs. Remember to log out of the computing node to end the interactive job. + +``` + +Reference: [https://www.msi.umn.edu/content/interactive-queue-use-qsub](https://www.msi.umn.edu/content/interactive-queue-use-qsub) + +```tip +The following PBS environment variable list is output by `env | grep PBS`, which maybe useful in your scripts. +``` + +```bash +PBS_VERSION=TORQUE-6.0.2 +PBS_JOBNAME=aaa +PBS_ENVIRONMENT=PBS_BATCH +PBS_O_WORKDIR=/home/user1 +PBS_TASKNUM=1 +PBS_O_HOME=/home/user1 +PBS_WALLTIME=25920000 +PBS_MOMPORT=15003 +PBS_GPUFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01gpu +PBS_O_QUEUE=normal +PBS_O_LOGNAME=user1 +PBS_O_LANG=en_US.utf8 +PBS_JOBCOOKIE=24CA13DFB269BB24B9E687EE94037901 +PBS_NODENUM=0 +PBS_NUM_NODES=1 +PBS_O_SHELL=/bin/bash +PBS_JOBID=1399.login01 +PBS_O_HOST=login02 +PBS_VNODENUM=0 +PBS_QUEUE=normal +PBS_O_MAIL=/var/spool/mail/user1 +PBS_O_SUBMIT_FILTER=/opt/tsce4/torque6/bin/submitfilter.sh +PBS_MICFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01mic +PBS_NP=1 +PBS_O_SERVER=login01 +PBS_NUM_PPN=1 +PBS_NODEFILE=/opt/tsce4/torque6/share/gr32/aux//1399.login01 +PBS_O_PATH=/usr/java/jre1.8.0_151/bin:/usr/java/jre1.8.0_151/bin:/opt/tsce4/maui/sbin:/opt/tsce4/maui/bin:/opt/tsce4/torque6/bin:/opt/tsce4/torque6/sbin:/usr/local/bin:/usr/local/bin:/usr/lib64/qt-3.3/bin:/home/user1/perl5/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/ibutils/bin:/opt/pdsh-2.29/bin:/opt/pdsh-2.29/sbin:.:/home/user1/.local/bin:/home/user1/bin +``` + +### Job array +When you need to run a series of jobs with different arguments, it is pretty useful. Because you do not need a loop any more. You can find more arguments [here](http://docs.adaptivecomputing.com/torque/6-1-1/adminGuide/Content/topics/torque/commands/qsub.htm). +1. You can use `-t` in the shell + + *example.pbs* script 👇 + + ```bash + #!/bin/bash + #PBS -N My_job + #PBS -l nodes=1:ppn=1 + #PBS -q normal + + # run your own program!!! + python test.py $PBS_ARRAYID + ``` + + run it 👇 + + ```bash + qsub example.pbs -t 1-666 + ``` + +2. Another way is writing `-t` in the script + + *example.pbs* script 👇 + + ```bash + #!/bin/bash + #PBS -N My_job + #PBS -l nodes=1:ppn=1 + #PBS -q normal + #PBS -t 1-666%5 + + # run your own program!!! + python test.py $PBS_ARRAYID + ``` + + run it 👇 + + ```bash + qsub example.pbs + ``` + The optional `%5` in the `-t` parameter specifies a maximum of 5 jobs to be allowed running simultaneously. + +**`PBS_ARRAYID`** here represents the range `1-666`, which means your job will run as + +```bash +python test.py 1 +python test.py 2 +python test.py 3 +... +... +python test.py 666 +``` + +### Openmp job +This is a sample *Fortran90* program using *openmp* to parallelize a *do* loop. + +```fortran +program testopenmp +use omp_lib +implicit none + +integer(4)::num_threads +integer(4)::thread +integer(4)::i + +num_threads=omp_get_max_threads() +write(*,*) 'number of threads:',num_threads + +!$omp parallel do schedule(static) default(private) +do i=1,16 + thread=omp_get_thread_num() + write(*,*) 'loop',i,'is processed by thread',thread +enddo +!$omp end parallel do + +endprogram testopenmp +``` + +Use *Intel compiler* to compile. + +```bash +module load compiler/intel-2018 +ifort testopenmp.f90 -fopenmp +``` + +This sample job submission script is applying one node. +The number of threads is set to 8. + +```bash +#!/bin/sh +#PBS -N testopenmp +#PBS -l nodes=1:ppn=8 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} +module load compiler/intel-2018 && echo $_ "LOADED" +export OMP_NUM_THREADS=8 + +### RUN ### +echo "###############################################" +./a.out > log + +``` + +The result shows that the *do* loop is been parallelized by 8 threads. + +```bash + number of threads: 8 + loop 1 is processed by thread 0 + loop 3 is processed by thread 1 + loop 5 is processed by thread 2 + loop 7 is processed by thread 3 + loop 2 is processed by thread 0 + loop 9 is processed by thread 4 + loop 4 is processed by thread 1 + loop 6 is processed by thread 2 + loop 8 is processed by thread 3 + loop 11 is processed by thread 5 + loop 13 is processed by thread 6 + loop 10 is processed by thread 4 + loop 12 is processed by thread 5 + loop 15 is processed by thread 7 + loop 14 is processed by thread 6 + loop 16 is processed by thread 7 +``` + +### MPI job +This sample shows a *MPI* parallel program in *Fortran90*. + +```fortran +program testmpi +use mpi +implicit none + +integer(4)::rank,nrank +integer(4)::ierr +character(len=MPI_MAX_PROCESSOR_NAME)::nodename +integer(4)::namelen + +call mpi_init(ierr) +call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr) +call mpi_comm_size(MPI_COMM_WORLD,nrank,ierr) +call mpi_get_processor_name(nodename,namelen,ierr) + +write(*,'(a6,i4,a18,a6)') 'rank',rank,'is run on node',trim(nodename) + +call mpi_finalize(ierr) + +endprogram testmpi +``` + +Use *Intel compiler* and *IntelMPI* to compile. + +```bash +module load compiler/intel-2018 +module load mpi/intel-2018 +mpif90 testmpi.f90 +``` + +This sample script is applying 4 nodes and setting 2 processes per node. + +```bash +#!/bin/sh +#PBS -N testmpi +#PBS -l nodes=4:ppn=2 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} +module load compiler/intel-2018 && echo $_ "LOADED" +module load mpi/intel-2018 && echo $_ "LOADED" + +### RUN ### +echo "###############################################" +mpirun -np ${PBS_NP} ./a.out > log + +``` + +The result shows that *gr25*-*gr28* is used to finish the job, with 2 processes per node. + +```bash + rank 2 is run on node gr27 + rank 6 is run on node gr25 + rank 3 is run on node gr27 + rank 7 is run on node gr25 + rank 0 is run on node gr28 + rank 4 is run on node gr26 + rank 1 is run on node gr28 + rank 5 is run on node gr26 +``` + +### Hybrid job +This sample *Fortran90* program use both *MPI* and *openmp*. + +```fortran +program checkhybrid +use omp_lib +use mpi +implicit none + +integer(4)::rank,nrank +integer(4)::ierr +integer(4)::NUM_THREADS +character(len=MPI_MAX_PROCESSOR_NAME)::nodename +integer(4)::namelen + +call mpi_init(ierr) +call mpi_comm_rank(MPI_COMM_WORLD,rank,ierr) +call mpi_comm_size(MPI_COMM_WORLD,nrank,ierr) +call mpi_get_processor_name(nodename,namelen,ierr) +NUM_THREADS=omp_get_max_threads() + +write(*,'(a6,i4,a6,i4)') 'rank',rank,trim(nodename),NUM_THREADS + +call mpi_finalize(ierr) + +endprogram checkhybrid +``` + +Use *Intel compiler* and *IntelMPI* to compile the code. + +```bash +module load compiler/intel-2018 +module load mpi/intel-2018 +mpif90 testhybrid.f90 -fopenmp +``` + +The first script is applying 4 nodes and setting 4 processes per node. +The number of openmp thread is not set in the environment. + +```bash +#!/bin/sh +#PBS -N testhybrid +#PBS -l nodes=4:ppn=4 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} +module load compiler/intel-2018 && echo $_ "LOADED" +module load mpi/intel-2018 && echo $_ "LOADED" + +### RUN ### +echo "###############################################" +mpirun -np ${PBS_NP} ./a.out > log.n4ppn4ompnull + +``` + +The result shows that 4 processes per node and 4 nodes in total is used to complete the job. +Each process has 18 threads automatically, which equals the number of cores per node divided by the number of processes per node, i.e. 18=72/4. + +```bash + rank 0 gr28 18 + rank 12 gr25 18 + rank 4 gr27 18 + rank 8 gr26 18 + rank 1 gr28 18 + rank 13 gr25 18 + rank 5 gr27 18 + rank 9 gr26 18 + rank 2 gr28 18 + rank 14 gr25 18 + rank 6 gr27 18 + rank 10 gr26 18 + rank 3 gr28 18 + rank 15 gr25 18 + rank 7 gr27 18 + rank 11 gr26 18 +```` + + +The second script is similar to the first one but setting 2 processes per node. + +```bash +#!/bin/sh +#PBS -N testhybrid +#PBS -l nodes=4:ppn=2 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} +module load compiler/intel-2018 && echo $_ "LOADED" +module load mpi/intel-2018 && echo $_ "LOADED" + +### RUN ### +echo "###############################################" +mpirun -np ${PBS_NP} ./a.out > log.n4ppn2ompnull + +``` + +One can find that each process has more number of threads. + +```bash + rank 0 gr28 36 + rank 2 gr27 36 + rank 4 gr26 36 + rank 1 gr28 36 + rank 3 gr27 36 + rank 5 gr26 36 + rank 6 gr25 36 + rank 7 gr25 36 +``` + +The third sample is specifying the number of `openmp` threads by `OMP_NUM_THREADS` environmental variable. + +```bash +#!/bin/sh +#PBS -N testhybrid +#PBS -l nodes=4:ppn=4 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} +module load compiler/intel-2018 && echo $_ "LOADED" +module load mpi/intel-2018 && echo $_ "LOADED" +export OMP_NUM_THREADS=12 + +### RUN ### +echo "###############################################" +mpirun -np ${PBS_NP} ./a.out > log.n4ppn4omp12 + +``` + +The result is same as the first one expect that the number of threads per process is limited. + +```bash + rank 4 gr27 12 + rank 8 gr26 12 + rank 12 gr25 12 + rank 0 gr28 12 + rank 5 gr27 12 + rank 9 gr26 12 + rank 13 gr25 12 + rank 1 gr28 12 + rank 6 gr27 12 + rank 10 gr26 12 + rank 14 gr25 12 + rank 2 gr28 12 + rank 7 gr27 12 + rank 11 gr26 12 + rank 15 gr25 12 + rank 3 gr28 12 +``` + +### GPU job +This sample use `nvcc` to compile a example *cuda* code + +```cuda +/* --------------------------------------------------- + My Hello world for CUDA programming + --------------------------------------------------- */ + +#include // C programming header file +#include // C programming header file + // cude.h is automatically included by nvcc... + +/* ------------------------------------ + Your first kernel (= GPU function) + ------------------------------------ */ +__global__ void hello( ) +{ + printf("Hello World From GPU!\n"); +} + +int main() +{ + /* ------------------------------------ + Call the hello( ) kernel function + ------------------------------------ */ + hello<<< 1, 4 >>>( ); + + printf("I am the CPU: Hello World ! \n"); + + sleep(1); // Necessary to give time to let GPU threads run !!! + + return 0; +} +``` + +```bash +nvcc hello.cu -o hello +``` + +A simply job submission script is as follow. + +```bash +#!/bin/sh +#PBS -N testgpu +#PBS -q gpu +#PBS -l nodes=1:ppn=72 +#PBS -W x=GRES:gpu at 1 +#PBS -l walltime=1:00:00 + +cd $PBS_O_WORKDIR + +### Set environment### +echo "###############################################" +echo "JOBID: " ${PBS_JOBID} +echo "JOBNAME: " ${PBS_JOBNAME} + +### RUN ### +echo "###############################################" +./hello > log +``` + +You will see the following output + +``` +I am the CPU: Hello World ! +Hello World From GPU! +Hello World From GPU! +Hello World From GPU! +Hello World From GPU! +``` + + +## PBS commands + +### submit a new job + +One can submit a job via the `qsub` command. + +`qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` +`[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` +`[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` +`[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` +`[-W additional_attributes]` `[-z]` `[script]` + +We recommend submit jobs via the job scripts 👉 `qsub JOB_SCRIPT`. +For job script sample, check the [samples](/Basic/Job?id=basic-example). + +### check the job status + +`qstat` can show the status of all the jobs in **S (status)** column. + +```bash +[testuser@GRAVITY:~]:qstat +Job ID Name User Time Use S Queue +------------------------- ---------------- --------------- -------- - ----- +1276.login01 athena++ user1 3342:50: R small +1277.login01 athena++ user1 0 Q small +1352.login01 run_velprof user3 1257:36: R fat +``` + +Use `qstat -u ` to show the jobs submitted by one user. + +### cancel job + +| Command | Description | +| ----------------------- | ------------------------------------------------------------------------ | +| `qdel [-W time] JOB_ID` | `-W time` can specify the delay of the job cancel. The unit is seconds. | +| `qselect -u $USER | xargs qdel` | Delete all of your jobs | +| `qselect -u $USER -s Q | xargs qdel` | Delete all of your **Queueing** jobs | +| `qselect -u $USER -s R | xargs qdel` | Delete all of your **Running** jobs | + + + +### check pending status + +`checkjob JOB_ID` can provide the information of a pending job. + +### References + +```note +`qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` +`[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` +`[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` +`[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` +`[-W additional_attributes]` `[-z]` `[script]` +-a date_time ： delay the job with time [[[[CC]YY]MM]DD]hhmm[.SS] +-c interval ： 定义作业的检查点间隔，如果机器不支持检查点，则忽略此选项。 +-C directive_prefix ：treat all lines begin with directive_prefix as qsub option. Otherwise, qsub option begins with '#PBS' +-e path ：specify error file +-I ：interactive +-j join ：join output and error file. +-l resource_list ： define resource list as follow +walltime=N ： wall time in unit of second, or in the form of hh:mm:ss +mem=N[K|M|G][B|W]：define memory usage +nodes=N:ppn=M ：define number of nodes N and processes per node M. +-m mail_options ：mail_option为a：作业abort时给用户发信；为b：作业开始运行发信；为e：作业结束运行时发信。若无此选项，默认为a。 +-M user_list ： mail addresss +-N name ： jobname, less than 15 characters. +-o path ： specify output file +-p priority ： adjust priority, [-1024，1023]，default 0. +-q destination ： specify which queue to use +-r y|n ： 指明作业是否可运行，y为可运行，n为不可运行。 +-S shell ： specify the SHELL to use, full path. +-u user_list ： specify the user to run the job +-v variable_list ： specify the environment variable list to export to this job. +-V ： export all environment variable to this job. +-W additional_attributes ： +-z ： do not show JOB_ID information after submission +``` + +```note +`qstat` `[-f JOB_ID]` `[-a]` `[-i]` `[-n]` `[-s]` `[-R]` `[-Q [QUEUE]]` `[-q]` `[-B]` `[-u USER]` +-f JOB_ID : specify the job +-a : list all the jobs +-i : list all non-running jobs +-r : list all the running jobs +-n : list the nodes assigned to jobs +-s : 列出队列管理员与scheduler所提供的建议 +-R : 列出磁盘预留信息 +-Q [QUEUE] : list queue status +-q [QUEUE] : list queue status +-B : list PBS Server information +-u USER : list the jobs for USER +``` + +- [qsub doc](http://docs.adaptivecomputing.com/torque/4-1-3/help.htm#topics/commands/qsub.htm) +- `man qsub` +- `man pbs_resources` + +## Slurm +```tip +*Slurm* is only available on **SGI** server! +``` + +### examples +create a job script `job.slurm` with the following content: + +```bash +#!/bin/bash + +#SBATCH --job-name=test +#SBATCH -n 16 # CPU cores +#SBATCH --output=%j.out +#SBATCH --error=%j.err + +echo "Start running ... ..." + +# run your own program!!! +sleep 10s + +echo "Job is done!" +``` + +Then, we can **submit** the job: + +```bash +sbatch job.slurm +``` + +### commands + +```bash +# submit a job +sbatch job.slurm + +# check job status +squeue + +# cancel a job +scancel JOB_ID + +# attach to a job (connect to the input/output/error stream) +sattach JOB_ID + +# view the queue info +sinfo +``` diff --git a/docs/Basic/Login.md b/docs/Basic/Login.md index e8a79a8..52c4d06 100644 --- a/docs/Basic/Login.md +++ b/docs/Basic/Login.md @@ -74,7 +74,7 @@ You will be asked to provide your *public key* (`id_rsa_Gravity.pub`) when regis There are **two ways** to add (*append*) `public key`: - use *Jupyter's* *terminal* to **append** `id_rsa_Gravity.pub` to `~/.ssh/authorized_keys` -- send the `id_rsa_Gravity.pub` and your `username` [via email📧](https://gravity-doc.github.io/#contact), we will help you to add it +- send the `id_rsa_Gravity.pub` and your `username` [via email📧](/?id=contact), we will help you to add it ### 3. log in @@ -276,7 +276,7 @@ The key's randomart image is: 有**两种方式**添加公钥： - 使用*Jupyter*的终端，将公钥`id_rsa_Gravity.pub`添加至`~/.ssh/authorized_keys`即可 -- 将公钥`id_rsa_Gravity.pub`通过[邮件发送](https://gravity-doc.github.io/#contact)给我们，我们将帮您添加。 +- 将公钥`id_rsa_Gravity.pub`通过[邮件发送](/?id=contact)给我们，我们将帮您添加。 ### 3. 登陆 diff --git a/docs/Basic/Resource_Monitor.md b/docs/Basic/Resource_Monitor.md index 0abf319..9e0ceb1 100644 --- a/docs/Basic/Resource_Monitor.md +++ b/docs/Basic/Resource_Monitor.md @@ -1,173 +1,178 @@ ---- -sort: 4 -title: Resource Monitor ---- -# Resource Monitor - -## Web interface -Considering the security 🛡️, you have to use [**SSH tunnel** to access it](https://gravity-doc.github.io/Advanced/status_website.html). - - -## User information - -```bash -[lalala@login01 ~]$ finger lalala -Login: lalala Name: LALALA -Directory: /home/lalala Shell: /bin/bash -Last login Thu Oct 15 09:14 (CST) on 6.6.6.6 -No mail. -No Plan. -``` - -## Disk quota -You can use the command on **login01** or **login02**: `icfsquota username` or `icfsquota`, just like this: - -```bash -[lalala@login01 ~]$ icfsquota lalala -############## Hi ༼ つ ◕_◕ ༽つ lalala Your Storage Quota Info ################ - -Max size = 5.00TB , Used size= 4.33TB , Left size= 666.00GB - -################################################################################ -``` - -The default quota per account is `3T`. - -```note -If you cannot run `icfsquota` successfully😅, please check your permissions setting of `$HOME` foler. -`chmod 755 $HOME` -So that you can run `icfsquota`🥳 -``` - -If you run out of quota, you can free out space by deleting some files that are no longer needed. If no more files can be deleted, ask your group owner to allocate more disk space to you. - -```note -Files deleted through the graphical desktop will be moved to Trash folder that still takes up space. You can empty the trash if this is the case, through either the desktop or on the command line: - -`rm -rf ./local/share/Trash/*` - -``` - -You can use the command on **SGI**, `quota -ls` like this - -```bash -# lalala in ~ [18:53:10] -$ quota -ls -Disk quotas for user lalala (uid 666): - Filesystem space quota limit grace files quota limit grace -/dev/mapper/system-home - 7876M 0K 10240M 105k 0 0 -/dev/mapper/SFA7700X-vol1 - 10G 0K 1024G 1771k 0 0 -``` - -Each account has a default quota of -- **10GB** at `/home/username` -- **1TB** at `/mnt/ddnfs/data_users/username` - -Gravity **Home Directory** is mounted at `/gravity/home` - -## CPU time usage and cost -You can use `cpuquota` or `cpuquota username` on **login01** or **login02**, just like this: - -```bash -[lalala@login01 ~]$ cpuquota -##################### Hi ༼ つ ◕_◕ ༽つ lalala Your Cost ######################### - -This year [20210101 ~ 20210531] cost: 6666 ￥ -This month [20210501 ~ 20210531] CPU time (hour): 666 -This month [20210501 ~ 20210531] cost: 66 ￥ - -######### Complete usage statistics: https://stat.gravity.sjtu.edu.cn ########## -``` - -```tip -Your advisor will bear the cost 🥳 -``` - -## PBS - -1. `pestat` provides the CPU and memory usage of all nodes. - -```bash -[testuser@GRAVITY:~]:pestat -node state load pmem ncpu mem resi usrs tasks jobids/users -gr01 free 0.00 385382 72 417382 14929 0/0 0 -gr02 free 2.08* 385382 72 417382 17059 0/0 0 -... -gr17 free 6.05* 385382 72 417382 6078 0/0 0 -gr18 excl 63.94* 385382 72 417382 63881 65/1 1 [1345:user1] -... -gr31 excl 72.00* 385382 72 417382 58531 87/1 1 [1345:user1] -gr32 free 1.25 385382 72 417382 68937 1/1 1 [1342:user2] -gr33 free 64.00* 385382 72 417382 76560 65/1 1 [1276:user3] -... -gr36 free 63.96* 385382 72 417382 76529 69/1 1 [1276:user3] -fat01 free 192.02* 6191128 192 6223128 1284016 1/1 1 [1340:user4] -``` - -PBS job management provides `pbsnodes -l all` to check whether the nodes is free or occupied. - -```bash -[testuser@GRAVITY:~]:pbsnodes -l all -gr01 free -... -gr17 free -gr18 job-exclusive -... -gr31 job-exclusive -... -gr36 free -fat01 free -``` - -2. `showq` provides the queue information. - -```bash -[testuser@GRAVITY:~]:showq -ACTIVE JOBS-------------------- -JOBNAME USERNAME STATE PROC REMAINING STARTTIME - -1342 user2 Running 1 21:07:51 Sat Sep 12 09:12:13 -1345 user1 Running 1008 22:59:41 Sat Sep 12 11:04:03 -1276 user3 Running 256 1:06:07:10 Thu Sep 10 18:11:32 -1340 user4 Running 96 2:11:21:33 Fri Sep 11 23:25:55 - - 4 Active Jobs 1361 of 2784 Processors Active (48.89%) - 20 of 37 Nodes Active (54.05%) - -IDLE JOBS---------------------- -JOBNAME USERNAME STATE PROC WCLIMIT QUEUETIME - -1277 user3 Idle 256 3:00:00:00 Thu Sep 10 18:11:29 - -1 Idle Job - -BLOCKED JOBS---------------- -JOBNAME USERNAME STATE PROC WCLIMIT QUEUETIME - - -Total Jobs: 5 Active Jobs: 4 Idle Jobs: 1 Blocked Jobs: 0 -``` - -1. One can also use `qstat` and `qstat -an` to check basic and detailed queue information. - -```bash -[testuser@GRAVITY:~]:qstat -an -login01: - Req'd Req'd Elap -Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time ------------------------ ----------- -------- ---------------- ------ ----- ------ --------- --------- - --------- -1276.login01 user1 small athena++ 13962 4 256 -- 72:00:00 R 42:01:48 - gr36/0-63+gr35/0-63+gr34/0-63+gr33/0-63 -1277.login01 user1 small athena++ -- 4 256 -- 72:00:00 Q -- - -- -1340.login01 user2 fat run_velprof 132765 1 96 1800gb 72:00:00 R 12:47:25 - fat01/0-95 -1342.login01 user3 normal test 173700 1 1 -- 24:00:00 R 03:01:07 - gr32/0 -1345.login01 user4 normal imgBDK 181608 14 1008 -- 24:00:00 R 01:09:17 - gr31/0-71+gr30/0-71+gr29/0-71+gr28/0-71+gr27/0-71+gr26/0-71+gr25/0-71 - +gr24/0-71+gr23/0-71+gr22/0-71+gr21/0-71+gr20/0-71+gr19/0-71+gr18/0-71 -``` - +--- +sort: 4 +title: Resource Monitor +--- + +## Web interface + +- You can access the [*Inspur Management Web*](https://gravity.sjtu.edu.cn/inspur/) to check your Job status/history and resource usage. + +- You can also use [**SSH tunnel** to access it](/Advanced/status_website?id=ssh-tunnel-access). + + +## User information + +```bash +[lalala@login01 ~]$ finger lalala +Login: lalala Name: LALALA +Directory: /home/lalala Shell: /bin/bash +Last login Thu Oct 15 09:14 (CST) on 6.6.6.6 +No mail. +No Plan. +``` + +## Disk quota + +You can use the command on **login01** or **login02**: `icfsquota username` or `icfsquota`, just like this: + +```bash +[lalala@login01 ~]$ icfsquota lalala +############## Hi ༼ つ ◕_◕ ༽つ lalala Your Storage Quota Info ################ + +Max size = 5.00TB , Used size= 4.33TB , Left size= 666.00GB + +################################################################################ +``` + +The default quota per account is `3T`. + +```note +If you cannot run `icfsquota` successfully😅, please check your permissions setting of `$HOME` foler. +`chmod 755 $HOME` +So that you can run `icfsquota`🥳 +``` + +If you run out of quota, you can free out space by deleting some files that are no longer needed. If no more files can be deleted, ask your group owner to allocate more disk space to you. + +```note +Files deleted through the graphical desktop will be moved to Trash folder that still takes up space. You can empty the trash if this is the case, through either the desktop or on the command line: + +`rm -rf ./local/share/Trash/*` + +``` + +You can use the command on **SGI**, `quota -ls` like this + +```bash +# lalala in ~ [18:53:10] +$ quota -ls +Disk quotas for user lalala (uid 666): + Filesystem space quota limit grace files quota limit grace +/dev/mapper/system-home + 7876M 0K 10240M 105k 0 0 +/dev/mapper/SFA7700X-vol1 + 10G 0K 1024G 1771k 0 0 +``` + +Each account has a default quota of + +- **10GB** at `/home/username` +- **1TB** at `/mnt/ddnfs/data_users/username` + +Gravity **Home Directory** is mounted at `/gravity/home` + +## CPU time usage and cost + +You can use `cpuquota` or `cpuquota username` on **login01** or **login02**, just like this: + +```bash +[lalala@login01 ~]$ cpuquota +##################### Hi ༼ つ ◕_◕ ༽つ lalala Your Cost ######################### + +This year [20210101 ~ 20210531] cost: 6666 ￥ +This month [20210501 ~ 20210531] CPU time (hour): 666 +This month [20210501 ~ 20210531] cost: 66 ￥ + +######### Complete usage statistics: https://stat.gravity.sjtu.edu.cn ########## +``` + +```tip +Your advisor will bear the cost 🥳 +``` + +## PBS + +1. `pestat` provides the CPU and memory usage of all nodes. + +```bash +[testuser@GRAVITY:~]:pestat +node state load pmem ncpu mem resi usrs tasks jobids/users +gr01 free 0.00 385382 72 417382 14929 0/0 0 +gr02 free 2.08* 385382 72 417382 17059 0/0 0 +... +gr17 free 6.05* 385382 72 417382 6078 0/0 0 +gr18 excl 63.94* 385382 72 417382 63881 65/1 1 [1345:user1] +... +gr31 excl 72.00* 385382 72 417382 58531 87/1 1 [1345:user1] +gr32 free 1.25 385382 72 417382 68937 1/1 1 [1342:user2] +gr33 free 64.00* 385382 72 417382 76560 65/1 1 [1276:user3] +... +gr36 free 63.96* 385382 72 417382 76529 69/1 1 [1276:user3] +fat01 free 192.02* 6191128 192 6223128 1284016 1/1 1 [1340:user4] +``` + +PBS job management provides `pbsnodes -l all` to check whether the nodes is free or occupied. + +```bash +[testuser@GRAVITY:~]:pbsnodes -l all +gr01 free +... +gr17 free +gr18 job-exclusive +... +gr31 job-exclusive +... +gr36 free +fat01 free +``` + +2. `showq` provides the queue information. + +```bash +[testuser@GRAVITY:~]:showq +ACTIVE JOBS-------------------- +JOBNAME USERNAME STATE PROC REMAINING STARTTIME + +1342 user2 Running 1 21:07:51 Sat Sep 12 09:12:13 +1345 user1 Running 1008 22:59:41 Sat Sep 12 11:04:03 +1276 user3 Running 256 1:06:07:10 Thu Sep 10 18:11:32 +1340 user4 Running 96 2:11:21:33 Fri Sep 11 23:25:55 + + 4 Active Jobs 1361 of 2784 Processors Active (48.89%) + 20 of 37 Nodes Active (54.05%) + +IDLE JOBS---------------------- +JOBNAME USERNAME STATE PROC WCLIMIT QUEUETIME + +1277 user3 Idle 256 3:00:00:00 Thu Sep 10 18:11:29 + +1 Idle Job + +BLOCKED JOBS---------------- +JOBNAME USERNAME STATE PROC WCLIMIT QUEUETIME + + +Total Jobs: 5 Active Jobs: 4 Idle Jobs: 1 Blocked Jobs: 0 +``` + +1. One can also use `qstat` and `qstat -an` to check basic and detailed queue information. + +```bash +[testuser@GRAVITY:~]:qstat -an +login01: + Req'd Req'd Elap +Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time +----------------------- ----------- -------- ---------------- ------ ----- ------ --------- --------- - --------- +1276.login01 user1 small athena++ 13962 4 256 -- 72:00:00 R 42:01:48 + gr36/0-63+gr35/0-63+gr34/0-63+gr33/0-63 +1277.login01 user1 small athena++ -- 4 256 -- 72:00:00 Q -- + -- +1340.login01 user2 fat run_velprof 132765 1 96 1800gb 72:00:00 R 12:47:25 + fat01/0-95 +1342.login01 user3 normal test 173700 1 1 -- 24:00:00 R 03:01:07 + gr32/0 +1345.login01 user4 normal imgBDK 181608 14 1008 -- 24:00:00 R 01:09:17 + gr31/0-71+gr30/0-71+gr29/0-71+gr28/0-71+gr27/0-71+gr26/0-71+gr25/0-71 + +gr24/0-71+gr23/0-71+gr22/0-71+gr21/0-71+gr20/0-71+gr19/0-71+gr18/0-71 +``` + diff --git a/docs/MISC/FAQ.md b/docs/MISC/FAQ.md index ef6e136..dcd5409 100644 --- a/docs/MISC/FAQ.md +++ b/docs/MISC/FAQ.md @@ -1,106 +1,120 @@ ---- -sort: 1 -title: FAQ ---- - -# FAQ - -```tip -If there is **NO** solution of your problem, feel free to [contact us](https://gravity-doc.github.io/#contact)! -``` - -## Permission denied -I got a **permission denied** error when trying to login with ssh -- your ip is **banned** (see below) -- you forgot to use *ssh-key* to login -- your `~/.ssh/authorized-keys` file on the server has the wrong permissions - - `~/.ssh` should have permission `700` - - `~/.ssh/authorized-keys` should have permission `600`) - - contact the admin to fix the permission - -You may be able to find out more information about the issue by adding the `-vvv` option to ssh, i.e., `ssh -vvv`. - -## Forgot password -What if I forgot my password? 🔐 -- [go to web authentication](https://jupyter.gravity.sjtu.edu.cn/auth/) to reset your password -- [send an **E-mail**](mailto:gravity-hpc@sjtu.edu.cn) with your *username*, we will help you to reset the password (it will take several hours maybe) - -## Forgot ssh key -What if I lost my *ssh-key*? 🛡️ - -- [Send an **E-mail**](mailto:gravity-hpc@sjtu.edu.cn) to us, including your [**ssh public key**](https://gravity-doc.github.io/Basic/Login.html#1-generate-ssh-key) and your **username** - -PS. You'd better use **@sjtu.edu.cn**, so that we can recognize who you really are 👀 - -```tip -You may be prompted for a passphrase when login with a ssh-key. This is the password you set-up to protect the key when generating your ssh-key. It is **different** from the password you use for jupyter. -``` - -## IP banned -What if my **IP** was **banned** ❌, so that I couldn't access **[JupyterHub](https://gravity.sjtu.edu.cn/)** or **ssh login**? - -- Contact us and tell us **your IP** - -## Extend time of job -Can I extend my time of running PBS job? ⌛ - -- Sure! Contact us and tell us how much time you need. - -## Add disk storage space -I want more **Disk storage space** 💾 -1. contact your advisor, get his/her permission -2. contact admins to increase your disk quota - -## Job never start -My PBS job is always waiting 😤 - -- Use `checkjob ` to check the **reason**. -- Use `showstart ` to see how much time is left to **start your job**. - -## Job failed -My PBS job cannot run successfully 🙄 - -- Check your **error file** and **output file** such as `xxx.e` and `xxx.o` -- you can run your program on the **login nodes** first to test. - -## WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! - -when you **ssh log in** *Gravity*, if you meet Warning: - -```text -@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ -@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! -Someone could be eavesdropping on you right now (man-in-the-middle attack)! -It is also possible that the RSA host key has just been changed. -The fingerprint for the RSA key sent by the remote host is -08:98:a9:cc:f8:37:20:6b:b4:b1:6c:3a:15:b9:a9:92. -Please contact your system administrator. -Add correct host key in /home/lalala/.ssh/known_hosts to get rid of this message. -Offending key in /home/lalala/.ssh/known_hosts:2 -RSA host key for gravity.sjtu.edu.cn has changed and you have requested strictecking. -Host key verification failed. -``` - -Don't worry, just delete the **specific row** of `~/.ssh/known_hosts` anderything will be OK -From the *Warning*，you can see `Offending key in /home/lalala/.ssh/known_hosts:2`, so we just need to remove the **2nd row**. - -## binary file can not be executed -## source code string cannot contain null bytes -`binary file can not be executed` or `source code string cannot contain null bytes` 👉 [issue of conda](https://gravity-doc.github.io/MISC/Issues.html#conda) - - > I am *so sorry* to tell you that your whole **CONDA** installation is destroyed. Cause this is a tricky bug🐛, we cannot fix it 😭. - > - > You should re-install your conda, and **never** install/update any package on **login01**. - -## Job logfile is empty -My job is *running* but my logfile is still **empty**? - -- This is probably because your log messages are being buffered on the computing node. You can try log into the computing node and check /var/spool/torque/spool/ to find the buffered logfile. -- `python -u xxxxx.py` will disable the buffer. - -**Refs:** -> https://guido.vonrudorff.de/2013/pbstorque-unbuffered-output-on-network-filesystems/ -> https://serverfault.com/questions/294218/is-there-a-way-to-redirect-output-to-a-file-without-buffering-on-unix-linux +--- +sort: 1 +title: FAQ +--- + +# FAQ + +```tip +If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! +``` + +## Permission denied + +I got a **permission denied** error when trying to login with ssh + +- your ip is **banned** (see below) +- you forgot to use *ssh-key* to login +- your `~/.ssh/authorized-keys` file on the server has the wrong permissions + - `~/.ssh` should have permission `700` + - `~/.ssh/authorized-keys` should have permission `600`) + + contact the admin to fix the permission + +You may be able to find out more information about the issue by adding the `-vvv` option to ssh, i.e., `ssh -vvv`. + +## Forgot password + +What if I forgot my password? 🔐 + +- [go to web authentication](https://jupyter.gravity.sjtu.edu.cn/auth/) to reset your password +- [send an **E-mail**](mailto:gravity-hpc@sjtu.edu.cn) with your *username*, we will help you to reset the password (it will take several hours maybe) + +## Forgot ssh key + +What if I lost my *ssh-key*? 🛡️ + +- [Send an **E-mail**](mailto:gravity-hpc@sjtu.edu.cn) to us, including your [**ssh public key**](/Basic/Login?id=_1-generate-ssh-key) and your **username** + +PS. You'd better use **@sjtu.edu.cn**, so that we can recognize who you really are 👀 + +```tip +You may be prompted for a passphrase when login with a ssh-key. This is the password you set-up to protect the key when generating your ssh-key. It is **different** from the password you use for jupyter. +``` + +## IP banned + +What if my **IP** was **banned** ❌, so that I couldn't access **[JupyterHub](https://gravity.sjtu.edu.cn/)** or **ssh login**? + +- [Check you IP here](https://ipinfo.io/), then contact us and tell us **your IP** + +## Extend time of job + +Can I extend my time of running PBS job? ⌛ + +- Sure! Contact us and tell us how much time you need. + +## Add disk storage space + +I want more **Disk storage space** 💾 + +1. contact your advisor, get his/her permission +2. contact admins to increase your disk quota + +## Job never start + +My PBS job is always waiting 😤 + +- Use `checkjob ` to check the **reason**. +- Use `showstart ` to see how much time is left to **start your job**. + +## Job failed + +My PBS job cannot run successfully 🙄 + +- Check your **error file** and **output file** such as `xxx.e` and `xxx.o` +- you can run your program on the **login nodes** first to test. + +## WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! + +when you **ssh log in** *Gravity*, if you meet Warning: + +```text +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that the RSA host key has just been changed. +The fingerprint for the RSA key sent by the remote host is +08:98:a9:cc:f8:37:20:6b:b4:b1:6c:3a:15:b9:a9:92. +Please contact your system administrator. +Add correct host key in /home/lalala/.ssh/known_hosts to get rid of this message. +Offending key in /home/lalala/.ssh/known_hosts:2 +RSA host key for gravity.sjtu.edu.cn has changed and you have requested strictecking. +Host key verification failed. +``` + +Don't worry, just delete the **specific row** of `~/.ssh/known_hosts` anderything will be OK +From the *Warning*，you can see `Offending key in /home/lalala/.ssh/known_hosts:2`, so we just need to remove the **2nd row**. + +## binary file can not be executed + +## source code string cannot contain null bytes + +`binary file can not be executed` or `source code string cannot contain null bytes` 👉 [issue of conda](/MISC/Issues?id=conda) + + > I am *so sorry* to tell you that your whole **CONDA** installation is destroyed. Cause this is a tricky bug🐛, we cannot fix it 😭. + > + > You should re-install your conda, and **never** install/update any package on **login01**. + +## Job logfile is empty + +My job is *running* but my logfile is still **empty**? + +- This is probably because your log messages are being buffered on the computing node. You can try log into the computing node and check /var/spool/torque/spool/ to find the buffered logfile. +- `python -u xxxxx.py` will disable the buffer. + +**Refs:** +> https://guido.vonrudorff.de/2013/pbstorque-unbuffered-output-on-network-filesystems/ +> https://serverfault.com/questions/294218/is-there-a-way-to-redirect-output-to-a-file-without-buffering-on-unix-linux diff --git a/docs/MISC/Issues.md b/docs/MISC/Issues.md index a28ca86..b59396e 100644 --- a/docs/MISC/Issues.md +++ b/docs/MISC/Issues.md @@ -1,98 +1,98 @@ ---- -sort: 2 -title: Issues ---- - -# Issues - -```note -If there is **NO** solution of your problem, feel free to [contact us](https://gravity-doc.github.io/#Contact)! -``` - -```danger -**NEVER EVER** install/update any package at *login01* using `conda/mamba`. Otherwise, your whole conda environment will be destroyed! -- You can use `pip install` instead -- You can go to *login02* to use `conda/mamba install/create` -``` - -## **Conda** -One recieves '**binary file can not be executed**' after installing of Anaconda latest version (2020-05). -The same happens when update conda from old version. -This issue is due to incompatiblity of the **parallel file system** mounted in kernel state. -As a temporary workaround, we have mounted the filesystem with **NFS on login02**, so that you can use login02 for installation and update of anaconda packages. This means degraded IO performance on login02 (~700MB/s), but it should suffice common usage requirements. If high IO performance is needed interactively, use login01. - -## hdf5 file open failure on login02 -When opening a hdf5 file on login02, one may sometimes run into the following error -using python: - -``` -----> 1 f=h5py.File('snap_001.0.hdf5','r') -IOError: Unable to open file (unable to lock file, errno = 37, error message = 'No locks available') -``` - -or using `h5ls`: - -``` -[user@login02]$ h5ls snap_001.0.hdf5 -snap_001.0.hdf5: unable to open file -``` - -This is a known problem in the NFS file system on `login02`. You can switch to read the file on `login01` instead. - -```warning -Alternatively, you may choose to disable hdf5 file locking before reading, by setting the environment variable `HDF5_USE_FILE_LOCKING` to `FALSE`, e.g., in bash: - - `export HDF5_USE_FILE_LOCKING=FALSE` - -This will disable hdf5 file locking, but you will have to be more careful about opening files to avoid problematic access patterns (i.e.: multiple writers to the same hdf5 file) that the file locking was designed to prevent. -``` - -## **日志残留问题** - - 问题描述:当用户提交两个节点以上的任务时，就会出现日志残留问题。 - - ```bash - [inspur@login02 gr01]$ cat inspur-test-3.e2619 - cgdelete: cannot remove group 'gr28.2619.login01': No such file or directory - ``` - - 此问题涉及内核模块cgroup，不会影响作业的正常运行，用户遇到该问题可暂时忽略，目前仍在定位问题。 - -- **compiler/intel-2020 module无法使用的问题** - 目前intel2020跟module适配存在问题，使用module load compiler/intel-2020载入环境变量无法生效。 - - 如果需要使用intel2020，可以直接写source路径，或者通过source载入。 - - ```bash - source /opt/intel-2020.sh - ``` - -## **openmpi-4.0.4** -High version *openmpi* use UCX (Unified Communication X) as the default pml (Physical Markup Language). -You need to specify this in the running command. - -```bash -mpirun -np 4 --mca pml ucx ./a.out -``` - -You may run into error/warning messages regarding openib, such as -``` -By default, for Open MPI 4.0 and later, infiniband ports on a device -are not used by default. The intent is to use UCX for these devices. -You can override this policy by setting the btl_openib_allow_ib MCA parameter -to true. - -WARNING: There was an error initializing an OpenFabrics device. -[login02:780991] 3 more processes have sent help message help-mpi-btl-openib.txt / ib port not selected -[login02:780991] 3 more processes have sent help message help-mpi-btl-openib.txt / error in device init -``` -These are because the openmpi has been configured with the deprecated openib (which is nowadays superceded by openfabric). To avoid these messages, you can disable openib when using mpirun, -```bash -mpirun -np 4 --mca btl "^openib" ./a.out -``` - - -[see more](https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX) - -## **performance issue on fat01** -The current hardware architecture of fat01 has a limited memory bandwidth which could become a bottleneck for memory intensive jobs. For example, for large memory (e.g., >2TB) OpenMP jobs, it may take longer to run using 48 or more cores than using 24 cores. When the memory usage of the system is high, the overall performance of the system will also degrade. One may also experience very slow IO speed under such situations. So it is advised to first do some test runs to pick an optimal number of threads to use. +--- +sort: 2 +title: Issues +--- + +# Issues + +```note +If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! +``` + +```danger +**NEVER EVER** install/update any package at *login01* using `conda/mamba`. Otherwise, your whole conda environment will be destroyed! +- You can use `pip install` instead +- You can go to *login02* to use `conda/mamba install/create` +``` + +## **Conda** +One recieves '**binary file can not be executed**' after installing of Anaconda latest version (2020-05). +The same happens when update conda from old version. +This issue is due to incompatiblity of the **parallel file system** mounted in kernel state. +As a temporary workaround, we have mounted the filesystem with **NFS on login02**, so that you can use login02 for installation and update of anaconda packages. This means degraded IO performance on login02 (~700MB/s), but it should suffice common usage requirements. If high IO performance is needed interactively, use login01. + +## hdf5 file open failure on login02 +When opening a hdf5 file on login02, one may sometimes run into the following error +using python: + +``` +----> 1 f=h5py.File('snap_001.0.hdf5','r') +IOError: Unable to open file (unable to lock file, errno = 37, error message = 'No locks available') +``` + +or using `h5ls`: + +``` +[user@login02]$ h5ls snap_001.0.hdf5 +snap_001.0.hdf5: unable to open file +``` + +This is a known problem in the NFS file system on `login02`. You can switch to read the file on `login01` instead. + +```warning +Alternatively, you may choose to disable hdf5 file locking before reading, by setting the environment variable `HDF5_USE_FILE_LOCKING` to `FALSE`, e.g., in bash: + + `export HDF5_USE_FILE_LOCKING=FALSE` + +This will disable hdf5 file locking, but you will have to be more careful about opening files to avoid problematic access patterns (i.e.: multiple writers to the same hdf5 file) that the file locking was designed to prevent. +``` + +## **日志残留问题** + + 问题描述:当用户提交两个节点以上的任务时，就会出现日志残留问题。 + + ```bash + [inspur@login02 gr01]$ cat inspur-test-3.e2619 + cgdelete: cannot remove group 'gr28.2619.login01': No such file or directory + ``` + + 此问题涉及内核模块cgroup，不会影响作业的正常运行，用户遇到该问题可暂时忽略，目前仍在定位问题。 + +- **compiler/intel-2020 module无法使用的问题** + 目前intel2020跟module适配存在问题，使用module load compiler/intel-2020载入环境变量无法生效。 + + 如果需要使用intel2020，可以直接写source路径，或者通过source载入。 + + ```bash + source /opt/intel-2020.sh + ``` + +## **openmpi-4.0.4** +High version *openmpi* use UCX (Unified Communication X) as the default pml (Physical Markup Language). +You need to specify this in the running command. + +```bash +mpirun -np 4 --mca pml ucx ./a.out +``` + +You may run into error/warning messages regarding openib, such as +``` +By default, for Open MPI 4.0 and later, infiniband ports on a device +are not used by default. The intent is to use UCX for these devices. +You can override this policy by setting the btl_openib_allow_ib MCA parameter +to true. + +WARNING: There was an error initializing an OpenFabrics device. +[login02:780991] 3 more processes have sent help message help-mpi-btl-openib.txt / ib port not selected +[login02:780991] 3 more processes have sent help message help-mpi-btl-openib.txt / error in device init +``` +These are because the openmpi has been configured with the deprecated openib (which is nowadays superceded by openfabric). To avoid these messages, you can disable openib when using mpirun, +```bash +mpirun -np 4 --mca btl "^openib" ./a.out +``` + + +[see more](https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX) + +## **performance issue on fat01** +The current hardware architecture of fat01 has a limited memory bandwidth which could become a bottleneck for memory intensive jobs. For example, for large memory (e.g., >2TB) OpenMP jobs, it may take longer to run using 48 or more cores than using 24 cores. When the memory usage of the system is high, the overall performance of the system will also degrade. One may also experience very slow IO speed under such situations. So it is advised to first do some test runs to pick an optimal number of threads to use. diff --git a/docs/Policy.md b/docs/Policy.md index 2279391..b82034e 100644 --- a/docs/Policy.md +++ b/docs/Policy.md @@ -37,7 +37,8 @@ title: Policy ### 五．软件 -服务器已安装部分开源软件和正版软件(https://gravity-doc.github.io/Software/Software_Installed.html)，若用户有其他需求，可在用户目录中自行安装或提出申请由管理员安装到系统仓库。 +服务器已安装部分[开源软件和正版软件](/Software/Software_Installed) +若用户有其他需求，[可在用户目录中自行安装](/Software/User_Software_Installation)或提出申请由管理员安装到系统仓库。 用户应当自觉尊重知识产权，不在服务器上安装、复制或传播盗版软件。 diff --git a/docs/QuickStart.md b/docs/QuickStart.md index cda7b0c..236a087 100644 --- a/docs/QuickStart.md +++ b/docs/QuickStart.md @@ -1,141 +1,141 @@ ---- -sort: 1 -title: Quick Start ---- -## Apply for an account 🙋‍♂️🙋‍♀️ -Follow the instructions [here](https://gravity-doc.github.io/Basic/Account.html) to apply for an account. Make sure you read the instructions and user policies before applying. Wait for a day or so, then you will receive the e-mail 🥳. - -## Log in to Gravity Cluster 💻 - -You are supposed to use **_ssh key_** to log in to Gravity, because we **DO NOT** allow password login. - -```bash -ssh -i ~/.ssh/id_rsa_For_Gravity username@gravity.sjtu.edu.cn -``` - -## Job (PBS) 🔭 - -- submit an **interactive** job - -```bash -qsub -I -l nodes=1:ppn=72,walltime=48:00:00 -q normal -``` - -- submit a **batch** job - -create _a file_ `myjob.pbs` like this: - -```bash -#!/bin/bash -#PBS -N My_job -#PBS -m abe -#PBS -M lalala@sjtu.edu.cn -#PBS -l nodes=1:ppn=72 -#PBS -l walltime=48:00:00 -#PBS -q normal -cd $PBS_O_WORKDIR - -# run your own program!!! -python test.py -``` - -then, submit it! - -```bash -qsub myjob.pbs -``` - -- **check**/**delete** job - -```bash -qstat -a -qdel 36162 # 36162 is your jobID -``` - -## Cheatsheet 📜 - -| **Command** | **Description** | -| :----------------------------------------------------------- | :----------------------------------------------------------- | -| `ssh -i ~/.ssh/id_rsa_For_Gravity username@gravity.sjtu.edu.cn` | log in to Gravity using _ssh key_: **~/.ssh/id_rsa_For_Gravity** | -| `ssh -X -i your_ssh_key username@gravity.sjtu.edu.cn ` | login with X11 (use GUI) | -| `ssh -i your_ssh_key -CNL localhost:8888:localhost:9999 username@gravity.sjtu.edu.cn` | **ssh tunnel** from local port 8888 to Gravity port 9999 (use jupyter-notebook, VNC) | -| `ssh -i your_ssh_key -CNL localhost:8888:gr33:9999 username@gravity.sjtu.edu.cn` | **ssh tunnel** from local port 8888 to Gravity **gr33 node** port 9999 (use [jupyter-notebook](http://localhost:8888), VNC) | -| `ssh -i your_ssh_key -CNL localhost:8888:localhost:8443 username@login01` | **ssh tunnel** from local 8888 to remote 8443 (access [**status website**](https://localhost:8888), you have to log in to **login01**) | -| `scp -i your_ssh_key ~/local_file username@gravity.sjtu.edu.cn:/home/username/remote_file` | copy local file to Gravity | -| `scp -r -i your_ssh_key local_folder username@gravity.sjtu.edu.cn:remote_folder` | copy local folder to Gravity | -| `scp -i your_ssh_key username@gravity.sjtu.edu.cn:remote_file local_file ` | copy Gravity file to local | -| `scp -r -i your_ssh_key username@gravity.sjtu.edu.cn:remote_folder local_folder` | copy Gravity folder to local | -| `rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file` | copy local files/folder to Gravity (**check time&size**, faster) | -| `rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file` | copy Gravity files/folder to local (**check time&size**, faster) | -| `rsync -acvP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file` | copy local files/folder to Gravity (**check MD5**, slower) | -| `rsync -acvP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file` | copy Gravity files/folder to local (**check MD5**, slower) | -| `wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | download file | -| `curl -O https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | download file | -| `axel -n 8 https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | ⚡ download file using **multi-threads** (8 threads) | -| `qsub -I -l nodes=1:ppn=72,walltime=48:00:00 -q normal` | submit an **interactive** job to _normal queue_ | -| `qsub -I -l nodes=gr06:ppn=72,walltime=48:00:00 -q normal` | submit an **interactive** job to _normal queue_, specifically, use **gr06** node | -| `qsub -I -X -v DISPLAY -l nodes=1:ppn=72,walltime=12:00:00 -q normal` | submit an **interactive** job to _normal queue_, use **X11** to show **GUI** | -| `qsub myjob.pbs` | submit a batch job | -| `qstat -a` | jobs status | -| `qstat -n` | jobs status (see which node you are using) | -| `qstat -f ` | job status (**in detail**) | -| `qstat -q` | queues info | -| `qstat -f -Q` | queues info (in detail) | -| `checkjob ` | job status (if you are **always waiting** or **something wrong**) | -| `showstart ` | ***when*** to start (if your job is waiting) | -| `qdel ` | **delete** a job | -| `qdel -p ` | force to **delete** a job | -| `qselect -u $USER | xargs qdel` | delete all of your jobs | -| `qselect -u $USER -s Q | xargs qdel` | delete all of your **Queueing** jobs | -| `qselect -u $USER -s R | xargs qdel` | delete all of your **Running** jobs | -| `qhold ` | **hold** a job until you release it | -| `qrls ` | **release** a job | -| `pestat` | every node status | -| `showq` | overview of nodes | -| `sbatch job.slurm` | submit a batch job (*slurm*) | -| `squeue` | check job state (*slurm*) | -| `scancel ` | cancel job (*slurm*) | -| `sattach ` | **connect** to standard input/output/error streams of job (*slurm*) | -| `sinfo` | overview of nodes (*slurm*) | -| `module av` | modules **available** | -| `module list` | modules loaded | -| `module load anaconda/anaconda/conda-4.12.0` | **load** module | -| `module unload anaconda/anaconda/conda-4.12.0` | **unload** module | -| `module purge` | **unload all** modules | -| `conda env list` | see conda environment | -| `conda activate ` | activate environment | -| `conda deactivate` | deactivate environment | -| `conda create -n myenv python=3.9 matplotlib scipy astropy` | create a virtual environment | -| `conda create -n new_env --clone old_env` | clone `new_env` from `old_env` | -| `conda search emcee` | search package | -| `conda install -c conda-forge emcee` | install package | -| `mamba create -n myenv python=3.9 matplotlib scipy astropy` | ⚡ create a virtual environment | -| `mamba create -n new_env --clone old_env` | ⚡ clone `new_env` from `old_env` | -| `mamba search emcee` | ⚡ search package | -| `mamba install -c conda-forge emcee` | ⚡ install package | -| `python -m ipykernel install --user --name ` | **install ipykernel** so that you can use it in **Jupyter** notebook/lab | -| `python -u test.py` | run python script **without buffering** (no stdout and stderr buffer) | -| `icfsquota ` | disk usage and quota **(Gravity)** | -| `quota -ls` | disk usage and quota **(SGI)** | -| `cpuquota` | CPU hours and **expense** | -| `finger ` | user info | -| `passwd` | change password | -| `yppasswd` | change password (**recommended**) | -| `top` | processes info | -| `top -u ` | my processes info | -| `htop` | processes info (beautiful) | -| `htop -u ` | my processes info (beautiful) | -| `ps aux | grep python` | process info including string: *python* | -| `ps -eo pid,tty,user,command,lstart,etime | grep ` | **my own** processes info | -| `kill -9 ` | **kill** the process using its PID | -| `pkill -u ` | **kill all** processes belonging to *username* | -| `tail -f My_PBS_Job.log` | see the file output **in time** | -| `git clone https://github.com/numpy/numpy.git` | clone a repository | -| `git pull` | pull | -| `pgit clone https://github.com/numpy/numpy.git` | ⚡ clone a repository | -| `pgit pull` | ⚡ pull | -| `eog` | view **images** | -| `display` | view **images** | -| `curl -4 ip.p3terx.com` | check my **IP** | -| `curl myip.ipip.net` | check my **IP** | - +--- +sort: 1 +title: Quick Start +--- +## Apply for an account 🙋‍♂️🙋‍♀️ +Follow the instructions [here](/Basic/Account) to apply for an account. Make sure you read the instructions and user policies before applying. Wait for a day or so, then you will receive the e-mail 🥳. + +## Log in to Gravity Cluster 💻 + +You are supposed to use **_ssh key_** to log in to Gravity, because we **DO NOT** allow password login. + +```bash +ssh -i ~/.ssh/id_rsa_For_Gravity username@gravity.sjtu.edu.cn +``` + +## Job (PBS) 🔭 + +- submit an **interactive** job + +```bash +qsub -I -l nodes=1:ppn=72,walltime=48:00:00 -q normal +``` + +- submit a **batch** job + +create _a file_ `myjob.pbs` like this: + +```bash +#!/bin/bash +#PBS -N My_job +#PBS -m abe +#PBS -M lalala@sjtu.edu.cn +#PBS -l nodes=1:ppn=72 +#PBS -l walltime=48:00:00 +#PBS -q normal +cd $PBS_O_WORKDIR + +# run your own program!!! +python test.py +``` + +then, submit it! + +```bash +qsub myjob.pbs +``` + +- **check**/**delete** job + +```bash +qstat -a +qdel 36162 # 36162 is your jobID +``` + +## Cheatsheet 📜 + +| **Command** | **Description** | +| :----------------------------------------------------------- | :----------------------------------------------------------- | +| `ssh -i ~/.ssh/id_rsa_For_Gravity username@gravity.sjtu.edu.cn` | log in to Gravity using _ssh key_: **~/.ssh/id_rsa_For_Gravity** | +| `ssh -X -i your_ssh_key username@gravity.sjtu.edu.cn ` | login with X11 (use GUI) | +| `ssh -i your_ssh_key -CNL localhost:8888:localhost:9999 username@gravity.sjtu.edu.cn` | **ssh tunnel** from local port 8888 to Gravity port 9999 (use jupyter-notebook, VNC) | +| `ssh -i your_ssh_key -CNL localhost:8888:gr33:9999 username@gravity.sjtu.edu.cn` | **ssh tunnel** from local port 8888 to Gravity **gr33 node** port 9999 (use [jupyter-notebook](http://localhost:8888), VNC) | +| `ssh -i your_ssh_key -CNL localhost:8888:localhost:8443 username@login01` | **ssh tunnel** from local 8888 to remote 8443 (access [**status website**](https://localhost:8888), you have to log in to **login01**) | +| `scp -i your_ssh_key ~/local_file username@gravity.sjtu.edu.cn:/home/username/remote_file` | copy local file to Gravity | +| `scp -r -i your_ssh_key local_folder username@gravity.sjtu.edu.cn:remote_folder` | copy local folder to Gravity | +| `scp -i your_ssh_key username@gravity.sjtu.edu.cn:remote_file local_file ` | copy Gravity file to local | +| `scp -r -i your_ssh_key username@gravity.sjtu.edu.cn:remote_folder local_folder` | copy Gravity folder to local | +| `rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file` | copy local files/folder to Gravity (**check time&size**, faster) | +| `rsync -avP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file` | copy Gravity files/folder to local (**check time&size**, faster) | +| `rsync -acvP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" local_file username@gravity.sjtu.edu.cn:remote_file` | copy local files/folder to Gravity (**check MD5**, slower) | +| `rsync -acvP -e "ssh -i ~/.ssh/id_rsa_For_Gravity" username@gravity.sjtu.edu.cn:remote_file local_file` | copy Gravity files/folder to local (**check MD5**, slower) | +| `wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | download file | +| `curl -O https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | download file | +| `axel -n 8 https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh` | ⚡ download file using **multi-threads** (8 threads) | +| `qsub -I -l nodes=1:ppn=72,walltime=48:00:00 -q normal` | submit an **interactive** job to _normal queue_ | +| `qsub -I -l nodes=gr06:ppn=72,walltime=48:00:00 -q normal` | submit an **interactive** job to _normal queue_, specifically, use **gr06** node | +| `qsub -I -X -v DISPLAY -l nodes=1:ppn=72,walltime=12:00:00 -q normal` | submit an **interactive** job to _normal queue_, use **X11** to show **GUI** | +| `qsub myjob.pbs` | submit a batch job | +| `qstat -a` | jobs status | +| `qstat -n` | jobs status (see which node you are using) | +| `qstat -f ` | job status (**in detail**) | +| `qstat -q` | queues info | +| `qstat -f -Q` | queues info (in detail) | +| `checkjob ` | job status (if you are **always waiting** or **something wrong**) | +| `showstart ` | ***when*** to start (if your job is waiting) | +| `qdel ` | **delete** a job | +| `qdel -p ` | force to **delete** a job | +| `qselect -u $USER | xargs qdel` | delete all of your jobs | +| `qselect -u $USER -s Q | xargs qdel` | delete all of your **Queueing** jobs | +| `qselect -u $USER -s R | xargs qdel` | delete all of your **Running** jobs | +| `qhold ` | **hold** a job until you release it | +| `qrls ` | **release** a job | +| `pestat` | every node status | +| `showq` | overview of nodes | +| `sbatch job.slurm` | submit a batch job (*slurm*) | +| `squeue` | check job state (*slurm*) | +| `scancel ` | cancel job (*slurm*) | +| `sattach ` | **connect** to standard input/output/error streams of job (*slurm*) | +| `sinfo` | overview of nodes (*slurm*) | +| `module av` | modules **available** | +| `module list` | modules loaded | +| `module load anaconda/anaconda/conda-4.12.0` | **load** module | +| `module unload anaconda/anaconda/conda-4.12.0` | **unload** module | +| `module purge` | **unload all** modules | +| `conda env list` | see conda environment | +| `conda activate ` | activate environment | +| `conda deactivate` | deactivate environment | +| `conda create -n myenv python=3.9 matplotlib scipy astropy` | create a virtual environment | +| `conda create -n new_env --clone old_env` | clone `new_env` from `old_env` | +| `conda search emcee` | search package | +| `conda install -c conda-forge emcee` | install package | +| `mamba create -n myenv python=3.9 matplotlib scipy astropy` | ⚡ create a virtual environment | +| `mamba create -n new_env --clone old_env` | ⚡ clone `new_env` from `old_env` | +| `mamba search emcee` | ⚡ search package | +| `mamba install -c conda-forge emcee` | ⚡ install package | +| `python -m ipykernel install --user --name ` | **install ipykernel** so that you can use it in **Jupyter** notebook/lab | +| `python -u test.py` | run python script **without buffering** (no stdout and stderr buffer) | +| `icfsquota ` | disk usage and quota **(Gravity)** | +| `quota -ls` | disk usage and quota **(SGI)** | +| `cpuquota` | CPU hours and **expense** | +| `finger ` | user info | +| `passwd` | change password | +| `yppasswd` | change password (**recommended**) | +| `top` | processes info | +| `top -u ` | my processes info | +| `htop` | processes info (beautiful) | +| `htop -u ` | my processes info (beautiful) | +| `ps aux | grep python` | process info including string: *python* | +| `ps -eo pid,tty,user,command,lstart,etime | grep ` | **my own** processes info | +| `kill -9 ` | **kill** the process using its PID | +| `pkill -u ` | **kill all** processes belonging to *username* | +| `tail -f My_PBS_Job.log` | see the file output **in time** | +| `git clone https://github.com/numpy/numpy.git` | clone a repository | +| `git pull` | pull | +| `pgit clone https://github.com/numpy/numpy.git` | ⚡ clone a repository | +| `pgit pull` | ⚡ pull | +| `eog` | view **images** | +| `display` | view **images** | +| `curl -4 ip.p3terx.com` | check my **IP** | +| `curl myip.ipip.net` | check my **IP** | + diff --git a/docs/README.md b/docs/README.md index f6bfc81..537cba5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -## [Gravity Documentation](https://gravity-doc.github.io) +## [Gravity Documentation](/README.md) This is the documentation for the **Gravity Cluster** of [Department of Astronomy (DOA)](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). ```tip diff --git a/docs/Software/User_Software_Installation.md b/docs/Software/User_Software_Installation.md index 3c74319..62e3608 100644 --- a/docs/Software/User_Software_Installation.md +++ b/docs/Software/User_Software_Installation.md @@ -1,87 +1,87 @@ ---- -sort: 2 -title: User Software Installation ---- - - -```tip -You can use `module` to manage your own software installed -Also, you can just ignore `module` if you do not have **too many** tools or they do not **conflict** with each other -``` - -# Use modules - -Gravity provides a lot of softwares and libraries pre-installed through the `module` tool, as documented on [this page](https://gravity-doc.github.io/Software/Software-Installed.html). - -## Add your own modulefiles - -Users can create a `.modulerc` file in your `HOME` directory. -In this file, you can add new paths to your own modulefiles to the environmental variable *MODULEPATH*. - -```bash -#%Module1.0##### -append-path MODULEPATH /home/user1/mymodulefiles -``` - -Note that `#%Module1.0#####` is required at the first line. - -When you use `module` command, system can also find modulefiles in this directory. - -## Write your own modulefiles - -After installing some software, saying `FFTW-3.3.4`, you may want to create your own modulefiles for it. - -First, you should create a folder named *FFTW* inside your own modulefiles path (*/home/user1/mymodulefiles/FFTW* in this case). -Second, create a file named *3.3.4*. -Add the following content to this file. - -```bash -#%Module1.0##################################################################### -## -## modules modulefile -## -## modulefiles/modules. Generated from modules.in by configure. -## -proc ModulesHelp { } { - global version prefix - - puts stderr "\tmodules - loads the modules software & application environment" - puts stderr "\n\tThis adds $prefix/* to several of the" - puts stderr "\tenvironment variables." - puts stderr "\n\tVersion $version\n" -} - -module-whatis "fftw 3.3.4" - -set version 3.3.4 -set prefix /home/user1/fftw-3.3.4 - -set root $prefix - -prepend-path PATH $root/bin -prepend-path MANPATH $root/man -prepend-path LIBRARY_PATH $root/lib -prepend-path LD_LIBRARY_PATH $root/lib -prepend-path LD_RUN_PATH $root/lib -prepend-path CPATH $root/include - -prereq softA -conflict softB -``` - -Again, the first line is required. - -`module-whatis` sets the content when you use `module whatis` shows. - -`set` command sets values for the variables. - -`prepend-path PATH new_path` adds new path before the paths already exist in the environmental variables *$PATH*. - -Similarly, `append-path` adds new path after the paths. -Note that the order matters if in the environmental variables there exist multiple paths to different versions of the same software. - -`prereq` can set the dependence. In this case, you can not load this module if *softA* has not been loaded. - -`conflict` specifies the module that conflict with this one. You can not load this module if *softB* has been loaded. - -Reference: [module-file](https://modules.readthedocs.io/en/stable/modulefile.html#) +--- +sort: 2 +title: User Software Installation +--- + + +```tip +You can use `module` to manage your own software installed +Also, you can just ignore `module` if you do not have **too many** tools or they do not **conflict** with each other +``` + +# Use modules + +Gravity provides a lot of softwares and libraries pre-installed through the `module` tool, as documented on [this page](/Software/Software_Installed). + +## Add your own modulefiles + +Users can create a `.modulerc` file in your `HOME` directory. +In this file, you can add new paths to your own modulefiles to the environmental variable *MODULEPATH*. + +```bash +#%Module1.0##### +append-path MODULEPATH /home/user1/mymodulefiles +``` + +Note that `#%Module1.0#####` is required at the first line. + +When you use `module` command, system can also find modulefiles in this directory. + +## Write your own modulefiles + +After installing some software, saying `FFTW-3.3.4`, you may want to create your own modulefiles for it. + +First, you should create a folder named *FFTW* inside your own modulefiles path (*/home/user1/mymodulefiles/FFTW* in this case). +Second, create a file named *3.3.4*. +Add the following content to this file. + +```bash +#%Module1.0##################################################################### +## +## modules modulefile +## +## modulefiles/modules. Generated from modules.in by configure. +## +proc ModulesHelp { } { + global version prefix + + puts stderr "\tmodules - loads the modules software & application environment" + puts stderr "\n\tThis adds $prefix/* to several of the" + puts stderr "\tenvironment variables." + puts stderr "\n\tVersion $version\n" +} + +module-whatis "fftw 3.3.4" + +set version 3.3.4 +set prefix /home/user1/fftw-3.3.4 + +set root $prefix + +prepend-path PATH $root/bin +prepend-path MANPATH $root/man +prepend-path LIBRARY_PATH $root/lib +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LD_RUN_PATH $root/lib +prepend-path CPATH $root/include + +prereq softA +conflict softB +``` + +Again, the first line is required. + +`module-whatis` sets the content when you use `module whatis` shows. + +`set` command sets values for the variables. + +`prepend-path PATH new_path` adds new path before the paths already exist in the environmental variables *$PATH*. + +Similarly, `append-path` adds new path after the paths. +Note that the order matters if in the environmental variables there exist multiple paths to different versions of the same software. + +`prereq` can set the dependence. In this case, you can not load this module if *softA* has not been loaded. + +`conflict` specifies the module that conflict with this one. You can not load this module if *softB* has been loaded. + +Reference: [module-file](https://modules.readthedocs.io/en/stable/modulefile.html#) diff --git a/docs/Updates.md b/docs/Updates.md index 70b453d..2181fd5 100644 --- a/docs/Updates.md +++ b/docs/Updates.md @@ -1,64 +1,69 @@ ---- -sort: 4 -title: Updates ---- - -## Future ✨ - -- [ ] support container (Singularity) -- [ ] switch to Slurm (PBS will be **Deprecated**) - -## Done 🥳 - -### 2022.11 - -- [x] **SGI** can access Gravity **Home** Directory at `/gravity/home` -- [x] **SGI** can access the Internet directly now 🏄‍ -- [x] **SGI** has a disk quota for each user, which is [**10G+1TB**](https://gravity-doc.github.io/Basic/Resource_Monitor.html#disk-quota) - -### 2022.7 - -- [x] [**Gravity Homepage**](https://jupyter.gravity.sjtu.edu.cn/) is ready! 🎉🎉🎉 -- [x] add [**FileServer**](https://gravity-doc.github.io/Basic/Data_Transfer.html#FileServer), support download/**share** files 🎉🎉🎉 ([File Sharing DEMO](https://gravity.sjtu.edu.cn/demo/)) -- [x] add [**Two-Factor Authentication (2FA)**](https://gravity-doc.github.io/Basic/Login.html#web-login)🔒 for our website -- [x] add [**NEW Spawner for Jupyter**](https://gravity-doc.github.io/Basic/Jupyter.html#start-a-server), now you can select and **input options** -- [x] support **change password** / **reset password**(if you forgot your password) on [Gravity Authentication](https://gravity-doc.github.io/Basic/Login.html#web-login) -- [x] NEW in *Jupyter*: [resource monitor](https://gravity-doc.github.io/Basic/Jupyter.html#resource-monitor), [Diagram](https://gravity-doc.github.io/Basic/Jupyter.html#diagram), [Variable Inspector](https://gravity-doc.github.io/Basic/Jupyter.html#variable-inspector), [**HDF5 viewer**](https://gravity-doc.github.io/Basic/Jupyter.html#hdf5-viewer), [**`` selection**](https://gravity-doc.github.io/Basic/Jupyter.html#tab-selection), [Retrolab](https://gravity-doc.github.io/Basic/Jupyter.html#retrolab), *dracula* theme, [中文语言包](https://gravity-doc.github.io/Basic/Jupyter.html#chinese). Remove some unused kernels -- [x] add **HTTP proxy server**, now you can [**🏄‍surf the Internet**](https://gravity-doc.github.io/Software/Proxy.html#surf-the-internet) such as `git/curl/wget` when using *Jupyter/VScode* on *computing nodes*. Meanwhile, it will automatically *speedup*⚡ `git clone/pull/push` from/to **Github** **[ NO WARRANTY ]** -- [x] add **HTTP cache** to accelerate *Jupyter/VScode* -- [x] add [**Monitor**](https://jupyter.gravity.sjtu.edu.cn/status/?theme=dark) to check Gravity status -- [x] add [**SpeedTest**](https://jupyter.gravity.sjtu.edu.cn/speedtest/) to test connection speed between you and Gravity - -### 2021.9 - -- [x] install [`Julia, Go, R, Rust, C, C++, Fortran` **kernels**](https://gravity-doc.github.io/Basic/JupyterHub.html#choose-a-kernel-environment) into *Jupyterlab* and install many useful packages in `R, Julia` -- [x] make [*VScode*](https://gravity-doc.github.io/Basic/JupyterHub.html#vscode) available on computing nodes - -### 2021.8 - -- [x] reconstruct the [*Gravity Documentation*](https://gravity-doc.github.io/) - -### 2021.7 - -- [x] [add entry of `VScode`](https://gravity-doc.github.io/Basic/JupyterHub.html#vscode) in jupyterlab, and support **C/C++, Fortran, Go, R, Julia, Python** if you choose *VScode [sys]* in Jupyterlab -- [x] run `jupyter-lab/notebook` from both *login* and *computing nodes* via **[JupyterHub](https://gravity.sjtu.edu.cn/)** -- [x] install **several extensions** on Jupyterlab (ie. git, drawio, execute-time, resource-monitor, jupyterlab-lsp) - -### 2021.6 -- [x] [enable priority of PBS queue](https://gravity-doc.github.io/Basic/Job.html#ordinary-queues) -- [x] [support **Data Sharing** service](https://jupyter.gravity.sjtu.edu.cn/share/) (need to contact admins to use it) -- [x] **limit** each user resource usage on *login nodes* (**20 cores + 50 GB**) -- [x] automatically **ban IP** with too many failed login attempts ( *ssh* + *JupyterHub*) - -### 2021.5 -- [x] [speedup git ⚡](https://gravity-doc.github.io/Software/Proxy.html#speedup-git) (including `git clone, git pull, etc.`) - -- [x] add and improve command to query CPU time (expense) and Disk usage - > `icfsquota` - > `cpuquota` - -- [x] [enable the **E-mail notification** of PBS ](https://gravity-doc.github.io/Basic/Job.html#complete-example) - - > In your **PBS script**, you can add `#PBS -m abe` and `#PBS -M @gravity.sjtu.edu.cn` - -- [x] shorten the waiting time when log in Gravity +--- +sort: 4 +title: Updates +--- + +## Future ✨ + +- [ ] support container (Singularity) +- [ ] switch to Slurm (PBS will be **Deprecated**) + +## Done 🥳 + +### 2023.11 + +- [x] self-hosted [*Gitlab*](https://git.gravity.sjtu.edu.cn/) ([how to use](/Advanced/Gitlab)) +- [x] self-hosted [*Gravity Documentation*](https://gravity.sjtu.edu.cn/doc/) + +### 2022.11 + +- [x] **SGI** can access Gravity **Home** Directory at `/gravity/home` +- [x] **SGI** can access the Internet directly now 🏄‍ +- [x] **SGI** has a disk quota for each user, which is [**10G+1TB**](/Basic/Resource_Monitor?id=disk-quota) + +### 2022.7 + +- [x] [**Gravity Homepage**](https://jupyter.gravity.sjtu.edu.cn/) is ready! 🎉🎉🎉 +- [x] add [**FileServer**](/Basic/Data_Transfer?id=fileserver), support download/**share** files 🎉🎉🎉 ([File Sharing DEMO](https://gravity.sjtu.edu.cn/demo/)) +- [x] add [**Two-Factor Authentication (2FA)**](/Basic/Login?id=web-login)🔒 for our website +- [x] add [**NEW Spawner for Jupyter**](Coding?id=start-a-server), now you can select and **input options** +- [x] support **change password** / **reset password**(if you forgot your password) on [Gravity Authentication](/Basic/Account?id=change-password) +- [x] NEW in *Jupyter*: [resource monitor](https://gravity-doc.github.io/Basic/Jupyter.html#resource-monitor), [Diagram](https://gravity-doc.github.io/Basic/Jupyter.html#diagram), [Variable Inspector](https://gravity-doc.github.io/Basic/Jupyter.html#variable-inspector), [**HDF5 viewer**](https://gravity-doc.github.io/Basic/Jupyter.html#hdf5-viewer), [**`` selection**](https://gravity-doc.github.io/Basic/Jupyter.html#tab-selection), [Retrolab](https://gravity-doc.github.io/Basic/Jupyter.html#retrolab), *dracula* theme, [中文语言包](https://gravity-doc.github.io/Basic/Jupyter.html#chinese). Remove some unused kernels +- [x] add **HTTP proxy server**, now you can [**🏄‍surf the Internet**](https://gravity-doc.github.io/Software/Proxy.html#surf-the-internet) such as `git/curl/wget` when using *Jupyter/VScode* on *computing nodes*. Meanwhile, it will automatically *speedup*⚡ `git clone/pull/push` from/to **Github** **[ NO WARRANTY ]** +- [x] add **HTTP cache** to accelerate *Jupyter/VScode* +- [x] add [**Monitor**](https://jupyter.gravity.sjtu.edu.cn/status/?theme=dark) to check Gravity status +- [x] add [**SpeedTest**](https://jupyter.gravity.sjtu.edu.cn/speedtest/) to test connection speed between you and Gravity + +### 2021.9 + +- [x] install [`Julia, Go, R, Rust, C, C++, Fortran` **kernels**](https://gravity-doc.github.io/Basic/JupyterHub.html#choose-a-kernel-environment) into *Jupyterlab* and install many useful packages in `R, Julia` +- [x] make [*VScode*](https://gravity-doc.github.io/Basic/JupyterHub.html#vscode) available on computing nodes + +### 2021.8 + +- [x] reconstruct the [*Gravity Documentation*](https://gravity-doc.github.io/) + +### 2021.7 + +- [x] [add entry of `VScode`](https://gravity-doc.github.io/Basic/JupyterHub.html#vscode) in jupyterlab, and support **C/C++, Fortran, Go, R, Julia, Python** if you choose *VScode [sys]* in Jupyterlab +- [x] run `jupyter-lab/notebook` from both *login* and *computing nodes* via **[JupyterHub](https://gravity.sjtu.edu.cn/)** +- [x] install **several extensions** on Jupyterlab (ie. git, drawio, execute-time, resource-monitor, jupyterlab-lsp) + +### 2021.6 +- [x] [enable priority of PBS queue](https://gravity-doc.github.io/Basic/Job.html#ordinary-queues) +- [x] [support **Data Sharing** service](https://jupyter.gravity.sjtu.edu.cn/share/) (need to contact admins to use it) +- [x] **limit** each user resource usage on *login nodes* (**20 cores + 50 GB**) +- [x] automatically **ban IP** with too many failed login attempts ( *ssh* + *JupyterHub*) + +### 2021.5 +- [x] [speedup git ⚡](https://gravity-doc.github.io/Software/Proxy.html#speedup-git) (including `git clone, git pull, etc.`) + +- [x] add and improve command to query CPU time (expense) and Disk usage + > `icfsquota` + > `cpuquota` + +- [x] [enable the **E-mail notification** of PBS ](https://gravity-doc.github.io/Basic/Job.html#complete-example) + + > In your **PBS script**, you can add `#PBS -m abe` and `#PBS -M @gravity.sjtu.edu.cn` + +- [x] shorten the waiting time when log in Gravity From 793901b5fddbb383d1e2f3a4bbb4ef9b7f135ee8 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 18:10:48 +0800 Subject: [PATCH 13/23] add highlight, fix search --- docs/index.html | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/index.html b/docs/index.html index d0ebdbd..b53fe9c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -27,10 +27,12 @@ repo: 'https://github.com/gravity-doc/gravity-doc.github.io/', logo: 'https://git.gravity.sjtu.edu.cn/gravity-hpc/gravity-doc/-/raw/master/docs/images/logo_DOA_mini.png', search: { + maxAge: 2592000000, paths: 'auto', placeholder: 'Search / 搜索关键词', noData: 'No results / 无结果 ╥﹏╥...', - depth: 2 + depth: 6, + hideOtherSidebarContent: true }, busuanzi: true, count:{ @@ -43,15 +45,15 @@ - + + - + - - + @@ -59,8 +61,9 @@ - + + From f230f347ad397157cf2c21085b83303b36e89e92 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 18:29:47 +0800 Subject: [PATCH 14/23] fix search and add git pull in script --- convert_to_docsify.sh | 7 +++++++ docs/index.html | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/convert_to_docsify.sh b/convert_to_docsify.sh index 916d9e5..93d1b33 100644 --- a/convert_to_docsify.sh +++ b/convert_to_docsify.sh @@ -11,6 +11,13 @@ if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then exit 1 fi +# switch to docsify branch +git checkout docsify +# undo any changes +git checkout . +# pull latest changes +git pull + source_file="docs/index.md" target_file="docs/README.md" diff --git a/docs/index.html b/docs/index.html index b53fe9c..985d666 100644 --- a/docs/index.html +++ b/docs/index.html @@ -32,22 +32,20 @@ placeholder: 'Search / 搜索关键词', noData: 'No results / 无结果 ╥﹏╥...', depth: 6, - hideOtherSidebarContent: true }, busuanzi: true, count:{ countable:true, fontsize:'0.9em', color:'rgb(90,90,90)', - language:'chinese' + language:'english' } } - - + From 7b50e950a346dbb5b858cc6f07f757c6c6fbde54 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 18:54:41 +0800 Subject: [PATCH 15/23] enhance search --- docs/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.html b/docs/index.html index 985d666..4540ac9 100644 --- a/docs/index.html +++ b/docs/index.html @@ -32,6 +32,7 @@ placeholder: 'Search / 搜索关键词', noData: 'No results / 无结果 ╥﹏╥...', depth: 6, + hideOtherSidebarContent: false }, busuanzi: true, count:{ From e79ad5ec59972ce9c650fa0529e6a404de6a497f Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 20:49:41 +0800 Subject: [PATCH 16/23] use local JS cache, add copy code --- docs/index.html | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/index.html b/docs/index.html index 4540ac9..98f088e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -5,8 +5,8 @@ Gravity Documentation - - + + @@ -21,8 +21,8 @@ loadSidebar: true, loadNavbar: true, mergeNavbar: true, - maxLevel: 4, - subMaxLevel: 2, + maxLevel: 6, + subMaxLevel: 4, name: 'Gravity Documentation', repo: 'https://github.com/gravity-doc/gravity-doc.github.io/', logo: 'https://git.gravity.sjtu.edu.cn/gravity-hpc/gravity-doc/-/raw/master/docs/images/logo_DOA_mini.png', @@ -40,32 +40,38 @@ fontsize:'0.9em', color:'rgb(90,90,90)', language:'english' - } + }, + copyCode: { + buttonText: 'Copy to clipboard', + errorText: 'Error', + successText: 'Copied', + }, } - - - - - + + + + + - + - + - - - + + + + - + From bc2e2563695ed6bbadaa641d423acb0fc494f037 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 21:04:05 +0800 Subject: [PATCH 17/23] success to make search perfect the reason is Above line is too bad!!!!!!! --- docs/index.html | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/index.html b/docs/index.html index 98f088e..c8a56b6 100644 --- a/docs/index.html +++ b/docs/index.html @@ -7,7 +7,7 @@ - +
@@ -22,10 +22,11 @@ loadNavbar: true, mergeNavbar: true, maxLevel: 6, - subMaxLevel: 4, + subMaxLevel: 6, name: 'Gravity Documentation', repo: 'https://github.com/gravity-doc/gravity-doc.github.io/', logo: 'https://git.gravity.sjtu.edu.cn/gravity-hpc/gravity-doc/-/raw/master/docs/images/logo_DOA_mini.png', + search: 'auto', search: { maxAge: 2592000000, paths: 'auto', @@ -53,8 +54,6 @@ - - @@ -62,8 +61,6 @@ - - From 290c311842da790a1714e5a667087ecbb52813b9 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 18 Nov 2023 21:23:05 +0800 Subject: [PATCH 18/23] adjust layout of coverpage and navbar --- docs/_coverpage.md | 1 + docs/_navbar.md | 2 +- docs/index.html | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 243b9f0..6155cb6 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,3 +1,4 @@ +[Gravity Home](https://jupyter.gravity.sjtu.edu.cn/) [Get started](/README.md) diff --git a/docs/_navbar.md b/docs/_navbar.md index a6cbb16..48e0e37 100644 --- a/docs/_navbar.md +++ b/docs/_navbar.md @@ -1,3 +1,3 @@ - [Cheatsheet](/QuickStart?id=cheatsheet-📜) -- [Gravity Homepage](https://jupyter.gravity.sjtu.edu.cn/) +- [Gravity Home](https://jupyter.gravity.sjtu.edu.cn/) - [Department of Astronomy (SJTU)](http://astro.sjtu.edu.cn/) \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index c8a56b6..da836e6 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,14 +3,24 @@ Gravity Documentation - + + -
+
Loading (/≧▽≦)/... ...
- - - - - + - - - + + + + + - - + + + + From 0180b251c43573ab673e2bd78f2516b02cd7c3c7 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Mon, 20 Nov 2023 16:45:20 +0800 Subject: [PATCH 20/23] add some future plans --- docs/Updates.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/Updates.md b/docs/Updates.md index 2181fd5..865d526 100644 --- a/docs/Updates.md +++ b/docs/Updates.md @@ -5,8 +5,14 @@ title: Updates ## Future ✨ -- [ ] support container (Singularity) -- [ ] switch to Slurm (PBS will be **Deprecated**) +- [ ] support container (*Singularity* + *Kubernetes*) + - [ ] migrate the entire Jupyter to Kubernetes +- [ ] switch to Slurm (*PBS* will be **Deprecated**) + - [ ] enable resource limit on computing nodes (use `cgroup`) +- [ ] update *NIS* to *LDAP* authentication (*NIS* will be **Deprecated**) + - [ ] Automatically handle new user registration and verification +- [ ] enable **load-balancing** for Gravity website +- [ ] add **real-time monitoring** and **alarm** for sensitive files modification ## Done 🥳 From 2d739c2ba9aacd6f59c79912a1a4ef7825497265 Mon Sep 17 00:00:00 2001 From: Jiaxin Han Date: Tue, 21 Nov 2023 11:51:02 +0800 Subject: [PATCH 21/23] Update Login.md --- docs/Basic/Login.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/Basic/Login.md b/docs/Basic/Login.md index 52c4d06..585eb93 100644 --- a/docs/Basic/Login.md +++ b/docs/Basic/Login.md @@ -17,11 +17,16 @@ If you fail to login, please do not try many times. Otherwise your IP will be ** ### 1. generate `ssh-key` -To generate a [*ssh-key pair*](https://wiki.archlinux.org/title/SSH_keys_(%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87)), open a terminal (*Linux/Windows/Mac*) (it is recommended to set a *passphrase* for your *private key*, more secure😊): +To generate a [*ssh-key pair*](https://wiki.archlinux.org/title/SSH_keys_(%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87)), open a terminal (*Linux/Windows/Mac*) ```bash ➜ ssh-keygen -t rsa -b 4096 -C "lalala@Gravity" +```tip +You will be asked to input a passphrase when generating your key. +It is highly recommended to use an non-empty passphrase. +``` + # Output👇 Generating public/private rsa key pair. Enter file in which to save the key (/home/lalala/.ssh/id_rsa): /home/lalala/.ssh/id_rsa_Gravity @@ -409,4 +414,4 @@ You cannot access **SGI** directly, you need to log in **Gravity** first, then o ssh sgi ``` -Here we go 🎉 \ No newline at end of file +Here we go 🎉 From 8bff4b2f3d107bd46f8cb20c5b17f6d780b58bc9 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 13 Apr 2024 21:32:44 +0800 Subject: [PATCH 22/23] officially transfer to docsify. fix some bugs of display/markdown indentation equally transfer tip/note/danger/warning block add more info in Job section --- README.md | 2 +- admin/websample.md | 39 ++--- docs/About.md | 6 +- docs/Advanced/Gitlab.md | 16 +- docs/Advanced/Globus.md | 12 +- docs/Advanced/README.md | 3 - docs/Advanced/VNC.md | 41 ++--- docs/Advanced/X11_Tunnel.md | 23 +-- docs/Advanced/jupyter_notebook.md | 14 +- docs/Advanced/status_website.md | 4 - docs/Basic/Account.md | 46 +++--- docs/Basic/Coding.md | 73 ++++----- docs/Basic/Data_Transfer.md | 18 +- docs/Basic/Job.md | 172 ++++++++++---------- docs/Basic/Login.md | 110 ++++++------- docs/Basic/README.md | 3 - docs/Basic/Resource_Monitor.md | 30 ++-- docs/MISC/FAQ.md | 16 +- docs/MISC/Issues.md | 32 ++-- docs/MISC/README.md | 4 - docs/Policy.md | 4 - docs/QuickStart.md | 14 +- docs/README.md | 23 ++- docs/Software/Proxy.md | 32 ++-- docs/Software/README.md | 3 - docs/Software/Software_Installed.md | 32 ++-- docs/Software/User_Software_Installation.md | 12 +- docs/Updates.md | 4 - docs/_coverpage.md | 2 +- docs/index.html | 108 ++++++------ 30 files changed, 391 insertions(+), 507 deletions(-) diff --git a/README.md b/README.md index 8dcf58f..5780554 100644 --- a/README.md +++ b/README.md @@ -36,4 +36,4 @@ We would appreciate it if all publications that include results generated using - **本工作的计算使用了上海交通大学天文系Gravity高性能计算集群** - **This work made use of the Gravity Supercomputer at the Department of Astronomy, Shanghai Jiao Tong University.** - +![logo_DOA_large.png](./docs/images/logo_DOA_large.png) diff --git a/admin/websample.md b/admin/websample.md index 5a1d9ca..5a8c9ca 100644 --- a/admin/websample.md +++ b/admin/websample.md @@ -1,37 +1,30 @@ ---- -sort: 10086 ---- # Samples for web - - ## 主题和主题使用说明 + [Theme](https://rundocs.io) ## examples for 各种框框 -```note -### This is a note - -Markdown is supported, Text can be **bold**, _italic_, or ~~strikethrough~~. [Links](https://github.com) should be blue with no underlines - -`inline code` +> [!NOTE] +> ### This is a note +> Markdown is supported, Text can be **bold**, _italic_, or ~~strikethrough~~. [Links](https://github.com) should be blue with no underlines +> `inline code` +> [`inline code inside link`](#) +> -[`inline code inside link`](#) -``` +> [!TIP] +> It's bigger than a bread box. +> -```tip -It's bigger than a bread box. -``` +> [!WARNING] +> Strong prose may provoke extreme mental exertion. Reader discretion is strongly advised. +> -```warning -Strong prose may provoke extreme mental exertion. Reader discretion is strongly advised. -``` - -```danger -Mad scientist at work! -``` +> [!ATTENTION] +> Mad scientist at work! +> ## 代码 diff --git a/docs/About.md b/docs/About.md index 9c632f2..dcbf1e3 100644 --- a/docs/About.md +++ b/docs/About.md @@ -1,9 +1,5 @@ ---- -sort: 2 -title: System Specs ---- # Hardware - +![hardware.jpg](./images/hardware.jpg) ### Login nodes diff --git a/docs/Advanced/Gitlab.md b/docs/Advanced/Gitlab.md index 75a7f50..b280e16 100644 --- a/docs/Advanced/Gitlab.md +++ b/docs/Advanced/Gitlab.md @@ -1,21 +1,17 @@ ---- -sort: 6 -title: Gitlab ---- -```note -There is a *self-hosted* **Gitlab** instance at **Gravity**: [https://git.gravity.sjtu.edu.cn](https://git.gravity.sjtu.edu.cn) -``` +> [!NOTE] +> There is a *self-hosted* **Gitlab** instance at **Gravity**: [https://git.gravity.sjtu.edu.cn](https://git.gravity.sjtu.edu.cn) +> ## Login [Open Gitlab on browser](https://git.gravity.sjtu.edu.cn/), there are **two ways** to login: 1. Your *Gravity* username and password - + ![gitlab-passwd-login.png](../images/Advanced/gitlab-passwd-login.png) 2. Use *Authelia* authorization at bottom of login page - - + ![gitlab-authelia-login.png](../images/Advanced/gitlab-authelia-login.png) + ![gitlab-authelia-confirm.png](../images/Advanced/gitlab-authelia-confirm.png) ## SSH Access diff --git a/docs/Advanced/Globus.md b/docs/Advanced/Globus.md index 9184bf2..1754a12 100644 --- a/docs/Advanced/Globus.md +++ b/docs/Advanced/Globus.md @@ -1,7 +1,3 @@ ---- -sort: 5 -title: Globus ---- # Globus [Globus](https://www.globus.org) is quite popular to transfer large scientific data. @@ -27,15 +23,15 @@ Here we provide the routine to install Globus personal version. You can also ch ./globusconnectpersonal -setup ``` - + ![globus-setup1.png](../images/Advanced/globus-setup1.png) A long link will be provided. Copy and open the above link in your browser. After sign in your globus account, an authorization code will be provided. - + ![globus-setup2.png](../images/Advanced/globus-setup2.png) Input this code in your terminal, type the name for this Endpoint (Gravity2 in the above case) and finish the setup. - + ![globus-setup3.png](../images/Advanced/globus-setup3.png) 5. Run Globus personal without GUI. ```bash @@ -44,7 +40,7 @@ Here we provide the routine to install Globus personal version. You can also ch Check whether the Endpoint is correctly appeared in your Globus Endpoint Collections. - + ![globus-setup4.png](../images/Advanced/globus-setup4.png) 6. Stop Globus personal. Please close your Globus personal when your transfer task is end, for safety and resources saving reason. diff --git a/docs/Advanced/README.md b/docs/Advanced/README.md index 8c5e09f..6b75f5a 100644 --- a/docs/Advanced/README.md +++ b/docs/Advanced/README.md @@ -1,6 +1,3 @@ ---- -sort: 2 ---- # Advanced diff --git a/docs/Advanced/VNC.md b/docs/Advanced/VNC.md index 87b8909..1e368f2 100644 --- a/docs/Advanced/VNC.md +++ b/docs/Advanced/VNC.md @@ -1,20 +1,16 @@ ---- -sort: 2 -title: VNC ---- - -```note -Considering security, *Gravity* closes ports except **22, 80, 443**. -Therefore, you need **ssh tunnel** to connect other service. -``` -```tip -VNC is a fast tool to connect to a graphical desktop on the server. -You should add **5900** to your port. That is the **real** port. For example: -`vncserver :66` -Then, the real port VNC uses is **`66 + 5900 = 5966`** -When you use *SSH tunnel*, please connect to the real port, otherwise you will fail🤣 -``` +> [!NOTE] +> Considering security, *Gravity* closes ports except **22, 80, 443**. +> Therefore, you need **ssh tunnel** to connect other service. +> + +> [!TIP] +> VNC is a fast tool to connect to a graphical desktop on the server. +> You should add **5900** to your port. That is the **real** port. For example: +> `vncserver :66` +> Then, the real port VNC uses is **`66 + 5900 = 5966`** +> When you use *SSH tunnel*, please connect to the real port, otherwise you will fail🤣 +> ## Getting started @@ -35,11 +31,10 @@ Take a note of the port number (37 in this case). You will need to connect to th 请记住这里显示的端口号（图中是37）。 -```tip - You will be prompted to setup a password the first time you do this. This will be the password you use to connect to vnc. This password is separate from the ssh password. - - 第一次开启vnc时在输入命令后会先让你设置密码，之后再使用或者开启新的端口都会使用默认密码. -``` +> [!TIP] +> You will be prompted to setup a password the first time you do this. This will be the password you use to connect to vnc. This password is separate from the ssh password. +> 第一次开启vnc时在输入命令后会先让你设置密码，之后再使用或者开启新的端口都会使用默认密码. +> - checking for running servers @@ -77,11 +72,11 @@ You will need to use a vnc client to connect to your server. On Linux, you can u In Remmina, select "new connection". In protocol, select VNC. In the Basic tab, fill in the server address and vncserver port, your username and vnc password. - +![VNC_1.png](../images/Advanced/VNC_1.png) For security reasons, you can only connect to vnc through ssh-tunnel. This can be done in the "SSH Tunnel" tab. Select "Enable SSH tunnel", input your username, and select Public key authentication. - +![VNC_2.png](../images/Advanced/VNC_2.png) 你可以使用喜欢的vnc客户端连到前面建立的vncserver。在客户端中输入服务器地址和vncserver端口号。由于安全原因，还需要选择打开ssh-tunnel选项。 diff --git a/docs/Advanced/X11_Tunnel.md b/docs/Advanced/X11_Tunnel.md index a144a65..b1e51b2 100644 --- a/docs/Advanced/X11_Tunnel.md +++ b/docs/Advanced/X11_Tunnel.md @@ -1,19 +1,14 @@ ---- -sort: 1 -title: X11 Tunnel ---- # X11 Tunnel -```note -Running graphical software on Gravity with [X11 tunneling](https://en.wikipedia.org/wiki/X_Window_System) -Actually, sometimes using **[VNC](/Advanced/VNC)** is better 😜 -X11 allows you to run a **graphical application** on the *login* or *computing node*, and to control it from your local computer. For example, you can **view images** in the *Gravity* using -- `eog` -- `display` - -Or, you can even run a browser -- `firefox` -``` +> [!NOTE] +> Running graphical software on Gravity with [X11 tunneling](https://en.wikipedia.org/wiki/X_Window_System) +> Actually, sometimes using **[VNC](/Advanced/VNC)** is better 😜 +> X11 allows you to run a **graphical application** on the *login* or *computing node*, and to control it from your local computer. For example, you can **view images** in the *Gravity* using +> `eog` +> `display` +> Or, you can even run a browser +> `firefox` +> ## Windows diff --git a/docs/Advanced/jupyter_notebook.md b/docs/Advanced/jupyter_notebook.md index f99fc2a..d884c05 100644 --- a/docs/Advanced/jupyter_notebook.md +++ b/docs/Advanced/jupyter_notebook.md @@ -1,13 +1,9 @@ ---- -sort: 3 -title: Jupyter Lab/Notebook ---- -```tip -We recommend you to use [*JupyterHub*](/Basic/Coding.html), it's more easier and convenient 😜 -You can also **use your own environment** in [*JupyterHub*](/Basic/Coding?id=create-new-environmentkernel) -Here, we introduce how to connect *jupyter-notebook/lab* on *Gravity* (login or computing nodes) 👇 -``` +> [!TIP] +> We recommend you to use [*JupyterHub*](/Basic/Coding.html), it's more easier and convenient 😜 +> You can also **use your own environment** in [*JupyterHub*](/Basic/Coding?id=create-new-environmentkernel) +> Here, we introduce how to connect *jupyter-notebook/lab* on *Gravity* (login or computing nodes) 👇 +> 1. Prepare your own `jupyter-notebook` (`jupter-lab`) or just use system's module 前期准备：拥有自己的conda环境，并且环境中安装配置了jupyter netobook；或者直接使用系统自带的 diff --git a/docs/Advanced/status_website.md b/docs/Advanced/status_website.md index 1bdeb4f..01a6fee 100644 --- a/docs/Advanced/status_website.md +++ b/docs/Advanced/status_website.md @@ -1,7 +1,3 @@ ---- -sort: 4 -title: Status/Statistics Website ---- # Status/Statistics Website - check your **cost/Job history** on this website. diff --git a/docs/Basic/Account.md b/docs/Basic/Account.md index d445bff..36a33db 100644 --- a/docs/Basic/Account.md +++ b/docs/Basic/Account.md @@ -1,14 +1,9 @@ ---- -sort: 1 -title: Account ---- ## Apply for a NEW Account -```note -请仔细阅读[用户协议](/Policy)，并征求导师同意。 - -Please make sure you have read the [user policy](/Policy), and get consent of your sponsor/supervisor before application. -``` +> [!NOTE] +> 请仔细阅读[用户协议](/Policy)，并征求导师同意。 +> Please make sure you have read the [user policy](/Policy), and get consent of your sponsor/supervisor before application. +> Use [👉**this link**👈](https://forms.office.com/Pages/ResponsePage.aspx?id=-f5HFYhWBkCG2kSQ-Sc_lW_CRAlVS3tEtz1OEMF6VRNUMUNLOUVOSFhSMTJSTzJSUVozQldJVlRDUy4u) to fill out the form, we will send an e-mail📧 to you after a while. @@ -21,19 +16,17 @@ The initial password is included. 2. In web, you can change password by clicking *Reset password?* to change your password. More details are here 👉 [**2FA Authentication**](/Basic/Login?id=web-login) -```tip -- Note that it may take a few minutes before the password change is synchronized between the terminal and the web login. - -- 更改密码后，新的密码将会需要几分钟在terminal和web之间得到同步。 -``` - -```warning -- Please change your password immediately the first time you login to your account. -- Never write your password explicitly to others, in emails or chats. The administrators will never ask you for your password and never give your password to anyone. +> [!TIP] +> Note that it may take a few minutes before the password change is synchronized between the terminal and the web login. +> 更改密码后，新的密码将会需要几分钟在terminal和web之间得到同步。 +> -- 请在第一次登陆后立即修改初始密码。 -- 请不要在邮件、微信等通信里向任何人提供密码。管理员维护账号也从**不会**需要你的密码。 -``` +> [!WARNING] +> Please change your password immediately the first time you login to your account. +> Never write your password explicitly to others, in emails or chats. The administrators will never ask you for your password and never give your password to anyone. +> 请在第一次登陆后立即修改初始密码。 +> 请不要在邮件、微信等通信里向任何人提供密码。管理员维护账号也从**不会**需要你的密码。 +> ## Forgot password @@ -60,12 +53,11 @@ If you want to change your email📧, please [contact us](/?id=contact), provide ## SGI Account -```tip -1. **SGI** is a **standalone** machine, and it has a different OS and disk system. -2. **SGI** use *slurm* rather than *pbs* as the job scheduler. - -**NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. -``` +> [!TIP] +> **SGI** is a **standalone** machine, and it has a different OS and disk system. +> **SGI** use *slurm* rather than *pbs* as the job scheduler. +> **NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. +> You cannot access **SGI** directly, you need to log in **Gravity** first, then on login nodes, use: diff --git a/docs/Basic/Coding.md b/docs/Basic/Coding.md index 8eb8dd3..d860486 100644 --- a/docs/Basic/Coding.md +++ b/docs/Basic/Coding.md @@ -1,25 +1,18 @@ ---- -sort: 6 -title: Coding ---- -```tip -[**JupyterHub**](https://jupyterhub.readthedocs.io/) is an entry to access jupyter-notebook/lab or other service ✨ -- Jupyter-notebook/lab is the **easiest way** to coding 🥳 -- VScode is the **best editor** in the Universe 💪 -- You can use jupyter-notebook/lab or VScode on **both login and computing nodes** 😎 - -You can access *Jupyter* on our website: -- [Jupyter](https://jupyter.gravity.sjtu.edu.cn) (needs **2FA**) - -- PS. We have enabled **Two-Factor Authentication (2FA)**🎉🎉🎉 More details are [Gravity Authentication](/Basic/Login?id=web-login) -``` - -```warning -**DO NOT** run heavy program on *login nodes* ❌🙅❌ - -If you fail to log in, **DO NOT** try many times, your IP will **be banned** for a period of time 😭 -``` +> [!TIP] +> [**JupyterHub**](https://jupyterhub.readthedocs.io/) is an entry to access jupyter-notebook/lab or other service ✨ +> Jupyter-notebook/lab is the **easiest way** to coding 🥳 +> VScode is the **best editor** in the Universe 💪 +> You can use jupyter-notebook/lab or VScode on **both login and computing nodes** 😎 +> You can access *Jupyter* on our website: +> [Jupyter](https://jupyter.gravity.sjtu.edu.cn) (needs **2FA**) +> PS. We have enabled **Two-Factor Authentication (2FA)**🎉🎉🎉 More details are [Gravity Authentication](/Basic/Login?id=web-login) +> + +> [!WARNING] +> **DO NOT** run heavy program on *login nodes* ❌🙅❌ +> If you fail to log in, **DO NOT** try many times, your IP will **be banned** for a period of time 😭 +> ## Jupyter @@ -44,9 +37,9 @@ input *username* + *password* + *TOTP* click *Start My Server* or *Named Servers* -```warning -If you use *Named Servers*, its name **CAN NOT** contain any space😢! -``` +> [!WARNING] +> If you use *Named Servers*, its name **CAN NOT** contain any space😢! +> choose one *Job queue* @@ -111,16 +104,12 @@ python -m ipykernel install --user --name myenv Now, refresh your *Jupyter*, and you can see your newly configured kernel. Select it for your notebook. enjoy 🎉 -```tip - - The kernel name is also displayed in the top right corner of your notebook. To see the exact path for this environment: - - `import sys` - - `sys.executable` - - This will show the path of the python executable for the current notebook. -``` +> [!TIP] +> The kernel name is also displayed in the top right corner of your notebook. To see the exact path for this environment: +> `import sys` +> `sys.executable` +> This will show the path of the python executable for the current notebook. +> ### Clone environment/kernel @@ -415,10 +404,10 @@ As is well-known, *VScode* is the best editor for coding in the world 😜 > 3. 💯 support C/C++, Fortran, R, Go, Julia, Python etc. > 4. 😎 install any extension you like, **customize** your own vscode! -```tip -1. **Only if** you stop the server from JupyterHub, can the VScode really stop. -2. It's better to use **Chrome** or **Edge** rather than ~~Firefox~~. -``` +> [!TIP] +> **Only if** you stop the server from JupyterHub, can the VScode really stop. +> It's better to use **Chrome** or **Edge** rather than ~~Firefox~~. +> ### Start @@ -525,7 +514,7 @@ Click *SSH TARGETS* button, then click *login02* or *login01* or *computing*, en ## PyCharm (remote) -```tip -**PyCharm** is a powerful IDE for Python. The professional version also supports **remote** mode. So you can run your code on *Gravity*. -Setting is similar to *VScode (remote)*, you can refer to it. -``` \ No newline at end of file +> [!TIP] +> **PyCharm** is a powerful IDE for Python. The professional version also supports **remote** mode. So you can run your code on *Gravity*. +> Setting is similar to *VScode (remote)*, you can refer to it. +> \ No newline at end of file diff --git a/docs/Basic/Data_Transfer.md b/docs/Basic/Data_Transfer.md index ea7d7d0..a381972 100644 --- a/docs/Basic/Data_Transfer.md +++ b/docs/Basic/Data_Transfer.md @@ -1,7 +1,3 @@ ---- -sort: 5 -title: Data Transfer ---- # Data Transfer @@ -11,13 +7,11 @@ When one is making plots or editing codes, it might be more convenient to use techniques with graphical user interface support. ## FileServer -```note -You need to `copy`/`move` your files into `~/.fileServer` to use *FileServer*. - -If you need to **share massive data**, feel free to [contact admin🧙‍](/?id=contact) - -This service is provided for sharing data on gravity to your collaborators. Please do not abuse it to share your personal/non-academic files. -``` +> [!NOTE] +> You need to `copy`/`move` your files into `~/.fileServer` to use *FileServer*. +> If you need to **share massive data**, feel free to [contact admin🧙‍](/?id=contact) +> This service is provided for sharing data on gravity to your collaborators. Please do not abuse it to share your personal/non-academic files. +> ### access We have enabled a **File Server** on Gravity 🎉🎉🎉 @@ -51,7 +45,7 @@ Because of the GUI of jupyter, you can easily transfer your data, as long as you [安装 Filezilla 软件](https://filezilla-project.org/)后，可以双击打开该软件，按照下图进行设置，完成后单击快速链接或者回车可以登录ftp 站点。 输入数据集群IP 用户名 密码和端口号，如下图所示： - +![Login__04_filezilla.png](../images/Basic/Login__04_filezilla.png) ## scp and rsync Linux/Unix/Mac 用户可以使用命令行工具在集群和您自己的计算机之间传输数据。下列语句会将文件夹`data/`上传到主文件夹的`tmp/`。 diff --git a/docs/Basic/Job.md b/docs/Basic/Job.md index 6f22abf..7a847d7 100644 --- a/docs/Basic/Job.md +++ b/docs/Basic/Job.md @@ -1,14 +1,23 @@ ---- -sort: 3 -title: Job ---- -```warning -You **should** run your job via PBS (Torque). -You can **ONLY** do some **simple** jobs on the **login nodes**, otherwise, you might crash the system and be *banned*. +> [!WARNING] +> You **should** run your job via PBS (Torque). +> You can **ONLY** do some **simple** jobs on the **login nodes**, otherwise, you might crash the system and be *banned*. +> Please only request resources that you actually need. **Do not** use more than or less than the number of cores you requested. +> + +> [!TIP] +> Here are some useful commands for you to manage your jobs. +> +> +> | command | explanation | +> | :--------: | :--------: | +> | `qstat -tan` | view all of the jobs in details | +> | `qstat -tan \| grep ` | view all of the jobs in details of *\* | +> | `pestat` | view all of nodes status | +> | `showq` | summary of running/waiting jobs | +> | `qsub -I -q debug -l nodes=1:ppn=72 ` | submit an interactive job | +> | `qsub pbsJob.sh` | submit a job via script | -Please only request resources that you actually need. **Do not** use more than or less than the number of cores you requested. -``` ## Queues @@ -87,18 +96,16 @@ To submit the job, qsub example.qsub ``` -```tip -Please only request resources that you actually need. In the above example, a full node (72 cores) is requested. However, if your job uses fewer cores, you should only request that many cores. For example -`#PBS -l nodes=1:ppn=8` -to request 8 cores on a single node. - -If your job uses OpenMP, you should set the number of threads to match your requested cores. For example, when requesting 8 cores for an OpenMP job, remember to also put the following in your job script before your final program line: -`export OMP_NUM_THREADS=8` -. - -If you are running a serial job, please only request **one** core: -`#PBS -l nodes=1:ppn=1` -``` +> [!TIP] +> Please only request resources that you actually need. In the above example, a full node (72 cores) is requested. However, if your job uses fewer cores, you should only request that many cores. For example +> `#PBS -l nodes=1:ppn=8` +> to request 8 cores on a single node. +> If your job uses OpenMP, you should set the number of threads to match your requested cores. For example, when requesting 8 cores for an OpenMP job, remember to also put the following in your job script before your final program line: +> `export OMP_NUM_THREADS=8` +> . +> If you are running a serial job, please only request **one** core: +> `#PBS -l nodes=1:ppn=1` +> ## Complete example @@ -143,20 +150,17 @@ qsub -l nodes=1:ppn=4,mem=8gb,walltime=12:00:00 -q normal -I You will be logged onto the assigned node once the above job get running, and you can start to run your actual computation interactively. -```warning -- Please use the system responsibly and only start an interactive job when necessary. - -- Do not use more resources than requested, otherwise you might affect other jobs or crash the node. - -- CPU time will be charged in the same way as normal jobs. Remember to log out of the computing node to end the interactive job. - -``` +> [!WARNING] +> Please use the system responsibly and only start an interactive job when necessary. +> Do not use more resources than requested, otherwise you might affect other jobs or crash the node. +> CPU time will be charged in the same way as normal jobs. Remember to log out of the computing node to end the interactive job. +> Reference: [https://www.msi.umn.edu/content/interactive-queue-use-qsub](https://www.msi.umn.edu/content/interactive-queue-use-qsub) -```tip -The following PBS environment variable list is output by `env | grep PBS`, which maybe useful in your scripts. -``` +> [!TIP] +> The following PBS environment variable list is output by `env | grep PBS`, which maybe useful in your scripts. +> ```bash PBS_VERSION=TORQUE-6.0.2 @@ -674,60 +678,60 @@ Use `qstat -u ` to show the jobs submitted by one user. ### References -```note -`qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` -`[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` -`[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` -`[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` -`[-W additional_attributes]` `[-z]` `[script]` --a date_time ： delay the job with time [[[[CC]YY]MM]DD]hhmm[.SS] --c interval ： 定义作业的检查点间隔，如果机器不支持检查点，则忽略此选项。 --C directive_prefix ：treat all lines begin with directive_prefix as qsub option. Otherwise, qsub option begins with '#PBS' --e path ：specify error file --I ：interactive --j join ：join output and error file. --l resource_list ： define resource list as follow -walltime=N ： wall time in unit of second, or in the form of hh:mm:ss -mem=N[K|M|G][B|W]：define memory usage -nodes=N:ppn=M ：define number of nodes N and processes per node M. --m mail_options ：mail_option为a：作业abort时给用户发信；为b：作业开始运行发信；为e：作业结束运行时发信。若无此选项，默认为a。 --M user_list ： mail addresss --N name ： jobname, less than 15 characters. --o path ： specify output file --p priority ： adjust priority, [-1024，1023]，default 0. --q destination ： specify which queue to use --r y|n ： 指明作业是否可运行，y为可运行，n为不可运行。 --S shell ： specify the SHELL to use, full path. --u user_list ： specify the user to run the job --v variable_list ： specify the environment variable list to export to this job. --V ： export all environment variable to this job. --W additional_attributes ： --z ： do not show JOB_ID information after submission -``` - -```note -`qstat` `[-f JOB_ID]` `[-a]` `[-i]` `[-n]` `[-s]` `[-R]` `[-Q [QUEUE]]` `[-q]` `[-B]` `[-u USER]` --f JOB_ID : specify the job --a : list all the jobs --i : list all non-running jobs --r : list all the running jobs --n : list the nodes assigned to jobs --s : 列出队列管理员与scheduler所提供的建议 --R : 列出磁盘预留信息 --Q [QUEUE] : list queue status --q [QUEUE] : list queue status --B : list PBS Server information --u USER : list the jobs for USER -``` +> [!NOTE] +> `qsub` `[-a date_time]` `[-c interval]` `[-C directive_prefix]` `[-e path]` +> `[-I]` `[-j join]` `[-k keep]` `[-l resource_list]` `[-m mail_options]` +> `[-M user_list]` `[-N name]` `[-o path]` `[-p priority]` `[-q destination]` +> `[-r c]` `[-S path_list]` `[-u user_list]` `[-v variable_list]` `[-V]` +> `[-W additional_attributes]` `[-z]` `[script]` +> a date_time ： delay the job with time [[[[CC]YY]MM]DD]hhmm[.SS] +> c interval ： 定义作业的检查点间隔，如果机器不支持检查点，则忽略此选项。 +> C directive_prefix ：treat all lines begin with directive_prefix as qsub option. Otherwise, qsub option begins with '#PBS' +> e path ：specify error file +> I ：interactive +> j join ：join output and error file. +> l resource_list ： define resource list as follow +> walltime=N ： wall time in unit of second, or in the form of hh:mm:ss +> mem=N[K|M|G][B|W]：define memory usage +> nodes=N:ppn=M ：define number of nodes N and processes per node M. +> m mail_options ：mail_option为a：作业abort时给用户发信；为b：作业开始运行发信；为e：作业结束运行时发信。若无此选项，默认为a。 +> M user_list ： mail addresss +> N name ： jobname, less than 15 characters. +> o path ： specify output file +> p priority ： adjust priority, [-1024，1023]，default 0. +> q destination ： specify which queue to use +> r y|n ： 指明作业是否可运行，y为可运行，n为不可运行。 +> S shell ： specify the SHELL to use, full path. +> u user_list ： specify the user to run the job +> v variable_list ： specify the environment variable list to export to this job. +> V ： export all environment variable to this job. +> W additional_attributes ： +> z ： do not show JOB_ID information after submission +> + +> [!NOTE] +> `qstat` `[-f JOB_ID]` `[-a]` `[-i]` `[-n]` `[-s]` `[-R]` `[-Q [QUEUE]]` `[-q]` `[-B]` `[-u USER]` +> f JOB_ID : specify the job +> a : list all the jobs +> i : list all non-running jobs +> r : list all the running jobs +> n : list the nodes assigned to jobs +> s : 列出队列管理员与scheduler所提供的建议 +> R : 列出磁盘预留信息 +> Q [QUEUE] : list queue status +> q [QUEUE] : list queue status +> B : list PBS Server information +> u USER : list the jobs for USER +> - [qsub doc](http://docs.adaptivecomputing.com/torque/4-1-3/help.htm#topics/commands/qsub.htm) - `man qsub` - `man pbs_resources` ## Slurm -```tip -*Slurm* is only available on **SGI** server! -``` +> [!TIP] +> *Slurm* is only available on **SGI** server! +> ### examples create a job script `job.slurm` with the following content: @@ -735,10 +739,10 @@ create a job script `job.slurm` with the following content: ```bash #!/bin/bash -#SBATCH --job-name=test -#SBATCH -n 16 # CPU cores -#SBATCH --output=%j.out -#SBATCH --error=%j.err +#SBATCH --job-name=test +#SBATCH -t 1-24:00:00 # 1 day and 24 hours +#SBATCH -n 16 # 16 CPU cores +#SBATCH --output=lalala-%x.%j.log echo "Start running ... ..." diff --git a/docs/Basic/Login.md b/docs/Basic/Login.md index 585eb93..1ff3453 100644 --- a/docs/Basic/Login.md +++ b/docs/Basic/Login.md @@ -1,17 +1,12 @@ ---- -sort: 2 -title: Login ---- -```note -For enhanced security, we have enabled **2 Factor Authentication (2FA)** on web login. You need to install an *Authenticator app* to log into the web interface. +> [!NOTE] +> For enhanced security, we have enabled **2 Factor Authentication (2FA)** on web login. You need to install an *Authenticator app* to log into the web interface. +> Using `ssh` to log in only requires your **ssh private key** as before. +> -Using `ssh` to log in only requires your **ssh private key** as before. -``` - -```danger -If you fail to login, please do not try many times. Otherwise your IP will be **banned** for a period of time 😭 -``` +> [!ATTENTION] +> If you fail to login, please do not try many times. Otherwise your IP will be **banned** for a period of time 😭 +> ## SSH login @@ -22,10 +17,10 @@ To generate a [*ssh-key pair*](https://wiki.archlinux.org/title/SSH_keys_(%E7%AE ```bash ➜ ssh-keygen -t rsa -b 4096 -C "lalala@Gravity" -```tip -You will be asked to input a passphrase when generating your key. -It is highly recommended to use an non-empty passphrase. -``` +> [!TIP] +> You will be asked to input a passphrase when generating your key. +> It is highly recommended to use an non-empty passphrase. +> # Output👇 Generating public/private rsa key pair. @@ -95,11 +90,10 @@ ssh -i @gravity.sjtu.edu.cn ssh -i /home/lalala/.ssh/id_rsa_Gravity lalala@gravity.sjtu.edu.cn ``` -```tip -You can set up your `~/.ssh/config` to **simplify** your command😎 as following: - -Edit your `~/.ssh/config` like this: -``` +> [!TIP] +> You can set up your `~/.ssh/config` to **simplify** your command😎 as following: +> Edit your `~/.ssh/config` like this: +> ```bash Host login01 @@ -154,11 +148,10 @@ Any of the listed authenticators below (and many others) will work. You can firs #### *Android* device -```tip -If you cannot run *authenticator app* due to **ERROR**: **You do not have Google Services Framework installed**, please [install **Google Services Framework** here](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk). - -若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) -``` +> [!TIP] +> If you cannot run *authenticator app* due to **ERROR**: **You do not have Google Services Framework installed**, please [install **Google Services Framework** here](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk). +> 若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) +> - [Google Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Google_Authenticator_5.20R3.apk) (stored on Gravity, can be accessed from anywhere 🎉🎉🎉) - [Microsoft Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Microsoft_Authenticator_6.2206.3973.apk) (stored on Gravity, can be accessed from anywhere 🎉🎉🎉) @@ -183,42 +176,42 @@ If you cannot run *authenticator app* due to **ERROR**: **You do not have Google open [Gravity Authentication webpage](https://gravity.sjtu.edu.cn/auth/), input your *username* + *password* - +![auth_login.png](../images/Basic/auth_login.png) click *Register device* (choose **One-Time Password** method!!!) - +![auth_add_device.png](../images/Basic/auth_add_device.png) check your mailbox, click *Register button* - +![auth_email.png](../images/Basic/auth_email.png) open your phone's authenticator app - +![auth_QR.png](../images/Basic/auth_QR.png) click *add* button - +![auth_add1.png](../images/Basic/auth_add1.png) *scan* QR code - +![auth_add2.png](../images/Basic/auth_add2.png) input your *Time-based one-time password (TOTP)*, enjoy! 🎉🎉🎉 - +![auth_totp.png](../images/Basic/auth_totp.png) # 登陆 Gravity（中文版） -```note -我们已在网站上启用*2FA*认证，您需要安装2FA应用才能够在网页端登陆 -若您使用ssh方式登陆，则仅需ssh private key私钥 -``` +> [!NOTE] +> 我们已在网站上启用*2FA*认证，您需要安装2FA应用才能够在网页端登陆 +> 若您使用ssh方式登陆，则仅需ssh private key私钥 +> -```danger -请勿多次失败登陆，您的IP将会被封禁一段时间 😭 -``` +> [!ATTENTION] +> 请勿多次失败登陆，您的IP将会被封禁一段时间 😭 +> ## SSH登陆 @@ -296,9 +289,9 @@ ssh -i <你的私钥> <用户名>@gravity.sjtu.edu.cn ssh -i /home/lalala/.ssh/id_rsa_Gravity lalala@gravity.sjtu.edu.cn ``` -```tip -您可以设置 `~/.ssh/config` 来**简化**命令😎，按照下方操作： -``` +> [!TIP] +> 您可以设置 `~/.ssh/config` 来**简化**命令😎，按照下方操作： +> 像这样修改文件`~/.ssh/config`： @@ -349,9 +342,9 @@ ssh login02 #### *Android* 设备 -```tip -若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) -``` +> [!TIP] +> 若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) +> - [Google Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Google_Authenticator_5.20R3.apk) (存储在Gravity，可以从任何地方访问 🎉🎉🎉) - [Microsoft Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Microsoft_Authenticator_6.2206.3973.apk) (存储在Gravity，可以从任何地方访问 🎉🎉🎉) @@ -373,40 +366,39 @@ ssh login02 打开 [Gravity认证页面](https://gravity.sjtu.edu.cn/auth/), 输入您的 *用户名* + *密码* - +![auth_login.png](../images/Basic/auth_login.png) 点击 *注册设备* （选择**One-Time Password**方式！！！） - +![auth_add_device.png](../images/Basic/auth_add_device.png) 检查您的邮箱，点击 *注册* 按钮 - +![auth_email.png](../images/Basic/auth_email.png) 打开您手机上的 authenticator 应用，准备扫描二维码 - +![auth_QR.png](../images/Basic/auth_QR.png) 点击*加号*按钮 - +![auth_add1.png](../images/Basic/auth_add1.png) 点击*扫描二维码* - +![auth_add2.png](../images/Basic/auth_add2.png) 输入您的 *二次验证码*, 开始征途吧！🎉🎉🎉 - +![auth_totp.png](../images/Basic/auth_totp.png) ## SGI login -```tip -1. **SGI** is a **standalone** machine, and it has a different OS and disk system. -2. **SGI** use *slurm* rather than *pbs* as the job scheduler. - -**NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. -``` +> [!TIP] +> **SGI** is a **standalone** machine, and it has a different OS and disk system. +> **SGI** use *slurm* rather than *pbs* as the job scheduler. +> **NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. +> You cannot access **SGI** directly, you need to log in **Gravity** first, then on login nodes, use: diff --git a/docs/Basic/README.md b/docs/Basic/README.md index f775be1..6e9b773 100644 --- a/docs/Basic/README.md +++ b/docs/Basic/README.md @@ -1,6 +1,3 @@ ---- -sort: 1 ---- # Basic diff --git a/docs/Basic/Resource_Monitor.md b/docs/Basic/Resource_Monitor.md index 9e0ceb1..e51ea22 100644 --- a/docs/Basic/Resource_Monitor.md +++ b/docs/Basic/Resource_Monitor.md @@ -1,7 +1,3 @@ ---- -sort: 4 -title: Resource Monitor ---- ## Web interface @@ -36,20 +32,18 @@ Max size = 5.00TB , Used size= 4.33TB , Left size= 666.00GB The default quota per account is `3T`. -```note -If you cannot run `icfsquota` successfully😅, please check your permissions setting of `$HOME` foler. -`chmod 755 $HOME` -So that you can run `icfsquota`🥳 -``` +> [!NOTE] +> If you cannot run `icfsquota` successfully😅, please check your permissions setting of `$HOME` foler. +> `chmod 755 $HOME` +> So that you can run `icfsquota`🥳 +> If you run out of quota, you can free out space by deleting some files that are no longer needed. If no more files can be deleted, ask your group owner to allocate more disk space to you. -```note -Files deleted through the graphical desktop will be moved to Trash folder that still takes up space. You can empty the trash if this is the case, through either the desktop or on the command line: - -`rm -rf ./local/share/Trash/*` - -``` +> [!NOTE] +> Files deleted through the graphical desktop will be moved to Trash folder that still takes up space. You can empty the trash if this is the case, through either the desktop or on the command line: +> `rm -rf ./local/share/Trash/*` +> You can use the command on **SGI**, `quota -ls` like this @@ -86,9 +80,9 @@ This month [20210501 ~ 20210531] cost: 66 ￥ ######### Complete usage statistics: https://stat.gravity.sjtu.edu.cn ########## ``` -```tip -Your advisor will bear the cost 🥳 -``` +> [!TIP] +> Your advisor will bear the cost 🥳 +> ## PBS diff --git a/docs/MISC/FAQ.md b/docs/MISC/FAQ.md index dcd5409..62ea0e9 100644 --- a/docs/MISC/FAQ.md +++ b/docs/MISC/FAQ.md @@ -1,13 +1,9 @@ ---- -sort: 1 -title: FAQ ---- # FAQ -```tip -If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! -``` +> [!TIP] +> If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! +> ## Permission denied @@ -38,9 +34,9 @@ What if I lost my *ssh-key*? 🛡️ PS. You'd better use **@sjtu.edu.cn**, so that we can recognize who you really are 👀 -```tip -You may be prompted for a passphrase when login with a ssh-key. This is the password you set-up to protect the key when generating your ssh-key. It is **different** from the password you use for jupyter. -``` +> [!TIP] +> You may be prompted for a passphrase when login with a ssh-key. This is the password you set-up to protect the key when generating your ssh-key. It is **different** from the password you use for jupyter. +> ## IP banned diff --git a/docs/MISC/Issues.md b/docs/MISC/Issues.md index b59396e..5b6230e 100644 --- a/docs/MISC/Issues.md +++ b/docs/MISC/Issues.md @@ -1,19 +1,15 @@ ---- -sort: 2 -title: Issues ---- # Issues -```note -If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! -``` +> [!NOTE] +> If there is **NO** solution of your problem, feel free to [contact us](/?id=contact)! +> -```danger -**NEVER EVER** install/update any package at *login01* using `conda/mamba`. Otherwise, your whole conda environment will be destroyed! -- You can use `pip install` instead -- You can go to *login02* to use `conda/mamba install/create` -``` +> [!ATTENTION] +> **NEVER EVER** install/update any package at *login01* using `conda/mamba`. Otherwise, your whole conda environment will be destroyed! +> You can use `pip install` instead +> You can go to *login02* to use `conda/mamba install/create` +> ## **Conda** One recieves '**binary file can not be executed**' after installing of Anaconda latest version (2020-05). @@ -39,13 +35,11 @@ snap_001.0.hdf5: unable to open file This is a known problem in the NFS file system on `login02`. You can switch to read the file on `login01` instead. -```warning -Alternatively, you may choose to disable hdf5 file locking before reading, by setting the environment variable `HDF5_USE_FILE_LOCKING` to `FALSE`, e.g., in bash: - - `export HDF5_USE_FILE_LOCKING=FALSE` - -This will disable hdf5 file locking, but you will have to be more careful about opening files to avoid problematic access patterns (i.e.: multiple writers to the same hdf5 file) that the file locking was designed to prevent. -``` +> [!WARNING] +> Alternatively, you may choose to disable hdf5 file locking before reading, by setting the environment variable `HDF5_USE_FILE_LOCKING` to `FALSE`, e.g., in bash: +> `export HDF5_USE_FILE_LOCKING=FALSE` +> This will disable hdf5 file locking, but you will have to be more careful about opening files to avoid problematic access patterns (i.e.: multiple writers to the same hdf5 file) that the file locking was designed to prevent. +> ## **日志残留问题** diff --git a/docs/MISC/README.md b/docs/MISC/README.md index c18ae19..4a13641 100644 --- a/docs/MISC/README.md +++ b/docs/MISC/README.md @@ -1,6 +1,2 @@ ---- -sort: 4 -title: MISC ---- {% include list.liquid %} diff --git a/docs/Policy.md b/docs/Policy.md index b82034e..b448b52 100644 --- a/docs/Policy.md +++ b/docs/Policy.md @@ -1,7 +1,3 @@ ---- -sort: 3 -title: Policy ---- # 上海交通大学天文系Gravity超算使用管理方法（试行） diff --git a/docs/QuickStart.md b/docs/QuickStart.md index 236a087..b6858bf 100644 --- a/docs/QuickStart.md +++ b/docs/QuickStart.md @@ -1,7 +1,3 @@ ---- -sort: 1 -title: Quick Start ---- ## Apply for an account 🙋‍♂️🙋‍♀️ Follow the instructions [here](/Basic/Account) to apply for an account. Make sure you read the instructions and user policies before applying. Wait for a day or so, then you will receive the e-mail 🥳. @@ -85,9 +81,9 @@ qdel 36162 # 36162 is your jobID | `showstart ` | ***when*** to start (if your job is waiting) | | `qdel ` | **delete** a job | | `qdel -p ` | force to **delete** a job | -| `qselect -u $USER | xargs qdel` | delete all of your jobs | -| `qselect -u $USER -s Q | xargs qdel` | delete all of your **Queueing** jobs | -| `qselect -u $USER -s R | xargs qdel` | delete all of your **Running** jobs | +| `qselect -u $USER \| xargs qdel` | delete all of your jobs | +| `qselect -u $USER -s Q \| xargs qdel` | delete all of your **Queueing** jobs | +| `qselect -u $USER -s R \| xargs qdel` | delete all of your **Running** jobs | | `qhold ` | **hold** a job until you release it | | `qrls ` | **release** a job | | `pestat` | every node status | @@ -125,8 +121,8 @@ qdel 36162 # 36162 is your jobID | `top -u ` | my processes info | | `htop` | processes info (beautiful) | | `htop -u ` | my processes info (beautiful) | -| `ps aux | grep python` | process info including string: *python* | -| `ps -eo pid,tty,user,command,lstart,etime | grep ` | **my own** processes info | +| `ps aux \| grep python` | process info including string: *python* | +| `ps -eo pid,tty,user,command,lstart,etime \| grep ` | **my own** processes info | | `kill -9 ` | **kill** the process using its PID | | `pkill -u ` | **kill all** processes belonging to *username* | | `tail -f My_PBS_Job.log` | see the file output **in time** | diff --git a/docs/README.md b/docs/README.md index 537cba5..c0a72c8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,17 +1,16 @@ ## [Gravity Documentation](/README.md) This is the documentation for the **Gravity Cluster** of [Department of Astronomy (DOA)](http://astro.sjtu.edu.cn/en/) at [Shanghai Jiao Tong University (SJTU)](https://www.sjtu.edu.cn/). -```tip -### Welcome to contribute -This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. Contributing is pretty simple as following: -1. **Click**. To contribute, click the **[edit](https://github.com/gravity-doc/gravity-doc.github.io/edit/master/docs/index.md)** :pencil: on the upper right of the page you feel like editing. Every document page is **editable**. It will bring you to the source files on github. You may will need to login to github to proceed. Once logged-in, a fork of the document repository will be created under your own account, so that you can edit it in whatever way you like. -2. **Edit**. Edit the page to contribute useful stuff. Check [here](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github/basic-writing-and-formatting-syntax) or the existing source pages for examples in the syntax. [This page](https://rundocs.io/writng/) contains theme-related syntax. -3. **Commit**. Finish the editing by leaving a commit message, describing what you have done. -4. **Submit**. Submit a **pull request** so that your changes will be submitted to the upstream branch (a.k.a., this website). The administrators will review your changes and absorb them into the website. - -Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. -### **中文内容也欢迎！🥳** -``` +> [!TIP] +> ### Welcome to contribute +> This doc is hosted on [Github](https://github.com/gravity-doc/gravity-doc.github.io). Feel free to submit a pull request. Contributing is pretty simple as following: +> **Click**. To contribute, click the **[edit](https://github.com/gravity-doc/gravity-doc.github.io/edit/master/docs/index.md)** :pencil: on the upper right of the page you feel like editing. Every document page is **editable**. It will bring you to the source files on github. You may will need to login to github to proceed. Once logged-in, a fork of the document repository will be created under your own account, so that you can edit it in whatever way you like. +> **Edit**. Edit the page to contribute useful stuff. Check [here](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github/basic-writing-and-formatting-syntax) or the existing source pages for examples in the syntax. [This page](https://rundocs.io/writng/) contains theme-related syntax. +> **Commit**. Finish the editing by leaving a commit message, describing what you have done. +> **Submit**. Submit a **pull request** so that your changes will be submitted to the upstream branch (a.k.a., this website). The administrators will review your changes and absorb them into the website. +> Your name will be honoured in the [contributor list](https://github.com/gravity-doc/gravity-doc.github.io/graphs/contributors) once your contributions are merged. +> ### **中文内容也欢迎！🥳** +> ## Contact You can contact us via @@ -38,4 +37,4 @@ We would appreciate it if all publications that include results generated using - [List of Publications powered by Gravity](https://ui.adsabs.harvard.edu/search/q=%3D%20full%3A%22gravity%20supercomputer%22&sort=date%20desc%2C%20bibcode%20desc&p_=0) - +![logo_DOA_large.png](./images/logo_DOA_large.png) diff --git a/docs/Software/Proxy.md b/docs/Software/Proxy.md index 58072fe..d44f071 100644 --- a/docs/Software/Proxy.md +++ b/docs/Software/Proxy.md @@ -1,14 +1,10 @@ ---- -sort: 3 -title: Proxy ---- ## Surf the Internet🏄‍ -```tip -- On [**Gravity Web**](https://jupyter.gravity.sjtu.edu.cn/), you can access the Internet🏄‍ when using **Jupyter/VScode** on **login/computing nodes** -- We will speed up🚀 your **git** connection automatically -``` +> [!TIP] +> On [**Gravity Web**](https://jupyter.gravity.sjtu.edu.cn/), you can access the Internet🏄‍ when using **Jupyter/VScode** on **login/computing nodes** +> We will speed up🚀 your **git** connection automatically +> ### 1. Spawn a server @@ -30,9 +26,9 @@ For example🌰: ## Speedup Git⚡ -```tip -When using **terminal**, you can use `pgit` to speed up🚀 your `git` connection -``` +> [!TIP] +> When using **terminal**, you can use `pgit` to speed up🚀 your `git` connection +> ### 1. Why @@ -44,10 +40,10 @@ Therefore, we provide `pgit` (*proxy git*) to solve this problem🥳 #### pgit -```tip -On **login01/login02**, `pgit` **==** `git`😎 -P.S. `pgit` means **proxy git** -``` +> [!TIP] +> On **login01/login02**, `pgit` **==** `git`😎 +> P.S. `pgit` means **proxy git** +> **Common usage:** @@ -109,6 +105,6 @@ git config --global --unset https.proxy git config --global --unset core.sshCommand ``` -```warning -**Attention!** There is **NO warranty**! But we will try our best to maintain this service😊 -``` +> [!WARNING] +> **Attention!** There is **NO warranty**! But we will try our best to maintain this service😊 +> diff --git a/docs/Software/README.md b/docs/Software/README.md index 17433aa..5dfde2d 100644 --- a/docs/Software/README.md +++ b/docs/Software/README.md @@ -1,6 +1,3 @@ ---- -sort: 3 ---- # Software diff --git a/docs/Software/Software_Installed.md b/docs/Software/Software_Installed.md index 15bf934..36b4e9e 100644 --- a/docs/Software/Software_Installed.md +++ b/docs/Software/Software_Installed.md @@ -1,11 +1,8 @@ ---- -sort: 1 ---- # Software Installed -```note -On the *Gravity*, we use **`module`** to manage software and tools. -``` +> [!NOTE] +> On the *Gravity*, we use **`module`** to manage software and tools. +> | **Command** | **Description** | | ------------------------------- | ------------------------------------------------- | @@ -66,11 +63,10 @@ $ module unload python/python-3.8.5 $ mdoule list ``` -```tip -The order of the module loading matters if some of them have dependence on others. - -Some libraries (in particular some C++ libraries) will only work properly with the compiler that is used to compile it. These modules will have the compiler version appended to the name, such as `hdf5/1.12.1-with-cxx-fortran-gcc10.2.0` -``` +> [!TIP] +> The order of the module loading matters if some of them have dependence on others. +> Some libraries (in particular some C++ libraries) will only work properly with the compiler that is used to compile it. These modules will have the compiler version appended to the name, such as `hdf5/1.12.1-with-cxx-fortran-gcc10.2.0` +> ## Conda(Mamba) @@ -82,9 +78,9 @@ module load anaconda/anaconda-mamba Note you will have to unload any previously loaded python module if you see conflicts. -```tip -[mamba](https://mamba.readthedocs.io/en/latest/) is almost the same with [conda](https://docs.conda.io/projects/conda/en/latest/commands.html), but much super faster⚡! For example 🌰: -``` +> [!TIP] +> [mamba](https://mamba.readthedocs.io/en/latest/) is almost the same with [conda](https://docs.conda.io/projects/conda/en/latest/commands.html), but much super faster⚡! For example 🌰: +> | mamba⚡ | conda | | ---------------------------------------- | ---------------------------------------- | @@ -155,10 +151,10 @@ virtualenv myenv ``` By default this will create an "virtual environment" under `myenv/` in your current directory. -```warning -Currently the `virtualenv` command only works for python2. For python3 users, after loading the python3 module, you can create the virtualenv by -`python3 -m venv myenv` -``` +> [!WARNING] +> Currently the `virtualenv` command only works for python2. For python3 users, after loading the python3 module, you can create the virtualenv by +> `python3 -m venv myenv` +> Once this is done, you can activate it by diff --git a/docs/Software/User_Software_Installation.md b/docs/Software/User_Software_Installation.md index 62e3608..55607b1 100644 --- a/docs/Software/User_Software_Installation.md +++ b/docs/Software/User_Software_Installation.md @@ -1,13 +1,9 @@ ---- -sort: 2 -title: User Software Installation ---- -```tip -You can use `module` to manage your own software installed -Also, you can just ignore `module` if you do not have **too many** tools or they do not **conflict** with each other -``` +> [!TIP] +> You can use `module` to manage your own software installed +> Also, you can just ignore `module` if you do not have **too many** tools or they do not **conflict** with each other +> # Use modules diff --git a/docs/Updates.md b/docs/Updates.md index 865d526..8d590d0 100644 --- a/docs/Updates.md +++ b/docs/Updates.md @@ -1,7 +1,3 @@ ---- -sort: 4 -title: Updates ---- ## Future ✨ diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 6155cb6..fc76f42 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,4 +1,4 @@ - +![logo_DOA_large.png](images/logo_DOA_large.png) [Gravity Home](https://jupyter.gravity.sjtu.edu.cn/) [Get started](/README.md) diff --git a/docs/index.html b/docs/index.html index bca82cd..560ae73 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,12 +1,16 @@ + Gravity Documentation - + - - + + + +
Loading (/≧▽≦)/... ...
- - - - - - - + - - - - - - + + + + + + - + - + + \ No newline at end of file From bb44a6aadd768cf6e5b9dae5f7552d316a54b107 Mon Sep 17 00:00:00 2001 From: lalalabox Date: Sat, 13 Apr 2024 21:50:30 +0800 Subject: [PATCH 23/23] fix tip block position --- docs/Basic/Login.md | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/Basic/Login.md b/docs/Basic/Login.md index 1ff3453..1f85773 100644 --- a/docs/Basic/Login.md +++ b/docs/Basic/Login.md @@ -1,12 +1,12 @@ -> [!NOTE] +> [!NOTE] > For enhanced security, we have enabled **2 Factor Authentication (2FA)** on web login. You need to install an *Authenticator app* to log into the web interface. > Using `ssh` to log in only requires your **ssh private key** as before. -> +> -> [!ATTENTION] +> [!ATTENTION] > If you fail to login, please do not try many times. Otherwise your IP will be **banned** for a period of time 😭 -> +> ## SSH login @@ -16,12 +16,14 @@ To generate a [*ssh-key pair*](https://wiki.archlinux.org/title/SSH_keys_(%E7%AE ```bash ➜ ssh-keygen -t rsa -b 4096 -C "lalala@Gravity" +``` -> [!TIP] -> You will be asked to input a passphrase when generating your key. -> It is highly recommended to use an non-empty passphrase. -> +> [!TIP] +> You will be asked to input a passphrase when generating your key. +> It is **highly recommended** to use an non-empty passphrase. +> +```bash # Output👇 Generating public/private rsa key pair. Enter file in which to save the key (/home/lalala/.ssh/id_rsa): /home/lalala/.ssh/id_rsa_Gravity @@ -90,10 +92,10 @@ ssh -i @gravity.sjtu.edu.cn ssh -i /home/lalala/.ssh/id_rsa_Gravity lalala@gravity.sjtu.edu.cn ``` -> [!TIP] +> [!TIP] > You can set up your `~/.ssh/config` to **simplify** your command😎 as following: > Edit your `~/.ssh/config` like this: -> +> ```bash Host login01 @@ -148,10 +150,10 @@ Any of the listed authenticators below (and many others) will work. You can firs #### *Android* device -> [!TIP] +> [!TIP] > If you cannot run *authenticator app* due to **ERROR**: **You do not have Google Services Framework installed**, please [install **Google Services Framework** here](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk). > 若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) -> +> - [Google Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Google_Authenticator_5.20R3.apk) (stored on Gravity, can be accessed from anywhere 🎉🎉🎉) - [Microsoft Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Microsoft_Authenticator_6.2206.3973.apk) (stored on Gravity, can be accessed from anywhere 🎉🎉🎉) @@ -204,14 +206,14 @@ input your *Time-based one-time password (TOTP)*, enjoy! 🎉🎉🎉 # 登陆 Gravity（中文版） -> [!NOTE] +> [!NOTE] > 我们已在网站上启用*2FA*认证，您需要安装2FA应用才能够在网页端登陆 > 若您使用ssh方式登陆，则仅需ssh private key私钥 -> +> -> [!ATTENTION] +> [!ATTENTION] > 请勿多次失败登陆，您的IP将会被封禁一段时间 😭 -> +> ## SSH登陆 @@ -289,9 +291,9 @@ ssh -i <你的私钥> <用户名>@gravity.sjtu.edu.cn ssh -i /home/lalala/.ssh/id_rsa_Gravity lalala@gravity.sjtu.edu.cn ``` -> [!TIP] +> [!TIP] > 您可以设置 `~/.ssh/config` 来**简化**命令😎，按照下方操作： -> +> 像这样修改文件`~/.ssh/config`： @@ -342,9 +344,9 @@ ssh login02 #### *Android* 设备 -> [!TIP] +> [!TIP] > 若您无法运行认证器应用，**报错**提示：**您未安装谷歌框架**，请[在这里安装**Google Services Framework**](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/%E8%B0%B7%E6%AD%8C%E6%A1%86%E6%9E%B6_%E5%AE%89%E8%A3%85%E5%99%A8.apk) -> +> - [Google Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Google_Authenticator_5.20R3.apk) (存储在Gravity，可以从任何地方访问 🎉🎉🎉) - [Microsoft Authenticator](https://gravity.sjtu.edu.cn/file/api/public/dl/bjEJwMgn/apk/Microsoft_Authenticator_6.2206.3973.apk) (存储在Gravity，可以从任何地方访问 🎉🎉🎉) @@ -394,11 +396,11 @@ ssh login02 ## SGI login -> [!TIP] +> [!TIP] > **SGI** is a **standalone** machine, and it has a different OS and disk system. > **SGI** use *slurm* rather than *pbs* as the job scheduler. > **NEW**: **SGI** account is the same as **Gravity** account, you can use the same username and password to log in both **SGI** and **Gravity**. -> +> You cannot access **SGI** directly, you need to log in **Gravity** first, then on login nodes, use: