diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index ab5ea86d..58771c5e 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -4,8 +4,6 @@ name: Check lesson and build for all configs on: pull_request: push: - branches: - - gh-pages jobs: lint_common_files: @@ -18,6 +16,21 @@ jobs: - name: Check spelling run: | make spellcheck + + proselint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Install proselint + run: | + python3 -m pip install proselint + - name: Check prose + run: | + proselint **/*.md - name: Check top-level Markdown syntax uses: DavidAnson/markdownlint-cli2-action@v9 - name: Check episode Markdown syntax @@ -60,14 +73,7 @@ jobs: strategy: matrix: HPC_JEKYLL_CONFIG: - - Birmingham_Baskerville_slurm - - ComputeCanada_Graham_slurm - - EPCC_Cirrus_slurm - - HPCC_MagicCastle_slurm - - Magic_Castle_EESSI_slurm - - NIST_CTCMS_slurm - - Norway_SIGMA2_SAGA_slurm - - UCL_Myriad_sge + - NeSI_Mahuika_slurm steps: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 diff --git a/Makefile b/Makefile index d8add2b7..c6990bf6 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ export SHELL = /bin/bash # Settings MAKEFILES=Makefile $(wildcard *.mk) -JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll +#JEKYLL=bundle config --local set path .vendor/bundle && bundle install && bundle update && bundle exec jekyll +JEKYLL=bundle exec jekyll HPC_JEKYLL_CONFIG?= PARSER=bin/markdown_ast.rb DST=_site diff --git a/README.md b/README.md index 7e492f63..aae87aef 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,22 @@ + +### For NeSI developers: + + +Our main branches are: + +* origin/gh-pages <-Do not use, this is for fetching changes from upstream +* origin/gh-pages-upstream-compat <- This is used for changes we want to share with community +* origin/gh-pages-nesi <- Our NeSI specific lessons, not compatible with upstream + +To test locally, make your changes and run this. + +```bash +HPC_JEKYLL_CONFIG=_includes/snippets_library/NeSI_Mahuika_slurm/_config_options.yml make serve +``` + +--- + +### General development notes: # Intro to HPC This lesson teaches the basics of interacting with high-performance computing @@ -269,3 +288,4 @@ provided to help guide planning and decision-making. [mpi4py]: https://mpi4py.readthedocs.io [setup]: http://carpentries.github.io/lesson-example/setup.html [upstream]: https://github.com/carpentries-incubator/hpc-intro + diff --git a/SETUP_FOR_INSTRUCTORS.md b/SETUP_FOR_INSTRUCTORS.md new file mode 100644 index 00000000..73490f9f --- /dev/null +++ b/SETUP_FOR_INSTRUCTORS.md @@ -0,0 +1,29 @@ +# + +Create workshop subdir. + +``` +mkdir +``` + +Get files. + +``` +wget https://raw.githubusercontent.com/nesi/hpc-intro/gh-pages-nesi/_includes/example_scripts/example-job.sh +wget https://raw.githubusercontent.com/nesi/hpc-intro/gh-pages-nesi/_includes/example_scripts/array_sum2.r -O {{ site.example.script }} +wget https://raw.githubusercontent.com/nesi/hpc-intro/gh-pages-nesi/_includes/example_scripts/whothis.sh +wget +wget +``` + +``` +mkdir birds +cd birds +touch kaka.txt kakapo.jpeg kea.txt kiwi.jpeg pukeko.jpeg +``` + +Set permissions. + +``` +chmod -R 755* +``` \ No newline at end of file diff --git a/_config.yml b/_config.yml index d6c69a82..3c402b01 100644 --- a/_config.yml +++ b/_config.yml @@ -1,8 +1,5 @@ -# HPC-Intro Jekyll configuration ---- - #------------------------------------------------------------ -# SITE specific configuration +# NeSI: Mahuika + SLURM #------------------------------------------------------------ # Cluster host and scheduler options: the defaults come from @@ -20,36 +17,25 @@ # installation. Remember to keep the leading slash on the # `snippets` variable below! -# --------------------------------------------------------------- -# HPC Carpentries in the Cloud: Slurm + Software Stack from EESSI -# --------------------------------------------------------------- -# -# The HPC Carpentry Cluster in the Cloud is provided as a public -# service by volunteers. It is provisioned with Magic Castle -# using the EESSI -# software stack. If you need an -# account, please visit . -# -# Compute responsibly. - -snippets: "/snippets_library/HPCC_MagicCastle_slurm" +snippets: "/snippets_library/NeSI_Mahuika_slurm" local: prompt: "[you@laptop:~]$" bash_shebang: "#!/usr/bin/env bash" remote: - name: "HPC Carpentry's Cloud Cluster" - login: "cluster.hpc-carpentry.org" - portal: "https://mokey.cluster.hpc-carpentry.org" - host: "login1" - node: "node1" - location: "cluster.hpc-carpentry.org" + name: "mahuika" + login: "nesi.org.nz" + portal: "" + host: "mahuika02" + node: "mahuika02" + location: "Greta Point Datacenter" homedir: "/home" user: "yourUsername" + prompt: "[yourUsername@mahuika ~]$" + bash_shebang: "#!/bin/bash -e" module_python3: "Python" - prompt: "[yourUsername@login1 ~]$" - bash_shebang: "#!/bin/bash" + sched: name: "Slurm" @@ -57,36 +43,51 @@ sched: name: "sbatch" options: "" queue: - debug: "smnode" - testing: "cpubase_bycore_b1" + debug: "debug" + testing: "" status: "squeue" flag: user: "-u yourUsername" + me: "--me" interactive: "" histdetail: "-l -j" - name: "-J" - time: "-t" - queue: "-p" - nodes: "-N" - tasks: "-n" + name: "--job-name" + time: "--time" + queue: "--partition" del: "scancel" interactive: "srun" info: "sinfo" comment: "#SBATCH" - hist: "sacct -u yourUsername" - hist_filter: "" + hist: "sacct" + efficiency: "nn_seff" + projectcode: "nesi99991" + +# For 'R' +example: + lang: "R" + shell: "Rscript " + script: "array_sum.r" + module: "R/4.3.1-gimkl-2022a" + +# For 'python' +# example: +# lang: "Python" +# shell: "python" +# script: "array_sum.py" +# module: "Python/3.11.3-gimkl-2022a" episode_order: - - 10-hpc-intro - - 11-connecting - - 12-cluster - - 13-scheduler - - 14-environment-variables - - 15-modules - - 16-transferring-files - - 17-parallel - - 18-resources - - 19-responsibility + - 01-cluster + - 02-filesystem + - 04-modules + - 03-break1 + - 05-scheduler + - 06-lunch + - 064-parallel + - 07-resources + - 08-break2 + - 09-scaling + #------------------------------------------------------------ # Values for this lesson @@ -103,10 +104,10 @@ carpentry: "incubator" title: "Introduction to High-Performance Computing" # Email address, no mailto: -email: "team@carpentries.org" +email: "support@nesi.org.nz" # Life cycle stage of the lesson ("pre-alpha", "alpha", "beta", "stable")? -life_cycle: "alpha" +life_cycle: "stable" #------------------------------------------------------------ # Generic settings (should not need to change). @@ -115,8 +116,20 @@ life_cycle: "alpha" # What kind of thing is this ("workshop" or "lesson")? kind: "lesson" +# Workshop working directory. +working_dir: + - /nesi/nobackup/nesi99991 + - introhpc2403 + # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). -start_time: 0 +# 600 is 10am +start_time: 600 +# Start time for people wanting to skip bash content +hpc_start_time: 780 +# Lesson to start at. +hpc_start_lesson: 01-cluster +# Script to run if skipping bash lessons. Should create neccicery files to get started. +hpc_start_script: late_start.sh # Magic to make URLs resolve both locally and on GitHub. Please don't change. repository: / @@ -141,6 +154,10 @@ training_site: "https://carpentries.github.io/instructor-training" workshop_repo: "https://github.com/carpentries/workshop-template" workshop_site: "https://carpentries.github.io/workshop-template" cc_by_human: "https://creativecommons.org/licenses/by/4.0/" +support_docs: "https://support.nesi.org.nz/hc/en-gb" +exercise: "https://docs.google.com/spreadsheets/d/1D5PnhE6iJOB3ZKkDCiBHnk5CNZlhmj_gS-IXKGkkVoI/edit?usp=sharing" + + # Specify that things in the Episodes and Extras collections should be output. collections: diff --git a/_config.yml.old b/_config.yml.old new file mode 100644 index 00000000..1c2aabf7 --- /dev/null +++ b/_config.yml.old @@ -0,0 +1,170 @@ +# HPC-Intro Jekyll configuration +--- + +#------------------------------------------------------------ +# SITE specific configuration +#------------------------------------------------------------ + +# Cluster host and scheduler options: the defaults come from +# Graham at Compute Canada, running Slurm. Other options can +# be found in the library of snippets, +# `_includes/snippets_library`. To use one, replace options +# below with those in `_config_options.yml` from the +# library. E.g, to customise for Cirrus at EPCC, running +# PBS, we could replace the options below with those from +# +# _includes/snippets_library/EPCC_Cirrus_pbs/_config_options.yml +# +# If your cluster is not represented in the library, please +# copy an existing folder, rename it, and customize for your +# installation. Remember to keep the leading slash on the +# `snippets` variable below! + +snippets: "/snippets_library/ComputeCanada_Graham_slurm" + +local: + prompt: "[user@laptop ~]$" + bash_shebang: "#!/usr/bin/env bash" + +remote: + name: "Graham" + login: "graham.computecanada.ca" + host: "gra-login1" + node: "aci-377" + location: "University of Waterloo" + homedir: "/home" + user: "yourUsername" + prompt: "[yourUsername@gra-login1 ~]$" + module_python3: "python3" + bash_shebang: "#!/usr/bin/env bash" + +sched: + name: "Slurm" + submit: + name: "sbatch" + options: "" + queue: + debug: "debug" + testing: "testing" + status: "squeue" + flag: + user: "-u yourUsername" + interactive: "" + histdetail: "-l -j" + name: "-J" + time: "-t" + queue: "-p" + nodes: "-N" + tasks: "-n" + del: "scancel" + interactive: "srun" + info: "sinfo" + comment: "#SBATCH" + hist: "sacct -u yourUsername" + +episode_order: + - 11-hpc-intro + - 12-cluster + - 13-scheduler + - 14-modules + - 15-transferring-files + - 16-parallel + - 17-resources + - 18-responsibility + +#------------------------------------------------------------ +# Values for this lesson +#------------------------------------------------------------ + +# Which carpentry is this? +# "swc": Software Carpentry +# "dc": Data Carpentry +# "lc": Library Carpentry +# "cp": Carpentries (e.g., instructor traning) +carpentry: "incubator" + +# Overall title for pages. +title: "Introduction to High-Performance Computing" + +# Email address, no mailto: +email: "team@carpentries.org" + +# Life cycle stage of the lesson ("pre-alpha", "alpha", "beta", "stable")? +life_cycle: "alpha" + +#------------------------------------------------------------ +# Generic settings (should not need to change). +#------------------------------------------------------------ + +# What kind of thing is this ("workshop" or "lesson")? +kind: "lesson" + +# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). +start_time: 0 + +# Magic to make URLs resolve both locally and on GitHub. Please don't change. +repository: / + +# Invoke the Carpentries theme rather than copying into our repository. +remote_theme: carpentries/carpentries-theme + +# Sites. +amy_site: "https://amy.carpentries.org/" +carpentries_github: "https://github.com/carpentries" +carpentries_pages: "https://carpentries.github.io" +carpentries_site: "https://carpentries.org/" +dc_site: "https://datacarpentry.org" +example_repo: "https://github.com/carpentries/lesson-example" +example_site: "https://carpentries.github.io/lesson-example" +lc_site: "https://librarycarpentry.org/" +swc_github: "https://github.com/swcarpentry" +swc_pages: "https://swcarpentry.github.io" +swc_site: "https://software-carpentry.org" +template_repo: "https://github.com/carpentries/styles" +training_site: "https://carpentries.github.io/instructor-training" +workshop_repo: "https://github.com/carpentries/workshop-template" +workshop_site: "https://carpentries.github.io/workshop-template" +cc_by_human: "https://creativecommons.org/licenses/by/4.0/" + +# Specify that things in the Episodes and Extras collections should be output. +collections: + episodes: + output: true + permalink: /:path/index.html + extras: + output: true + permalink: /:path/index.html + +# Set the default layout for things in the episodes collection. +defaults: + - values: + root: . + layout: page + - scope: + path: "" + type: episodes + values: + root: .. + layout: episode + - scope: + path: "" + type: extras + values: + root: .. + layout: page + +# Files and directories that are not to be copied. +exclude: + - Makefile + - bin/ + - .Rproj.user/ + - .vendor/ + - vendor/ + - .docker-vendor/ + +# Turn on built-in syntax highlighting. +highlighter: rouge + +plugins: + - jekyll-redirect-from + - jekyll-paginate diff --git a/_episodes/01-cluster.md b/_episodes/01-cluster.md new file mode 100644 index 00000000..91b85125 --- /dev/null +++ b/_episodes/01-cluster.md @@ -0,0 +1,91 @@ +--- +title: "Working on a remote HPC system" +# teaching: 10 +teaching: 20 +exercises: 0 +questions: +- "What is an HPC system?" +- "How does an HPC system work?" +- "How do I log in to a remote HPC system?" +objectives: +- "Connect to a remote HPC system." +- "Understand the general HPC system architecture." +keypoints: +- "An HPC system is a set of networked machines." +- "HPC systems typically provide login nodes and a set of compute nodes." +- "The resources found on independent (compute) nodes can vary in volume and + type (amount of RAM, processor architecture, availability of network mounted + filesystems, etc.)." +- "Files saved on shared storage are available on all nodes." +- "The login node is a shared machine: be considerate of other users." +--- + +## What Is an HPC System? + +The words "cloud", "cluster", and the phrase "high-performance computing" or +"HPC" are used a lot in different contexts and with various related meanings. +So what do they mean? And more importantly, how do we use them in our work? + +A *Remote* computer is one you have no access to physically and must connect via a network (as opposed to *Local*) + +*Cloud* refers to remote computing resources +that are provisioned to users on demand or as needed. + +*HPC*, *High Performance Computer*, *High Performance Computing* or *Supercomputer* are all general terms for a large or powerful computing resource. + +*Cluster* is a more specific term describing a type of supercomputer comprised of multiple smaller computers (nodes) working together. Almost all supercomputers are clusters. + +## Access + +You will connect to a cluster over the internet either with a web client (Jupyter) or with SSH (**S**ecure **Sh**ell). Your main interface with the cluster will be using command line. + +## Nodes + +Individual computers that compose a cluster are typically called *nodes* +On a cluster, there are different types of nodes for different +types of tasks. The node where you are now will be different depending on +how you accessed the cluster. + +Most of you (using JupyterHub) will be on an interactive *compute node*. +This is because Jupyter sessions are launched as a job. If you are using SSH to connect to the cluster, you will be on a +*login node*. Both JupyterHub and SSH login nodes serve as an access point to the cluster. + + + +The real work on a cluster gets done by the *compute nodes*. +Compute nodes come in many shapes and sizes, but generally are dedicated to long +or hard tasks that require a lot of computational resources. + +## What's in a Node? + +A node is similar in makeup to a regular desktop or laptop, composed of *CPUs* (sometimes also called *processors* or *cores*), *memory* +(or *RAM*), and *disk* space. Although, where your laptop might have 8 CPUs and 16GB of memory, a compute node will have hundreds of cores and GB of memory. + +* **CPUs** are a computer's tool for running programs and calculations. + +* **Memory** is for short term storage, containing the information currently being operated on by the CPUs. + +* **Disk** is for long term storage, data stored here is permanent, i.e. still there even if the computer has been restarted. +It is common for nodes to connect to a shared, remote disk. + +{% include figure.html url="" max-width="40%" + file="/fig/clusterDiagram.png" + alt="Node anatomy" caption="" %} + +> ## Differences Between Nodes +> +> Many HPC clusters have a variety of nodes optimized for particular workloads. +> Some nodes may have larger amount of memory, or specialized resources such as +> Graphical Processing Units (GPUs). +{: .callout} + +> ## Dedicated Transfer Nodes +> +> If you want to transfer larger amounts of data to or from the cluster, NeSI +> offers dedicated transfer nodes using the Globus service. More information on using Globus for large data transfer to and from +> the cluster can be found here: [Globus Transfer Service](https://support.nesi.org.nz/hc/en-gb/sections/360000040596) +{: .callout} + +{% include links.md %} +[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard \ No newline at end of file diff --git a/_episodes/02-filesystem.md b/_episodes/02-filesystem.md new file mode 100644 index 00000000..0c4671a6 --- /dev/null +++ b/_episodes/02-filesystem.md @@ -0,0 +1,213 @@ +--- +title: "NeSI Filesystem" +teaching: 30 +exercises: 10 +questions: +- "How can I move around the cluster filesystem" +- "How can I see what files and directories I have?" +- "How can I make new files and directories." +objectives: +- "Create, edit, manipulate and remove files from command line" +keypoints: +- "The file system is responsible for managing information on the disk." + +--- + +The NeSI filesystem looks something like this: + +![The file system is made up of a root directory that contains sub-directories +titled home, nesi, and system files](../fig/NesiFiletree.svg) + +The directories that are relevant to us are. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LocationDefault StorageDefault FilesBackupAccess Speed
Home is for user-specific files such as configuration files, environment setup, source code, etc./home/<username>20GB1,000,000DailyNormal
Project is for persistent project-related data, project-related software, etc./nesi/project/<projectcode>100GB100,000DailyNormal
Nobackup is a 'scratch space', for data you don't need to keep long term. Old data is periodically deleted from nobackup/nesi/nobackup/<projectcode>10TB1,000,000NoneFast
+ +### Managing your data and storage (backups and quotas) + +NeSI performs backups of the `/home` and `/nesi/project` (persistent) filesystems. However, backups are only captured once per day. So, if you edit or change code or data and then immediately delete it, it likely cannot be recovered. Note, as the name suggests, NeSI does **not** backup the `/nesi/nobackup` filesystem. + +Protecting critical data from corruption or deletion is primarily your +responsibility. Ensure you have a data management plan and stick to the plan to reduce the chance of data loss. Also important is managing your storage quota. To check your quotas, use the `nn_storage_quota` command, eg + +{% include {{ site.snippets }}/filedir/sinfo.snip %} + +As well as disk space, 'inodes' are also tracked, this is the *number* of files. + +Notice that the project space for this user is over quota and has been locked, meaning no more data can be added. When your space is locked you will need to move or remove data. Also note that none of the nobackup space is being used. Likely data from project can be moved to nobackup. `nn_storage_quota` uses cached data, and so will no immediately show changes to storage use. + +For more details on our persistent and nobackup storage systems, including data retention and the nobackup autodelete schedule, please see our [Filesystem and Quota](https://support.nesi.org.nz/hc/en-gb/articles/360000177256-NeSI-File-Systems-and-Quotas) documentation. + +### Working Directory + +We will be working from the directory `{ site.working_dir[-1] }}`. + +``` +{{ site.remote.prompt }} cd {{ site.working_dir | join: '/' }} +``` + +{: .language-bash} + +### Creating directories + + + +As previously mentioned, it is general useful to organise your work in a hierarchical file structure to make managing and finding files easier. It is also is especially important when working within a shared directory with colleagues, such as a project, to minimise the chance of accidentally affecting your colleagues work. So for this workshop you will each make a directory using the `mkdir` command within the workshops directory for you to personally work from. + +``` +{{ site.remote.prompt }} mkdir +``` + +{: .language-bash} + +You should then be able to see your new directory is there using `ls`. + +``` +{{ site.remote.prompt }} ls {{ site.working_dir | join: '/' }} +``` + +{: .language-bash} + +{% include {{ site.snippets }}/filedir/dir-contents1.snip %} + +## Create a text file + +Now let's create a file. To do this we will use a text editor called Nano to create a file called `draft.txt`: + +We will want to do this from inside the directory we just created. + +``` +{{ site.remote.prompt }} cd +{{ site.remote.prompt }} nano draft.txt +``` + +{: .language-bash} + +> ## Which Editor? +> +> When we say, '`nano` is a text editor' we really do mean 'text': it can +> only work with plain character data, not tables, images, or any other +> human-friendly media. We use it in examples because it is one of the +> least complex text editors. However, because of this trait, it may +> not be powerful enough or flexible enough for the work you need to do +> after this workshop. On Unix systems (such as Linux and macOS), +> many programmers use [Emacs](http://www.gnu.org/software/emacs/) or +> [Vim](http://www.vim.org/) (both of which require more time to learn), +> or a graphical editor such as +> [Gedit](http://projects.gnome.org/gedit/). On Windows, you may wish to +> use [Notepad++](http://notepad-plus-plus.org/). Windows also has a built-in +> editor called `notepad` that can be run from the command line in the same +> way as `nano` for the purposes of this lesson. +> +> No matter what editor you use, you will need to know where it searches +> for and saves files. If you start it from the shell, it will (probably) +> use your current working directory as its default location. If you use +> your computer's start menu, it may want to save files in your desktop or +> documents directory instead. You can change this by navigating to +> another directory the first time you 'Save As...' +{: .callout} + +Let's type in a few lines of text. +Once we're happy with our text, we can press Ctrl+O +(press the Ctrl or Control key and, while +holding it down, press the O key) to write our data to disk +(we'll be asked what file we want to save this to: +press Return to accept the suggested default of `draft.txt`). + +
screenshot of nano text editor in action
+ +Once our file is saved, we can use Ctrl+X to quit the editor and +return to the shell. + +> ## Control, Ctrl, or ^ Key +> +> The Control key is also called the 'Ctrl' key. There are various ways +> in which using the Control key may be described. For example, you may +> see an instruction to press the Control key and, while holding it down, +> press the X key, described as any of: +> +> * `Control-X` +> * `Control+X` +> * `Ctrl-X` +> * `Ctrl+X` +> * `^X` +> * `C-x` +> +> In nano, along the bottom of the screen you'll see `^G Get Help ^O WriteOut`. +> This means that you can use `Control-G` to get help and `Control-O` to save your +> file. +{: .callout} + +`nano` doesn't leave any output on the screen after it exits, +but `ls` now shows that we have created a file called `draft.txt`: + +``` +{{ site.remote.prompt }} ls +``` + +{: .language-bash} + +``` +draft.txt +``` + +{: .output} + +## Copying files and directories + +In a future lesson, we will be running the R script ```{{ site.working_dir | join: '/' }}/{{ site.example.script }} ```, but as we can't all work on the same file at once you will need to take your own copy. This can be done with the **c**o**p**y command `cp`, at least two arguments are needed the file (or directory) you want to copy, and the directory (or file) where you want the copy to be created. We will be copying the file into the directory we made previously, as this should be your current directory the second argument can be a simple `.`. + +``` +{{ site.remote.prompt }} cp {{ site.working_dir | join: '/' }}/{{ site.example.script }} . +``` + +{: .output} + +We can check that it did the right thing using `ls` + +``` +{{ site.remote.prompt }} ls +``` + +{: .language-bash} + +``` +draft.txt {{ site.example.script }} +``` + +{: .output} diff --git a/_episodes/02-unix-shell.md b/_episodes/02-unix-shell.md new file mode 100644 index 00000000..b01fe93b --- /dev/null +++ b/_episodes/02-unix-shell.md @@ -0,0 +1,803 @@ +--- +title: "Navigating Files and Directories" +teaching: 30 +exercises: 10 +questions: +- "How can I move around the cluster filesystem" +- "How can I see what files and directories I have?" +- "How can I make new files and directories." +objectives: +- "Create, edit, manipulate and remove files from command line" +- "Translate an absolute path into a relative path and vice versa." +- "Use options and arguments to change the behaviour of a shell command." +- "Demonstrate the use of tab completion and explain its advantages." +keypoints: +- "The file system is responsible for managing information on the disk." +- "Information is stored in files, which are stored in directories (folders)." +- "Directories can also store other directories, which then form a directory tree." +- "`cd [path]` changes the current working directory." +- "`ls [path]` prints a listing of a specific file or directory; `ls` on its own lists the current working directory." +- "`pwd` prints the user's current working directory." +- "`cp [file] [path]` copies [file] to [path]" +- "`mv [file] [path]` moves [file] to [path]" +- "`rm [file]` deletes [file]" +- "`/` on its own is the root directory of the whole file system." +- "Most commands take options (flags) that begin with a `-`." +- "A relative path specifies a location starting from the current location." +- "An absolute path specifies a location from the root of the file system." +- "Directory names in a path are separated with `/` on Unix, but `\\` on Windows." +- "`..` means 'the directory above the current one'; `.` on its own means 'the current directory'." +--- +> ## The Unix Shell +> +> This episode will be a quick introduction to the Unix shell, only the bare minimum required to use the cluster. +> +> The Software Carpentry '[Unix Shell](https://swcarpentry.github.io/shell-novice/)' lesson covers the subject in more depth, we recommend you check it out. +> +{: .callout} + +The part of the operating system responsible for managing files and directories +is called the **file system**. +It organizes our data into files, +which hold information, +and directories (also called 'folders'), +which hold files or other directories. + +Understanding how to navigate the file system using command line is essential for using an HPC. + +Directories are like *places* — at any time +while we are using the shell, we are in exactly one place called +our **current working directory**. +Commands mostly read and write files in the +current working directory, i.e. 'here', so knowing where you are before running +a command is important. + +First, let's find out where we are by running the command `pwd` for '**p**rint **w**orking **d**irectory'. + +``` +{{ site.remote.prompt }} pwd +``` + +{: .language-bash} + +``` +/home/ +``` + +{: .output} + +The output we see is what is known as a 'path'. +The path can be thought of as a series of directions given to navigate the file system. + +At the top is the **root directory** +that holds all the files in a filesystem. + +We refer to it using a slash character, `/`, on its own. +This is what the leading slash in `/home/` is referring to, it is telling us our path starts at the root directory. + +Next is `home`, as it is the next part of the path we know it is inside the root directory, +we also know that home is another directory as the path continues. +Finally, stored inside `home` is the directory with your username. + +#### Version Control + +Version control systems (such as Git) often have free, cloud-based offerings +(e.g., BitBucket, GitHub and GitLab) that are generally used for storing source code. Even +if you are not writing your own programs, these can be very useful for storing +job submit scripts, notes and other files. Git is not an appropriate solution for storing data. + +> ## Slashes +> +> Notice that there are two meanings for the `/` character. +> When it appears at the front of a file or directory name, +> it refers to the root directory. When it appears *inside* a path, +> it's just a separator. +{: .callout} + +As you may now see, using a bash shell is strongly dependent on the idea that +your files are organized in a hierarchical file system. +Organizing things hierarchically in this way helps us keep track of our work: +it's possible to put hundreds of files in our home directory, +just as it's possible to pile hundreds of printed papers on our desk, +but it's a self-defeating strategy. + +## Listing the contents of directories + +To **l**i**s**t the contents of a directory, we use the command `ls` followed by the path to the directory whose contents we want listed. + +We will now list the contents of the directory we we will be working from. We can +use the following command to do this: + +``` +{{ site.remote.prompt }} ls {{ site.working_dir[0] }} +``` + +{: .language-bash} + +``` +{{ site.working_dir[1] }} +``` + +{: .output} + +You should see a directory called `{{ site.working_dir[1] }}`, and possibly several other directories. For the purposes of this workshop you will be working within `{{ site.working_dir | join: '/' }}` + +> ## Command History +> +> You can cycle through your previous commands with the ↑ and ↓ keys. +> A convenient way to repeat your last command is to type ↑ then enter. +> +{: .callout} + +> ## `ls` Reading Comprehension +> +> What command would you type to get the following output +> +> ``` +> original pnas_final pnas_sub +> ``` +> +> {: .output} +> +> ![A directory tree below the Users directory where "/Users" contains the +directories "backup" and "thing"; "/Users/backup" contains "original", +"pnas_final" and "pnas_sub"; "/Users/thing" contains "backup"; and +"/Users/thing/backup" contains "2012-12-01", "2013-01-08" and +"2013-01-27"](../fig/filesystem-challenge.svg) +> +> 1. `ls pwd` +> 2. `ls backup` +> 3. `ls /Users/backup` +> 4. `ls /backup` +> +> > ## Solution +> > +> > 1. No: `pwd` is not the name of a directory. +> > 2. Possibly: It depends on your current directory (we will explore this more shortly). +> > 3. Yes: uses the absolute path explicitly. +> > 4. No: There is no such directory. +> {: .solution} +{: .challenge} + +## Moving about + +Currently we are still in our home directory, we want to move into the`project` directory from the previous command. + +The command to **c**hange **d**irectory is `cd` followed by the path to the directory we want to move to. + +The `cd` command is akin to double clicking a folder in a graphical interface. + +We will use the following command: + +``` +{{ site.remote.prompt }} cd {{ site.working_dir | join: '/' }} +``` + +{: .language-bash} + +``` +``` + +{: .output} +You will notice that `cd` doesn't print anything. This is normal. Many shell commands will not output anything to the screen when successfully executed. +We can check we are in the right place by running `pwd`. + +``` +{{ site.remote.prompt }} pwd +``` + +{: .language-bash} + +``` +{{ site.working_dir | join: '/' }} +``` + +{: .output} + +## Creating directories + + + +As previously mentioned, it is general useful to organise your work in a hierarchical file structure to make managing and finding files easier. It is also is especially important when working within a shared directory with colleagues, such as a project, to minimise the chance of accidentally affecting your colleagues work. So for this workshop you will each make a directory using the `mkdir` command within the workshops directory for you to personally work from. + +``` +{{ site.remote.prompt }} mkdir +``` + +{: .language-bash} + +You should then be able to see your new directory is there using `ls`. + +``` +{{ site.remote.prompt }} ls {{ site.working_dir | join: '/' }} +``` + +{: .language-bash} + +{% include {{ site.snippets }}/filedir/dir-contents1.snip %} + +## General Syntax of a Shell Command + +We are now going to use `ls` again but with a twist, this time we will also use what are known as **options**, **flags** or **switches**. +These options modify the way that the command works, for this example we will add the flag `-l` for "long listing format". + +``` +{{ site.remote.prompt }} ls -l {{ site.working_dir | join: '/' }} +``` + +{: .language-bash} + +{% include {{ site.snippets }}/filedir/dir-contents2.snip %} + +We can see that the `-l` option has modified the command and now our output has listed all the files in alphanumeric order, which can make finding a specific file easier. +It also includes information about the file size, time of its last modification, and permission and ownership information. + +Most unix commands follow this basic structure. +![Structure of a Unix command](../fig/Unix_Command_Struc.svg) + +The **prompt** tells us that the terminal is accepting inputs, prompts can be customised to show all sorts of info. + +The **command**, what are we trying to do. + +**Options** will modify the behavior of the command, multiple options can be specified. +Options will either start with a single dash (`-`) or two dashes (`--`).. +Often options will have a short and long format e.g. `-a` and `--all`. + +**Arguments** tell the command what to operate on (usually files and directories). + +Each part is separated by spaces: if you omit the space +between `ls` and `-l` the shell will look for a command called `ls-l`, which +doesn't exist. Also, capitalization can be important. +For example, `ls -s` will display the size of files and directories alongside the names, +while `ls -S` will sort the files and directories by size. + +Another useful option for `ls` is the `-a` option, lets try using this option together with the `-l` option: + +``` +{{ site.remote.prompt }} ls -la +``` + +{: .language-bash} + +{% include {{ site.snippets }}/filedir/dir-contents3.snip %} + +Single letter options don't usually need to be separate. In this case `ls -la` is performing the same function as if we had typed `ls -l -a`. + +You might notice that we now have two extra lines for directories `.` and `..`. These are hidden directories which the `-a` option has been used to reveal, you can make any file or directory hidden by beginning their filenames with a `.`. + +These two specific hidden directories are special as they will exist hidden inside every directory, with the `.` hidden directory representing your current directory and the `..` hidden directory representing the **parent** directory above your current directory. + +> ## Exploring More `ls` Flags +> +> You can also use two options at the same time. What does the command `ls` do when used +> with the `-l` option? What about if you use both the `-l` and the `-h` option? +> +> Some of its output is about properties that we do not cover in this lesson (such +> as file permissions and ownership), but the rest should be useful +> nevertheless. +> +> > ## Solution +> > +> > The `-l` option makes `ls` use a **l**ong listing format, showing not only +> > the file/directory names but also additional information, such as the file size +> > and the time of its last modification. If you use both the `-h` option and the `-l` option, +> > this makes the file size '**h**uman readable', i.e. displaying something like `5.3K` +> > instead of `5369`. +> {: .solution} +{: .challenge} + +## Relative paths + +You may have noticed in the last command we did not specify an argument for the directory path. +Until now, when specifying directory names, or even a directory path (as above), +we have been using what are know **absolute paths**, which work no matter where you are currently located on the machine +since it specifies the full path from the top level root directory. + +An **absolute path** always starts at the root directory, which is indicated by a +leading slash. The leading `/` tells the computer to follow the path from +the root of the file system, so it always refers to exactly one directory, +no matter where we are when we run the command. + +Any path without a leading `/` is a **relative path**. + +When you use a relative path with a command +like `ls` or `cd`, it tries to find that location starting from where we are, +rather than from the root of the file system. + +In the previous command, since we did not specify an **absolute path** it ran the command on the relative path from our current directory +(implicitly using the `.` hidden directory), and so listed the contents of our current directory. + +We will now navigate to the parent directory, the simplest way do do this is to use the relative path `..`. + +``` +{{ site.remote.prompt }} cd .. +``` + +{: .language-bash} + +We should now be back in `{{ site.working_dir[0] }}`. + +``` +{{ site.remote.prompt }} pwd +``` + +{: .language-bash} + +``` +{{ site.working_dir[0] }} +``` + +{: .output} + +> ## Two More Shortcuts +> +> The shell interprets a tilde (`~`) character at the start of a path to +> mean "the current user's home directory". For example, if Nelle's home +> directory is `/home/nelle`, then `~/data` is equivalent to +> `/home/nelle/data`. This only works if it is the first character in the +> path: `here/there/~/elsewhere` is *not* `here/there//home/nelle/elsewhere`. +> +> Another shortcut is the `-` (dash) character. `cd` will translate `-` into +> *the previous directory I was in*, which is faster than having to remember, +> then type, the full path. This is a *very* efficient way of moving +> *back and forth between two directories* -- i.e. if you execute `cd -` twice, +> you end up back in the starting directory. +> +> The difference between `cd ..` and `cd -` is +> that the former brings you *up*, while the latter brings you *back*. +> +{: .callout} + +> ## Absolute vs Relative Paths +> +> Starting from `/home/amanda/data`, +> which of the following commands could Amanda use to navigate to her home directory, +> which is `/home/amanda`? +> +> 1. `cd .` +> 2. `cd /` +> 3. `cd home/amanda` +> 4. `cd ../..` +> 5. `cd ~` +> 6. `cd home` +> 7. `cd ~/data/..` +> 8. `cd` +> 9. `cd ..` +> +> > ## Solution +> > +> > 1. No: `.` stands for the current directory. +> > 2. No: `/` stands for the root directory. +> > 3. No: Amanda's home directory is `/home/amanda`. +> > 4. No: this command goes up two levels, i.e. ends in `/home`. +> > 5. Yes: `~` stands for the user's home directory, in this case `/home/amanda`. +> > 6. No: this command would navigate into a directory `home` in the current directory if it exists. +> > 7. Yes: unnecessarily complicated, but correct. +> > 8. Yes: shortcut to go back to the user's home directory. +> > 9. Yes: goes up one level. +> {: .solution} +{: .challenge} + +> ## Relative Path Resolution +> +> Using the filesystem diagram below, if `pwd` displays `/Users/thing`, +> what will `ls ../backup` display? +> +> 1. `../backup: No such file or directory` +> 2. `2012-12-01 2013-01-08 2013-01-27` +> 3. `original pnas_final pnas_sub` +> +> ![A directory tree below the Users directory where "/Users" contains the +directories "backup" and "thing"; "/Users/backup" contains "original", +"pnas_final" and "pnas_sub"; "/Users/thing" contains "backup"; and +"/Users/thing/backup" contains "2012-12-01", "2013-01-08" and +"2013-01-27"](../fig/filesystem-challenge.svg) +> +> > ## Solution +> > +> > 1. No: there *is* a directory `backup` in `/Users`. +> > 2. No: this is the content of `Users/thing/backup`, +> > but with `..`, we asked for one level further up. +> > 3. Yes: `../backup/` refers to `/Users/backup/`. +> > +> {: .solution} +{: .challenge} + +> ## Clearing your terminal +> +> If your screen gets too cluttered, you can clear your terminal using the +> `clear` command. You can still access previous commands using ↑ +> and ↓ to move line-by-line, or by scrolling in your terminal. +{: .callout} + +> ## Listing in Reverse Chronological Order +> +> By default, `ls` lists the contents of a directory in alphabetical +> order by name. The command `ls -t` lists items by time of last +> change instead of alphabetically. The command `ls -r` lists the +> contents of a directory in reverse order. +> Which file is displayed last when you combine the `-t` and `-r` flags? +> Hint: You may need to use the `-l` flag to see the +> last changed dates. +> +> > ## Solution +> > +> > The most recently changed file is listed last when using `-rt`. This +> > can be very useful for finding your most recent edits or checking to +> > see if a new output file was written. +> {: .solution} +{: .challenge} + +> ## Globbing +> +> One of the most powerful features of bash is *filename expansion*, otherwise known as *globbing*. +> This allows you to use *patterns* to match a file name (or multiple files), +> which will then be operated on as if you had typed out all of the matches. +> +> `*` is a **wildcard**, which matches zero or more characters. +> +> Consider a directory containing the following files +> `kaka.txt kakapo.jpeg kea.txt kiwi.jpeg pukeko.jpeg` +> +> The pattern `ka*` will match `kaka.txt`and `kakapo.jpeg` as these both start with "ka". +> Where as `*.jpeg` will match `kakapo.jpeg`, `kiwi.jpeg` and `pukeko.jpeg` as they all end in ".jpeg" +> `k*a.*` will match just `kaka.txt` and `kea.txt` +> +> `?` is also a wildcard, but it matches exactly one character. +> +> `????.*` would return `kaka.txt` and `kiwi.jpeg`. +> +> When the shell sees a wildcard, it expands the wildcard to create a +> list of matching filenames *before* running the command that was +> asked for. As an exception, if a wildcard expression does not match +> any file, Bash will pass the expression as an argument to the command +> as it is. +> However, generally commands like `wc` and `ls` see the lists of +> file names matching these expressions, but not the wildcards +> themselves. It is the shell, not the other programs, that deals with +> expanding wildcards. +{: .callout} + +> ## List filenames matching a pattern +> +> Running `ls` in a directory gives the output +> `cubane.pdb ethane.pdb methane.pdb octane.pdb pentane.pdb propane.pdb` +> +> Which `ls` command(s) will +> produce this output? +> +> `ethane.pdb methane.pdb` +> +> 1. `ls *t*ane.pdb` +> 2. `ls *t?ne.*` +> 3. `ls *t??ne.pdb` +> 4. `ls ethane.*` +> +>> ## Solution +>> +>> The solution is `3.` +>> +>> `1.` shows all files whose names contain zero or more characters (`*`) +>> followed by the letter `t`, +>> then zero or more characters (`*`) followed by `ane.pdb`. +>> This gives `ethane.pdb methane.pdb octane.pdb pentane.pdb`. +>> +>> `2.` shows all files whose names start with zero or more characters (`*`) followed by +>> the letter `t`, +>> then a single character (`?`), then `ne.` followed by zero or more characters (`*`). +>> This will give us `octane.pdb` and `pentane.pdb` but doesn't match anything +>> which ends in `thane.pdb`. +>> +>> `3.` fixes the problems of option 2 by matching two characters (`??`) between `t` and `ne`. +>> This is the solution. +>> +>> `4.` only shows files starting with `ethane.`. +> {: .solution} +{: .challenge} + +include in terminal excersise (delete slurm files later on maybe?) + +> ## Tab completion +> +> Sometimes file paths and file names can be very long, making typing out the path tedious. +> One trick you can use to save yourself time is to use something called **tab completion**. +> If you start typing the path in a command and their is only one possible match, +> if you hit tab the path will autocomplete (until there are more than one possible matches). +> +> > ## Solution +> > +> >For example, if you type: +> > +> > ``` +> > {{ site.remote.prompt }} cd {{ site.working_dir | last | slice: 0,3 }} +> > ``` +> > +> > {: .language-bash} +> > +> > and then press Tab (the tab key on your keyboard), +> > the shell automatically completes the directory name for you (since there is only one possible match): +> > +> > ``` +> > {{ site.remote.prompt }} cd {{ site.working_dir | last }}/ +> > ``` +> > +> > {: .language-bash} +> > +> > However, you want to move to your personal working directory. If you hit Tab once you will +> > likely see nothing change, as there are more than one possible options. Hitting Tab +> > a second time will print all possible autocomplete options. +> > +> > ``` +> > cwal219/ riom/ harrellw/ +> > ``` +> > +> > {: .output} +> > +> >Now entering in the first few characters of the path (just enough that the possible options are no longer ambiguous) and pressing Tab again, should complete the path. +> > +> > Now press Enter to execute the command. +> > +> > ``` +> > {{ site.remote.prompt }} cd {{ site.working_dir | last }}/ +> > ``` +> > +> > {: .language-bash} +> > +> > Check that we've moved to the right place by running `pwd`. +> > +> > ``` +> > {{ site.working_dir | join: '/' }}/ +> > ``` +> > +> > {: .output} +> {: .solution} +{: .challenge} + +## Create a text file + +Now let's create a file. To do this we will use a text editor called Nano to create a file called `draft.txt`: + +``` +{{ site.remote.prompt }} nano draft.txt +``` + +{: .language-bash} + +> ## Which Editor? +> +> When we say, '`nano` is a text editor' we really do mean 'text': it can +> only work with plain character data, not tables, images, or any other +> human-friendly media. We use it in examples because it is one of the +> least complex text editors. However, because of this trait, it may +> not be powerful enough or flexible enough for the work you need to do +> after this workshop. On Unix systems (such as Linux and macOS), +> many programmers use [Emacs](http://www.gnu.org/software/emacs/) or +> [Vim](http://www.vim.org/) (both of which require more time to learn), +> or a graphical editor such as +> [Gedit](http://projects.gnome.org/gedit/). On Windows, you may wish to +> use [Notepad++](http://notepad-plus-plus.org/). Windows also has a built-in +> editor called `notepad` that can be run from the command line in the same +> way as `nano` for the purposes of this lesson. +> +> No matter what editor you use, you will need to know where it searches +> for and saves files. If you start it from the shell, it will (probably) +> use your current working directory as its default location. If you use +> your computer's start menu, it may want to save files in your desktop or +> documents directory instead. You can change this by navigating to +> another directory the first time you 'Save As...' +{: .callout} + +Let's type in a few lines of text. +Once we're happy with our text, we can press Ctrl+O +(press the Ctrl or Control key and, while +holding it down, press the O key) to write our data to disk +(we'll be asked what file we want to save this to: +press Return to accept the suggested default of `draft.txt`). + +
screenshot of nano text editor in action
+ +Once our file is saved, we can use Ctrl+X to quit the editor and +return to the shell. + +> ## Control, Ctrl, or ^ Key +> +> The Control key is also called the 'Ctrl' key. There are various ways +> in which using the Control key may be described. For example, you may +> see an instruction to press the Control key and, while holding it down, +> press the X key, described as any of: +> +> * `Control-X` +> * `Control+X` +> * `Ctrl-X` +> * `Ctrl+X` +> * `^X` +> * `C-x` +> +> In nano, along the bottom of the screen you'll see `^G Get Help ^O WriteOut`. +> This means that you can use `Control-G` to get help and `Control-O` to save your +> file. +{: .callout} + +`nano` doesn't leave any output on the screen after it exits, +but `ls` now shows that we have created a file called `draft.txt`: + +``` +{{ site.remote.prompt }} ls +``` + +{: .language-bash} + +``` +draft.txt +``` + +{: .output} + +## Copying files and directories + +In a future lesson, we will be running the R script ```{{ site.working_dir | join: '/' }}/{{ site.example.script }} ```, but as we can't all work on the same file at once you will need to take your own copy. This can be done with the **c**o**p**y command `cp`, at least two arguments are needed the file (or directory) you want to copy, and the directory (or file) where you want the copy to be created. We will be copying the file into the directory we made previously, as this should be your current directory the second argument can be a simple `.`. + +``` +{{ site.remote.prompt }} cp {{ site.working_dir | join: '/' }}/{{ site.example.script }} . +``` + +{: .output} + +We can check that it did the right thing using `ls` + +``` +{{ site.remote.prompt }} ls +``` + +{: .language-bash} + +``` +draft.txt {{ site.example.script }} +``` + +{: .output} + +## Other File operations + +`mv` to **m**o**v**e move a file, is used similarly to `cp` taking a source argument(s) and a destination argument. +`rm` will **r**e**m**ove move a file and only needs one argument. + +The `mv` command is also used to rename a file, for example `mv my_fiel my_file`. This is because as far as the computer is concerned *moving and renaming a file are the same operation*. + +In order to `cp` a directory (and all its contents) the `-r` for [recursive](https://en.wikipedia.org/wiki/Recursion) option must be used. +The same is true when deleting directories with `rm` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
commandnameusage
cpcopycp file1 file2
cp -r directory1/ directory2/
mvmovemv file1 file2
mv directory1/ directory2/
rmremoverm file1 file2
rm -r directory1/ directory2/
+ +For `mv` and `cp` if the destination path (final argument) is an existing directory the file will be placed inside that directory with the same name as the source. + +> ## Moving vs Copying +> +> When using the `cp` or `rm` commands on a directory the 'recursive' flag `-r` must be used, but `mv` *does not* require it? +> +>> ## Solution +>> +>> We mentioned previously that as far the computer is concerned, *renaming* is the same operation as *moving*. +>> Contrary to what the commands name implies, *all moving is actually renaming*. +>> The data on the hard drive stays in the same place, +>> only the label applied to that block of memory is changed. +>> To copy a directory, each *individual file* inside that directory must be read, and then written to the copy destination. +>> To delete a directory, each *individual file* in the directory must be marked for deletion, +>> however when moving a directory the files inside are the data inside the directory is not interacted with, +>> only the parent directory is "renamed" to a different place. +>> +>> This is also why `mv` is faster than `cp` as no reading of the files is required. +> {: .solution} +{: .challenge} + +> ## Unsupported command-line options +> +> If you try to use an option (flag) that is not supported, `ls` and other commands +> will usually print an error message similar to: +> +> ``` +> $ ls -j +> ``` +> +> {: .language-bash} +> +> ``` +> ls: invalid option -- 'j' +> Try 'ls --help' for more information. +> ``` +> +> {: .error} +{: .callout} + +## Getting help + +Commands will often have many **options**. Most commands have a `--help` flag, as can be seen in the error above. You can also use the manual pages (aka manpages) by using the `man` command. The manual page provides you with all the available options and their use in more detail. For example, for thr `ls` command: + +``` +{{ site.remote.prompt }} man ls +``` + +{: .language-bash} + +``` +Usage: ls [OPTION]... [FILE]... +List information about the FILEs (the current directory by default). +Sort entries alphabetically if neither -cftuvSUX nor --sort is specified. + +Mandatory arguments to long options are mandatory for short options, too. + -a, --all do not ignore entries starting with . + -A, --almost-all do not list implied . and .. + --author with -l, print the author of each file + -b, --escape print C-style escapes for nongraphic characters + --block-size=SIZE scale sizes by SIZE before printing them; e.g., + '--block-size=M' prints sizes in units of + 1,048,576 bytes; see SIZE format below + -B, --ignore-backups do not list implied entries ending with ~ + -c with -lt: sort by, and show, ctime (time of last + modification of file status information); + with -l: show ctime and sort by name; + otherwise: sort by ctime, newest first + -C list entries by columns + --color[=WHEN] colorize the output; WHEN can be 'always' (default + if omitted), 'auto', or 'never'; more info below + -d, --directory list directories themselves, not their contents + -D, --dired generate output designed for Emacs' dired mode + -f do not sort, enable -aU, disable -ls --color + -F, --classify append indicator (one of */=>@|) to entries +...       ...       ... +``` + +{: .output} + +To navigate through the `man` pages, +you may use ↑ and ↓ to move line-by-line, +or try B and Spacebar to skip up and down by a full page. +To search for a character or word in the `man` pages, +use / followed by the character or word you are searching for. +Sometimes a search will result in multiple hits. If so, you can move between hits using N (for moving forward) and Shift+N (for moving backward). + +To **quit** the `man` pages, press Q. + +> ## Manual pages on the web +> +> Of course, there is a third way to access help for commands: +> searching the internet via your web browser. +> When using internet search, including the phrase `unix man page` in your search +> query will help to find relevant results. +> +> GNU provides links to its +> [manuals](http://www.gnu.org/manual/manual.html) including the +> [core GNU utilities](http://www.gnu.org/software/coreutils/manual/coreutils.html), +> which covers many commands introduced within this lesson. +{: .callout} diff --git a/_episodes/03-break1.md b/_episodes/03-break1.md new file mode 100644 index 00000000..65ba3c2d --- /dev/null +++ b/_episodes/03-break1.md @@ -0,0 +1,5 @@ +--- +title: "Morning Break" +layout: break +break: 10 +--- \ No newline at end of file diff --git a/_episodes/035-filedir-cont.md b/_episodes/035-filedir-cont.md new file mode 100644 index 00000000..ff893324 --- /dev/null +++ b/_episodes/035-filedir-cont.md @@ -0,0 +1,5 @@ +--- +title: "Navigating Files and Directories (Continued)" +layout: break +break: 50 +--- \ No newline at end of file diff --git a/_episodes/04-modules.md b/_episodes/04-modules.md new file mode 100644 index 00000000..d0812c54 --- /dev/null +++ b/_episodes/04-modules.md @@ -0,0 +1,258 @@ +--- +title: "Accessing software via Modules" +teaching: 20 +exercises: 10 +questions: +- "How do we load and unload software packages?" +objectives: +- "Load and use a software package." +- "Explain how the shell environment changes when the module mechanism loads or unloads packages." +keypoints: +- "Load software with `module load softwareName`." +- "Unload software with `module unload`" +- "The module system handles software versioning and package conflicts for you + automatically." +--- + +On a high-performance computing system, it is seldom the case that the software +we want to use is available when we log in. It is installed, but we will need +to "load" it before it can run. + +Before we start using individual software packages, however, we should +understand the reasoning behind this approach. The three biggest factors are: + +- software incompatibilities +- versioning +- dependencies + +Software incompatibility is a major headache for programmers. Sometimes the +presence (or absence) of a software package will break others that depend on +it. Two of the most famous examples are Python 2 and 3 and C compiler versions. +Python 3 famously provides a `python` command that conflicts with that provided +by Python 2. Software compiled against a newer version of the C libraries and +then used when they are not present will result in a nasty `'GLIBCXX_3.4.20' +not found` error, for instance. + + + +Software versioning is another common issue. A team might depend on a certain +package version for their research project - if the software version was to +change (for instance, if a package was updated), it might affect their results. +Having access to multiple software versions allows a set of researchers to +prevent software versioning issues from affecting their results. + +Dependencies are where a particular software package (or even a particular +version) depends on having access to another software package (or even a +particular version of another software package). For example, the VASP +materials science software may depend on having a particular version of the +FFTW (Fastest Fourier Transform in the West) software library available for it +to work. + +## Environment + +Before understanding environment modules we first need to understand what is meant by _environment_. + +The environment is defined by it's _environment variables_. + +_Environment Variables_ are writable named-variables. + +We can assign a variable named "FOO" with the value "bar" using the syntax. + +``` +{{ site.remote.prompt }} FOO="bar" +``` +{: .language-bash} + +Convention is to name fixed variables in all caps. + +Our new variable can be referenced using `$FOO`, you could also use `${FOO}`, +enclosing a variable in curly brackets is good practice as it avoids possible ambiguity. + +``` +{{ site.remote.prompt }} $FOO +``` +{: .language-bash} + +``` +-bash: bar: command not found +``` +{: .output} + +We got an error here because the variable is evalued _in the terminal_ then executed. +If we just want to print the variable we can use the command, + +``` +{{ site.remote.prompt }} echo $FOO +``` +{: .language-bash} +``` +bar +``` +{: .output} + +We can get a full list of environment variables using the command, + +``` +{{ site.remote.prompt }} env +``` +{: .language-bash} +{% include {{ site.snippets }}/modules/env-output.snip %} + +These variables control many aspects of how your terminal, and any software launched from your terminal works. + +## Environment Modules + +Environment modules are the solution to these problems. A _module_ is a +self-contained description of a software package -- it contains the +settings required to run a software package and, usually, encodes required +dependencies on other software packages. + +There are a number of different environment module implementations commonly +used on HPC systems: the two most common are _TCL modules_ and _Lmod_. Both of +these use similar syntax and the concepts are the same so learning to use one +will allow you to use whichever is installed on the system you are using. In +both implementations the `module` command is used to interact with environment +modules. An additional subcommand is usually added to the command to specify +what you want to do. For a list of subcommands you can use `module -h` or +`module help`. As for all commands, you can access the full help on the _man_ +pages with `man module`. + +### Purging Modules + +Depending on how you are accessing the HPC the modules you have loaded by default will be different. So before we start listing our modules we will first use the `module purge` command to clear all but the minimum default modules so that we are all starting with the same modules. + +``` +{{ site.remote.prompt }} module purge +``` +{: .language-bash} + +``` + +The following modules were not unloaded: + (Use "module --force purge" to unload all): + + 1) XALT/minimal 2) slurm 3) NeSI +``` +{: .output} + +Note that `module purge` is informative. It lets us know that all but a minimal default +set of packages have been unloaded (and how to actually unload these if we +truly so desired). + +We are able to unload individual modules, unfortunately within the NeSI system it does not always unload it's dependencies, therefore we recommend `module purge` to bring you back to a state where only those modules needed to perform your normal work on the cluster. + +`module purge` is a useful tool for ensuring repeatable research by guaranteeing that the environment that you build your software stack from is always the same. This is important since some modules have the potential to silently effect your results if they are loaded (or not loaded). + +### Listing Available Modules + +To see available software modules, use `module avail`: + +``` +{{ site.remote.prompt }} module avail +``` +{: .language-bash} + +{% include {{ site.snippets }}/modules/available-modules.snip %} + +### Listing Currently Loaded Modules + +You can use the `module list` command to see which modules you currently have +loaded in your environment. On {{ site.remote.name }} you will have a few default modules loaded when you login. + +``` +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +{% include {{ site.snippets }}/modules/module-list-default.snip %} + +## Loading and Unloading Software + +You can load software using the `module load` command. In this example we will be using the programming language _R_. + +Initially, R is not loaded. We can test this by using the `which` +command. `which` looks for programs the same way that Bash does, so we can use +it to tell us where a particular piece of software is stored. + +``` +{{ site.remote.prompt }} which R +``` +{: .language-bash} + +{% include {{ site.snippets }}/modules/missing-r.snip %} + +The important bit here being: + +``` +/usr/bin/which: no R in (...) +``` + +Now lets try loading the R environment module, and try again. + +{% include {{ site.snippets }}/modules/module-load-r.snip %} + +> ## Tab Completion +> +> The module command also supports tab completion. You may find this the easiest way to find the right software. +{: .callout} + +So, what just happened? + +To understand the output, first we need to understand the nature of the `$PATH` +environment variable. `$PATH` is a special environment variable that controls +where a UNIX system looks for software. Specifically `$PATH` is a list of +directories (separated by `:`) that the OS searches through for a command +before giving up and telling us it can't find it. As with all environment +variables we can print it out using `echo`. + +{% include {{ site.snippets }}/modules/r-module-path.snip %} + +You'll notice a similarity to the output of the `which` command. However, in this case, +there are a lot more directories at the beginning. When we +ran the `module load` command, it added many directories to the beginning of our +`$PATH`. + +The path to NeSI XALT utility will normally show up first. This helps us track software usage, but the more important directory is the second one: `/opt/nesi/CS400_centos7_bdw/R/4.2.1-gimkl-2022a/bin` Let's examine what's there: + +{% include {{ site.snippets }}/modules/r-ls-dir-command.snip %} + +`module load` "loads" not only the specified software, but it also loads software dependencies. That is, the software that the application you load requires to run. + +{% include {{ site.snippets }}/modules/software-dependencies.snip %} + +Before moving onto the next session lets use `module purge` again to return to the minimal environment. + +``` +{{ site.remote.prompt }} module purge +``` +{: .language-bash} + +``` +The following modules were not unloaded: + (Use "module --force purge" to unload all): + + 1) XALT/minimal 2) slurm 3) NeSI +``` +{: .output} + +## Software Versioning + +So far, we've learned how to load and unload software packages. However, we have not yet addressed the issue of software versioning. At +some point or other, you will run into issues where only one particular version +of some software will be suitable. Perhaps a key bugfix only happened in a +certain version, or version _X_ broke compatibility with a file format you use. +In either of these example cases, it helps to be very specific about what +software is loaded. + +Let's examine the output of `module avail` more closely. + +``` +{{ site.remote.prompt }} module avail +``` +{: .language-bash} + +{% include {{ site.snippets }}/modules/available-modules.snip %} + +{% include {{ site.snippets }}/modules/wrong-python-version.snip %} + +{% include links.md %} diff --git a/_episodes/05-scheduler.md b/_episodes/05-scheduler.md new file mode 100644 index 00000000..f4d303df --- /dev/null +++ b/_episodes/05-scheduler.md @@ -0,0 +1,338 @@ +--- +title: "Scheduler Fundamentals" +teaching: 20 +exercises: 10 +questions: +- "What is a scheduler and why does a cluster need one?" +- "How do I launch a program to run on a compute node in the cluster?" +- "How do I capture the output of a program that is run on a node in the + cluster?" +objectives: +- "Run a simple script on the login node, and through the scheduler." +- "Use the batch system command line tools to monitor the execution of your + job." +- "Inspect the output and error files of your jobs." +- "Find the right place to put large datasets on the cluster." +keypoints: +- "The scheduler handles how compute resources are shared between users." +- "A job is just a shell script." +- "Request _slightly_ more resources than you will need." +--- + +## Job Scheduler + +An HPC system might have thousands of nodes and thousands of users. How do we +decide who gets what and when? How do we ensure that a task is run with the +resources it needs? This job is handled by a special piece of software called +the _scheduler_. On an HPC system, the scheduler manages which jobs run where +and when. + +The following illustration compares these tasks of a job scheduler to a waiter +in a restaurant. If you can relate to an instance where you had to wait for a +while in a queue to get in to a popular restaurant, then you may now understand +why sometimes your job do not start instantly as in your laptop. + +{% include figure.html max-width="75%" caption="" + file="/fig/restaurant_queue_manager.svg" + alt="Compare a job scheduler to a waiter in a restaurant" %} + +The scheduler used in this lesson is {{ site.sched.name }}. Although +{{ site.sched.name }} is not used everywhere, running jobs is quite similar +regardless of what software is being used. The exact syntax might change, but +the concepts remain the same. + +## Interactive vs Batch + +So far, whenever we have entered a command into our terminals, we have received the response immediately in the same terminal, this is said to be an _interactive session_. + +[//]: # TODO ??Diagram?? + +This is all well for doing small tasks, but what if we want to do several things one after another without without waiting in-between? Or what if we want to repeat a series of command again later? + +This is where _batch processing_ becomes useful, this is where instead of entering commands directly to the terminal we write them down in a text file or _script_. Then, the script can be _executed_ by calling it with `bash`. + +[//]: # TODO ??Diagram?? + +Lets try this now, create and open a new file in your current directory called `example-job.sh`. +(If you prefer another text editor than nano, feel free to use that), we will put to use some things we have learnt so far. + +``` +{{ site.remote.prompt }} nano example-job.sh +``` +{: .language-bash} + + +``` +{% include example_scripts/example-job.sh %} +``` +{: .language-bash} + +> ## shebang +> +> _shebang_ or _shabang_, also referred to as _hashbang_ is the character sequence consisting of the number sign (aka: hash) and exclamation mark (aka: bang): `#!` at the beginning of a script. It is used to describe the _interpreter_ that will be used to run the script. In this case we will be using the Bash Shell, which can be found at the path `/bin/bash`. The job scheduler will give you an error if your script does not start with a shebang. +> +{: .callout} + +We can now run this script using +``` +{{ site.remote.prompt }} bash example-job.sh +``` +{: .language-bash} + +``` +Loading required package: foreach +Loading required package: iterators +Loading required package: parallel +[1] "Using 1 cpus to sum [ 2.000000e+04 x 2.000000e+04 ] matrix." +[1] "0% done..." +... +[1] "99% done..." +[1] "100% done..." +[1] "Sum is '10403.632886'." +Done! +``` +{: .output} + +You will get the output printed to your terminal as if you had just run those commands one after another. + +> ## Cancelling Commands +> +> You can kill a currently running task by pressing the keys ctrl + c. +> If you just want your terminal back, but want the task to continue running you can 'background' it by pressing ctrl + v. +> Note, a backgrounded task is still attached to your terminal session, and will be killed when you close the terminal (if you need to keep running a task after you log out, have a look at [tmux](https://support.nesi.org.nz/hc/en-gb/articles/4563511601679-tmux-Reference-sheet)). +{: .callout} + +## Scheduled Batch Job + +Up until now the scheduler has not been involved, our scripts were run directly on the login node (or Jupyter node). + +First lets rename our batch script script to clarify that we intend to run it though the scheduler. + +``` +mv example-job.sh example-job.sl +``` +{: .output} + +> ## File Extensions +> +> A files extension in this case does not in any way affect how a script is read, +> it is just another part of the name used to remind users what type of file it is. +> Some common conventions: +> `.sh`: **Sh**ell Script. +> `.sl`: **Sl**urm Script, a script that includes a *slurm header* and is intended to be submitted to the cluster. +> `.out`: Commonly used to indicate the file contains the std**out** of some process. +> `.err`: Same as `.out` but for std**err**. +{: .callout} + +In order for the job scheduler to do it's job we need to provide a bit more information about our script. +This is done by specifying _slurm parameters_ in our batch script. Each of these parameters must be preceded by the special token `#SBATCH` and placed _after_ the _shebang_, but before the content of the rest of your script. + +{% include figure.html max-width="100%" caption="" + file="/fig/parts_slurm_script.svg" + alt="slurm script is a regular bash script with a slurm header after the shebang" %} + +These parameters tell SLURM things around how the script should be run, like memory, cores and time required. + +All the parameters available can be found by checking `man sbatch` or on the online [slurm documentation](https://slurm.schedmd.com/sbatch.html). + +[//]: # TODO ??Vet table + +{% include {{ site.snippets }}/scheduler/option-flags-list.snip %} +> ## Comments +> +> Comments in UNIX shell scripts (denoted by `#`) are ignored by the bash interpreter. +> Why is it that we start our slurm parameters with `#` if it is going to be ignored? +> > ## Solution +> > Commented lines are ignored by the bash interpreter, but they are _not_ ignored by slurm. +> > The `{{ site.sched.comment }}` parameters are read by slurm when we _submit_ the job. When the job starts, +> > the bash interpreter will ignore all lines starting with `#`. +> > +> > This is similar to the _shebang_ mentioned earlier, +> > when you run your script, the system looks at the `#!`, then uses the program at the subsequent +> > path to interpret the script, in our case `/bin/bash` (the program 'bash' found in the 'bin' directory). +> {: .solution} +{: .challenge} + +Note that just *requesting* these resources does not make your job run faster, +nor does it necessarily mean that you will consume all of these resources. It +only means that these are made available to you. Your job may end up using less +memory, or less time, or fewer tasks or nodes, than you have requested, and it +will still run. + +It's best if your requests accurately reflect your job's requirements. We'll +talk more about how to make sure that you're using resources effectively in a +later episode of this lesson. + +Now, rather than running our script with `bash` we _submit_ it to the scheduler using the command `sbatch` (**s**lurm **batch**). + +``` +{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sl +``` +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/basic-job-script.snip %} + +And that's all we need to do to submit a job. Our work is done -- now the +scheduler takes over and tries to run the job for us. + +## Checking on Running/Pending Jobs + +While the job is waiting +to run, it goes into a list of jobs called the *queue*. To check on our job's +status, we check the queue using the command +`{{ site.sched.status }}` (**s**lurm **queue**). We will need to filter to see only our jobs, by including either the flag `--user ` or `--me`. + +``` +{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.me }} +``` +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/basic-job-status.snip %} + +We can see many details about our job, most importantly is it's _STATE_, the most common states you might see are.. + +- `PENDING`: The job is waiting in the queue, likely waiting for resources to free up or higher prioroty jobs to run. +because other jobs have priority. +- `RUNNING`: The job has been sent to a compute node and it is processing our commands. +- `COMPLETED`: Your commands completed successfully as far as Slurm can tell (e.g. exit 0). +- `FAILED`: (e.g. exit not 0). +- `CANCELLED`: +- `TIMEOUT`: Your job has running for longer than your `--time` and was killed. +- `OUT_OF_MEMORY`: Your job tried to use more memory that it is allocated (`--mem`) and was killed. + +## Cancelling Jobs + +Sometimes we'll make a mistake and need to cancel a job. This can be done with +the `{{ site.sched.del }}` command. + + + + + +In order to cancel the job, we will first need its 'JobId', this can be found in the output of '{{ site.sched.status }} {{ site.sched.flag.me }}'. + +``` +{{ site.remote.prompt }} {{site.sched.del }} 231964 +``` +{: .language-bash} + +A clean return of your command prompt indicates that the request to cancel the job was +successful. + +Now checking `{{ site.sched.status }}` again, the job should be gone. + +``` +{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.me }} +``` +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/terminate-job-cancel.snip %} + +(If it isn't wait a few seconds and try again). + +{% include {{ site.snippets }}/scheduler/terminate-multiple-jobs.snip %} + +## Checking Finished Jobs + +There is another command `{{ site.sched.hist }}` (**s**lurm **acc**oun**t**) that includes jobs that have finished. +By default `{{ site.sched.hist }}` only includes jobs submitted by you, so no need to include additional commands at this point. + +``` +{{ site.remote.prompt }} {{ site.sched.hist }} +``` +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/basic-job-status-sacct.snip %} + +Note that despite the fact that we have only run one job, there are three lines shown, this because each _job step_ is also shown. +This can be suppressed using the flag `-X`. + +> ## Where's the Output? +> +> On the login node, when we ran the bash script, the output was printed to the terminal. +> Slurm batch job output is typically redirected to a file, by default this will be a file named `slurm-.out` in the directory where the job was submitted, this can be changed with the slurm parameter `--output`. +{: .discussion} +> +> > ## Hint +> > +> > You can use the _manual pages_ for {{ site.sched.name }} utilities to find +> > more about their capabilities. On the command line, these are accessed +> > through the `man` utility: run `man `. You can find the same +> > information online by searching > "man ". +> > +> > ``` +> > {{ site.remote.prompt }} man {{ site.sched.submit.name }} +> > ``` +> > {: .language-bash} +> {: .solution} +{: .challenge} + +{% include {{ site.snippets }}/scheduler/print-sched-variables.snip %} + +[//]: # TODO ??Sacct more info on checking jobs. Checking log files during run. + + + + +{% include links.md %} + +[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard +[hisat]: https://ccb.jhu.edu/software/hisat2/index.shtml diff --git a/_episodes/06-lunch.md b/_episodes/06-lunch.md new file mode 100644 index 00000000..db16afe4 --- /dev/null +++ b/_episodes/06-lunch.md @@ -0,0 +1,5 @@ +--- +title: "Lunch" +layout: break +break: 45 +--- \ No newline at end of file diff --git a/_episodes/064-parallel.md b/_episodes/064-parallel.md new file mode 100644 index 00000000..67189ca4 --- /dev/null +++ b/_episodes/064-parallel.md @@ -0,0 +1,360 @@ +--- +title: "What is Parallel Computing" +teaching: 15 +exercises: 5 +questions: +- "How do we execute a task in parallel?" +- "What benefits arise from parallel execution?" +- "What are the limits of gains from execution in parallel?" +- "What is the difference between implicit and explicit parallelisation." +objectives: +- "Prepare a job submission script for the parallel executable." +keypoints: +- "Parallel programming allows applications to take advantage of + parallel hardware; serial code will not 'just work.'" +- "There are multiple ways you can run " +--- + +## Methods of Parallel Computing + +To understand the different types of Parallel Computing we first need to clarify some terms. + +{% include figure.html url="" max-width="40%" + file="/fig/clusterDiagram.png" + alt="Node anatomy" caption="" %} + +**CPU**: Unit that does the computations. + +**Task**: One or more CPUs that share memory. + +**Node**: The physical hardware. The upper limit on how many CPUs can be in a task. + +**Shared Memory**: When multiple CPUs are used within a single task. + +**Distributed Memory**: When multiple tasks are used. + +Which methods are available to you is largely dependent on the nature of the problem and software being used. + + + +### Shared-Memory (SMP) + +Shared-memory multiproccessing divides work among _CPUs_ or _threads_, all of these threads require access to the same memory. + +Often called *Multithreading*. + +This means that all CPUs must be on the same node, most Mahuika nodes have 72 CPUs. + +Shared memory parallelism is what is used in our example script `{{ site.example.script }}`. + +Number of threads to use is specified by the Slurm option `--cpus-per-task`. + +{% capture example_smp %} +> {% include example_scripts/example_smp.sl %} +{% endcapture %} + +> ## Shared Memory Example +> +> Create a new script called `example_smp.sl` +> +> ``` +> #!/bin/bash -e +> +> #SBATCH --job-name smp_job +> #SBATCH --output %x.out +> #SBATCH --mem-per-cpu 500 +> #SBATCH --cpus-per-task 8 +> +> echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" +> ``` +> {: .language-bash} +> +> then submit with +> +> ``` +> {{ site.remote.prompt }} sbatch example_smp.sl +> ``` +> {: .language-bash} +> +> > ## Solution +> > +> > Checking the output should reveal +> > +> > ``` +> > {{ site.remote.prompt }} cat smp_job.out +> > ``` +> > {: .language-bash} +> > +> > ``` +> > I am task #0 running on node 'wbn224' with 8 CPUs +> > ``` +> > +> > {: .output} +> {: .solution} +{: .challenge} + +### Distributed-Memory (MPI) + +Distributed-memory multiproccessing divides work among _tasks_, a task may contain multiple CPUs (provided they all share memory, as discussed previously). + +Message Passing Interface (MPI) is a communication standard for distributed-memory multiproccessing. While there are other standards, often 'MPI' is used synonymously with Distributed parallelism. + +Each task has it's own exclusive memory, tasks can be spread across multiple nodes, communicating via and _interconnect_. This allows MPI jobs to be much larger than shared memory jobs. It also means that memory requirements are more likely to increase proportionally with CPUs. + +Distributed-Memory multiproccessing predates shared-memory multiproccessing, and is more common with classical high performance applications (older computers had one CPU per node). + +Number of tasks to use is specified by the Slurm option `--ntasks`, because the number of tasks ending up on one node is variable you should use `--mem-per-cpu` rather than `--mem` to ensure each task has enough. + +Tasks cannot share cores, this means in most circumstances leaving `--cpus-per-task` unspecified will get you `2`. + +> ## Distributed Memory Example +> +> Create a new script called `example_mpi.sl` +> +> ``` +> #!/bin/bash -e +> +> #SBATCH --job-name mpi_job +> #SBATCH --output %x.out +> #SBATCH --mem-per-cpu 500 +> #SBATCH --ntasks 4 +> +> srun echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" +> ``` +> {: .language-bash} +> +> then submit with +> +> ``` +> {{ site.remote.prompt }} sbatch example_mpi.sl +> ``` +> {: .language-bash} +> +> > ## Solution +> > +> > ``` +> > {{ site.remote.prompt }} cat mpi_job.out +> > ``` +> > {: .language-bash} +> > +> >``` +> > I am task #1 running on node 'wbn012' with 2 CPUs +> > I am task #3 running on node 'wbn010' with 2 CPUs +> > I am task #0 running on node 'wbn009' with 2 CPUs +> > I am task #2 running on node 'wbn063' with 2 CPUs +> > ``` +> > {: .output} +> {: .solution} +{: .challenge} + +Using a combination of Shared and Distributed memory is called _Hybrid Parallel_. + +> ## Hybrid Example +> +> ``` +> #!/bin/bash -e +> +> #SBATCH --job-name hybrid_job +> #SBATCH --output %x.out +> #SBATCH --mem-per-cpu 500 +> #SBATCH --ntasks 2 +> #SBATCH --cpus-per-task 4 +> +> srun echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" +> ``` +> {: .language-bash} +> +> ``` +> {{ site.remote.prompt }} sbatch example_hybrid.sl +> ``` +> {: .language-bash} +> +> > ## Solution +> > +> > ``` +> > {{ site.remote.prompt }} cat hybrid_job.out +> > +> > ``` +> > +> > ``` +> > I am task #0 running on node 'wbn016' with 4 CPUs +> > I am task #1 running on node 'wbn022' with 4 CPUs +> > ``` +> > {: .output} +> {: .solution} +{: .challenge} + +### GPGPU's + +GPUs compute large number of simple operation in parallel, making them well suited for Graphics Processing (hence the name), or any other large matrix operations. + + + +On NeSI, GPU's are specialised pieces of hardware that you request in addition to your CPUs and memory. + +You can find an up-to-date(ish) list of GPUs available on NeSI in our [Support Documentation](https://support.nesi.org.nz/hc/en-gb/articles/4963040656783-Available-GPUs-on-NeSI) + +GPUs can be requested using `--gpus-per-node=:` + +Depending on the GPU type, we *may* also need to specify a partition using `--partition`. + +> ## GPU Job Example +> +> Create a new script called `example_gpu.sl` +> +> ``` +> #!/bin/bash -e +> +> #SBATCH --job-name gpu +> #SBATCH --account {{site.sched.projectcode}} +> #SBATCH --output %x_%a.out +> #SBATCH --mem-per-cpu 2G +> #SBATCH --gpu-per-node P100:1 +> +> module load CUDA +> nvidia-smi +> ``` +> {: .language-bash} +> +> then submit with +> +> ``` +> {{ site.remote.prompt }} sbatch example_gpu.sl +> ``` +> {: .language-bash} +> +> > ## Solution +> > +> > ``` +> > {{ site.remote.prompt }} cat gpu_job.out +> > +> > ``` +> > {: .language-bash} +> > +> > ``` +> > Tue Mar 12 19:40:51 2024 +> > +-----------------------------------------------------------------------------+ +> > | NVIDIA-SMI 525.85.12 Driver Version: 525.85.12 CUDA Version: 12.0 | +> > |-------------------------------+----------------------+----------------------+ +> > | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +> > | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +> > | | | MIG M. | +> > |===============================+======================+======================| +> > | 0 Tesla P100-PCIE... On | 00000000:05:00.0 Off | 0 | +> > | N/A 28C P0 24W / 250W | 0MiB / 12288MiB | 0% Default | +> > | | | N/A | +> > +-------------------------------+----------------------+----------------------+ +> > +> > +-----------------------------------------------------------------------------+ +> > | Processes: | +> > | GPU GI CI PID Type Process name GPU Memory | +> > | ID ID Usage | +> > |=============================================================================| +> > | No running processes found | +> > +-----------------------------------------------------------------------------+ +> > ``` +> > {: .output} +> {: .solution} +{: .challenge} + + +### Job Array + +Job arrays are not "multiproccessing" in the same way as the previous two methods. +Ideal for _embarrassingly parallel_ problems, where there are little to no dependencies between the different jobs. + +Can be thought of less as running a single job in parallel and more about running multiple serial-jobs simultaneously. +Often this will involve running the same process is run on multiple inputs. + +Embarrassingly parallel jobs should be able scale without any loss of efficiency. If this type of parallelisation is an option, it will almost certainly be the best choice. + +A job array can be specified using `--array` + +If you are writing your own code, then this is something you will probably have to specify yourself. + +> ## Job Array Example +> +> Create a new script called `example_jobarray.sl` +> +> ``` +> #!/bin/bash -e +> +> #SBATCH --job-name job_array +> #SBATCH --output %x_%a.out +> #SBATCH --mem-per-cpu 500 +> #SBATCH --array 0-3 +> +> echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" +> ``` +> {: .language-bash} +> +> then submit with +> +> ``` +> {{ site.remote.prompt }} sbatch example_jobarray.sl +> ``` +> {: .language-bash} +> +> > ## Solution +> > +> > ``` +> > ls +> > ``` +> > {: .language-bash} +> > +> > ``` +> > job_array_0.out job_array_1.out job_array_2.out job_array_3.out +> > ``` +> > {: .output} +> > +> > Each of which should contain, +> > +> > +> > ``` +> > {{ site.remote.prompt }} cat job_array_*.out +> > ``` +> > {: .language-bash} +> > +> > ``` +> > I am task #0 running on node 'wbn*' with 2 CPUs +> > ``` +> > {: .output} +> {: .solution} +{: .challenge} + +## How to Utilise Multiple CPUs + +Requesting extra resources through Slurm only means that more resources will be available, it does not guarantee your program will be able to make use of them. + +Generally speaking, Parallelism is either _implicit_ where the software figures out everything behind the scenes, or _explicit_ where the software requires extra direction from the user. + +### Scientific Software + +The first step when looking to run particular software should always be to read the documentation. +On one end of the scale, some software may claim to make use of multiple cores implicitly, but this should be verified as the methods used to determine available resources are not guaranteed to work. + +Some software will require you to specify number of cores (e.g. `-n 8` or `-np 16`), or even type of paralellisation (e.g. `-dis` or `-mpi=intelmpi`). + +Occasionally your input files may require rewriting/regenerating for every new CPU combintation (e.g. domain based parallelism without automatic partitioning). + +### Writing Code + +Occasionally requesting more CPUs in your Slurm job is all that is required and whatever program you are running will automagically take advantage of the additional resources. +However, it's more likely to require some amount of effort on your behalf. + +It is important to determine this before you start requesting more resources through Slurm + +If you are writing your own code, some programming languages will have functions that can make use of multiple CPUs without requiring you to changes your code. +However, unless that function is where the majority of time is spent, this is unlikely to give you the performance you are looking for. + + +*Python: [Multiproccessing](https://docs.python.org/3/library/multiprocessing.html)* (not to be confused with `threading` which is not really parallel.) + +*MATLAB: [Parpool](https://au.mathworks.com/help/parallel-computing/parpool.html)* + +{% include links.md %} diff --git a/_episodes/07-resources.md b/_episodes/07-resources.md new file mode 100644 index 00000000..087f562a --- /dev/null +++ b/_episodes/07-resources.md @@ -0,0 +1,365 @@ +--- +title: "Using resources effectively" +teaching: 25 +exercises: 10 +questions: +- "How can I review past jobs?" +- "How can I use this knowledge to create a more accurate submission script?" +objectives: +- "Understand how to look up job statistics and profile code." +- "Understand job size implications." +- "Understand problems and limitations involved in using multiple CPUs." +keypoints: +- "As your task gets larger, so does the potential for inefficiencies." +- "The smaller your job (time, CPUs, memory, etc), the faster it will schedule." +--- + + +## What Resources? + +Last time we submitted a job, we did not specify a number of CPUs, and therefore +we were provided the default of `2` (1 _core_). + +As a reminder, our slurm script `example-job.sl` currently looks like this. + +``` +{% include example_scripts/example-job.sl.1 %} +``` + +{: .language-bash} + +We will now submit the same job again with more CPUs. +We ask for more CPUs using by adding `#SBATCH --cpus-per-task 4` to our script. +Your script should now look like this: + +``` +{% include example_scripts/example-job.sl.2 %} +``` + +{: .language-bash} + +And then submit using `sbatch` as we did before. + +``` +{{ site.remote.prompt }} sbatch example-job.sl +``` + +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/basic-job-script.snip %} + +> ## Watch +> +> We can prepend any command with `watch` in order to periodically (default 2 seconds) run a command. e.g. `watch +> squeue --me` will give us up to date information on our running jobs. +> Care should be used when using `watch` as repeatedly running a command can have adverse effects. +> Exit `watch` with ctrl + c. +{: .callout} + +Note in squeue, the number under cpus, should be '4'. + +Checking on our job with `sacct`. +Oh no! + +{% include {{ site.snippets }}/scaling/OOM.snip %} +{: .language-bash} + +To understand why our job failed, we need to talk about the resources involved. + +Understanding the resources you have available and how to use them most efficiently is a vital skill in high performance computing. + +Below is a table of common resources and issues you may face if you do not request the correct amount. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Not enoughToo Much
CPU The job will run more slowly than expected, and so may run out of time and get killed for exceeding its time limit.The job will wait in the queue for longer.
+ You will be charged for CPUs regardless of whether they are used or not.
+ Your fair share score will fall more. +
Memory Your job will fail, probably with an 'OUT OF MEMORY' error, segmentation fault or bus error (may not happen immediately).The job will wait in the queue for longer.
+ You will be charged for memory regardless of whether it is used or not.
+ Your fair share score will fall more.
Walltime The job will run out of time and be terminated by the scheduler.The job will wait in the queue for longer.
+ +## Measuring Resource Usage of a Finished Job + +Since we have already run a job (successful or otherwise), this is the best source of info we currently have. +If we check the status of our finished job using the `sacct` command we learned earlier. + +``` +{{ site.remote.prompt }} sacct +``` + +{: .language-bash} + +{% include {{ site.snippets }}/scheduler/basic-job-status-sacct.snip %} + +With this information, we may determine a couple of things. + +Memory efficiency can be determined by comparing ReqMem (requested memory) with MaxRSS (maximum used memory), MaxRSS is given in KB, so a unit conversion is usually required. + +{% include figure.html url="" max-width="75%" caption="" + file="/fig/mem_eff.svg" + alt="Memory Efficiency Formula" %} + +So for the above example we see that 0.1GB (102048K) of our requested 1GB meaning the memory efficincy was about 10%. + +CPU efficiency can be determined by comparing TotalCPU(CPU time), with the maximum possible CPU time. The maximum possible CPU time equal to Alloc (number of allocated CPUs) multiplied by Elapsed (Walltime, actual time passed). + +{% include figure.html url="" max-width="75%" caption="" + file="/fig/cpu_eff.svg" + alt="CPU Efficiency Formula" %} + +For the above example 33 seconds of computation was done, + +where the maximum possible computation time was **96 seconds** (2 CPUs multiplied by 48 seconds), meaning the CPU efficiency was about 35%. + +Time Efficiency is simply the Elapsed Time divided by Time Requested. + +{% include figure.html url="" max-width="75%" caption="" + file="/fig/time_eff.svg" + alt="Time Efficiency Formula" %} + +48 seconcds out of 15 minutes requested give a time efficiency of about 5% + +> ## Efficiency Exercise +> +> Calculate for the job shown below, +> +> ``` +> JobID JobName Alloc Elapsed TotalCPU ReqMem MaxRSS State +> --------------- ---------------- ----- ----------- ------------ ------- -------- ---------- +> 37171050 Example-job 8 00:06:03 00:23:04 32G FAILED +> 37171050.batch batch 8 00:06:03 23:03.999 14082672k FAILED +> 37171050.extern extern 8 00:06:03 00:00.001 0 COMPLETED +> ``` +> +> a. CPU efficiency. +> +> b. Memory efficiency. +> +> > ## Solution +> > +> > a. CPU efficiency is `( 23 / ( 8 * 6 ) ) x 100` or around **48%**. +> > +> > b. Memory efficiency is `( 14 / 32 ) x 100` or around **43%**. +> {: .solution} +{: .challenge} + +For convenience, NeSI has provided the command `nn_seff ` to calculate **S**lurm **Eff**iciency (all NeSI commands start with `nn_`, for **N**eSI **N**IWA). + +``` +{{ site.remote.prompt }} nn_seff +``` + +{: .language-bash} + +{% include {{ site.snippets }}/resources/seff.snip %} + +Knowing what we do now about job efficiency, lets submit the previous job again but with more appropriate resources. + +``` +{% include example_scripts/example-job.sl.2 %} +``` +{: .language-bash} + + +``` +{{ site.remote.prompt }} sbatch example-job.sl +``` +{: .language-bash} + +Hopefully we will have better luck with this one! + +### A quick description of Simultaneous Multithreading - SMT (aka Hyperthreading) + +Modern CPU cores have 2 threads of operation that can execute independently of one +another. SMT is the technology that allows the 2 threads within one physical core to present +as multiple logical cores, sometimes referred to as virtual CPUS (vCPUS). + +Note: _Hyperthreading_ is Intel's marketing name for SMT. Both Intel and AMD +CPUs have SMT technology. + +Some types of processes can take advantage of multiple threads, and can gain a +performance boost. Some software is +specifically written as multi-threaded. You will need to check or test if your +code can take advantage of threads (we can help with this). + +However, because each thread shares resources on the physical core, +there can be conflicts for resources such as onboard cache. +This is why not all processes get a performance boost from SMT and in fact can +run slower. These types of jobs should be run without multithreading. There +is a Slurm parameter for this: `--hint=nomultithread` + +SMT is why you are provided 2 CPUs instead of 1 as we do not allow +2 different jobs to share a core. This also explains why you will sometimes +see CPU efficiency above 100%, since CPU efficiency is based on core and not thread. + +For more details please see our documentation: + + +## Measuring the System Load From Currently Running Tasks + +On Mahuika, we allow users to connect directly to compute nodes from the +login node. This is useful to check on a running job and see how it's doing, however, we +only allow you to connect to nodes on which you have running jobs. + +The most reliable way to check current system stats is with `htop`. +`htop` is an interactive process viewer that can be launched from command line. + +### Finding job node + +Before we can check on our job, we need to find out where it is running. +We can do this with the command `squeue --me`, and looking under the 'NODELIST' column. + +``` +{{ site.remote.prompt }} squeue --me +``` + +{: .language-bash} + +{% include {{ site.snippets }}/resources/get-job-node.snip %} + +Now that we know the location of the job (wbn189) we can use `ssh` to run `htop` _on that node_. + +``` +{{ site.remote.prompt }} ssh wbn189 -t htop -u $USER +``` + +{: .language-bash} + +You may get a message: + +``` +ECDSA key fingerprint is SHA256:############################################ +ECDSA key fingerprint is MD5:9d:############################################ +Are you sure you want to continue connecting (yes/no)? +``` + +{: .language-bash} + +If so, type `yes` and Enter + +You may also need to enter your cluster password. + +If you cannot connect, it may be that the job has finished and you have lost permission to `ssh` to that node. + +### Reading Htop + +You may see something like this, + +{% include {{ site.snippets }}/resources/monitor-processes-top.snip %} + +Overview of the most important fields: + +* `PID`: What is the numerical id of each process? +* `USER`: Who started the process? +* `RES`: What is the amount of memory currently being used by a process (in + bytes)? +* `%CPU`: How much of a CPU is each process using? Values higher than 100 + percent indicate that a process is running in parallel. +* `%MEM`: What percent of system memory is a process using? +* `TIME+`: How much CPU time has a process used so far? Processes using 2 CPUs + accumulate time at twice the normal rate. +* `COMMAND`: What command was used to launch a process? + +To exit press q. + +Running this command as is will show us information on tasks running on the login node (where we should not be running resource intensive jobs anyway). + +## Running Test Jobs + +As you may have to run several iterations before you get it right, you should choose your test job carefully. +A test job should not run for more than 15 mins. This could involve using a smaller input, coarser parameters or using a subset of the calculations. +As well as being quick to run, you want your test job to be quick to start (e.g. get through queue quickly), the best way to ensure this is keep the resources requested (memory, CPUs, time) small. +Similar as possible to actual jobs e.g. same functions etc. +Use same workflow. (most issues are caused by small issues, typos, missing files etc, your test job is a jood chance to sort out these issues.). +Make sure outputs are going somewhere you can see them. + +> ## Serial Test +> +> Often a good first test to run, is to execute your job _serially_ e.g. using only 1 CPU. +> This not only saves you time by being fast to start, but serial jobs can often be easier to debug. +> If you confirm your job works in its most simple state you can identify problems caused by +> paralellistaion much more easily. +{: .callout} + +You generally should ask for 20% to 30% more time and memory than you think the job will use. +Testing allows you to become more more precise with your resource requests. We will cover a bit more on running tests in the last lesson. + +> ## Efficient way to run tests jobs using debug QOS (Quality of Service) +> +> Before submitting a large job, first submit one as a test to make +> sure everything works as expected. Often, users discover typos in their submit +> scripts, incorrect module names or possibly an incorrect pathname after their job +> has queued for many hours. Be aware that your job is not fully scanned for +> correctness when you submit the job. While you may get an immediate error if your +> SBATCH directives are malformed, it is not until the job starts to run that the +> interpreter starts to process the batch script. +> +> NeSI has an easy way for you to test your job submission. One can employ the debug +> QOS to get a short, high priority test job. Debug jobs have to run within 15 +> minutes and cannot use more that 2 nodes. To use debug QOS, add or change the +> following in your batch submit script +> +>``` +>#SBATCH --qos=debug +>#SBATCH --time=15:00 +> ``` +> +>{: .language-bash} +> +> Adding these SBATCH directives will provide your job with the highest priority +> possible, meaning it should start to run within a few minutes, provided +> your resource request is not too large. +{: .callout} + +## Initial Resource Requirements + +As we have just discussed, the best and most reliable method of determining resource requirements is from testing, +but before we run our first test there are a couple of things you can do to start yourself off in the right area. + +### Read the Documentation + +NeSI maintains documentation that does have some guidance on using resources for some software +However, as you noticed in the Modules lessons, we have a lot of software. So it is also advised to search +the web for others that may have written up guidance for getting the most out of your specific software. + +### Ask Other Users + +If you know someone who has used the software before, they may be able to give you a ballpark figure. + + + +> ## Next Steps +> +> You can use this knowledge to set up the +> next job with a closer estimate of its load on the system. +> A good general rule +> is to ask the scheduler for **30%** more time and memory than you expect the +> job to need. +{: .callout} + +{% include links.md %} diff --git a/_episodes/08-break2.md b/_episodes/08-break2.md new file mode 100644 index 00000000..5620aa08 --- /dev/null +++ b/_episodes/08-break2.md @@ -0,0 +1,5 @@ +--- +title: "Afternoon Break" +layout: break +break: 10 +--- \ No newline at end of file diff --git a/_episodes/09-scaling.md b/_episodes/09-scaling.md new file mode 100644 index 00000000..ee87ad0f --- /dev/null +++ b/_episodes/09-scaling.md @@ -0,0 +1,60 @@ +--- +title: "Scaling" +teaching: 5 +exercises: 30 +questions: +- "How do we go from running a job on a small number of CPUs to a larger one." +objectives: +- "Understand scaling procedure." +keypoints: +- Start small. +- Test one thing at a time (unit tests). +- Record everything. +--- + +The aim of these tests will be to establish how a jobs requirements change with size (CPUs, inputs) and ultimately figure out the best way to run your jobs. +Unfortunately we cannot assume speedup will be linear (e.g. double CPUs won't usually half runtime, doubling the size of your input data won't necessarily double runtime) therefore more testing is required. This is called *scaling testing*. + +In order to establish an understanding of the scaling properties we may have to repeat this test several times, giving more resources each iteration. + +## Scaling Behavior + +### Amdahl's Law + +Most computational tasks will have a certain amount of work that must be computed serially. + +![Larger fractions of parallel code will have closer to linear scaling performance.](../fig/AmdahlsLaw2.svg) + +Eventually your performance gains will plateau. + +The fraction of the task that can be run in parallel determines the point of this plateau. +Code that has no serial components is said to be "embarrassingly parallel". + +It is worth noting that Amdahl's law assumes all other elements of scaling are happening with 100% efficient, in reality there are additional computational and communication overheads. + +> ## Scaling Exercise +> +> 1. Find your name in the [spreadsheet]({{ site.exercise }}) and modify your `example-job.sl` to request +> "x" `--cpus-per-task`. +> For example `#SBATCH --cpus-per-task 10`. +> 2. Estimate memory requirement based on our previous runs and the cpus requested, memory +> is specified with the `--mem ` flag, it does not accept decimal values, however you may +> specify a unit (`K`|`M`|`G`), if no unit is specified it is assumed to be `M`. +> For example `#SBATCH --mem 1200`. +> 3. Now submit your job, we will include an extra argument `--acctg-freq 1`. +> By default SLURM records job data every 30 seconds. +> This means any job running for less than 30 +> seconds will not have it's memory use recorded. +> Submit the job with `sbatch --acctg-freq 1 example-job.sl`. +> 4. Watch the job with `squeue --me` or `watch squeue --me`. +> 5. On completion of job, use `nn_seff `. +> 6. Record the jobs "Elapsed", "TotalCPU", and "Memory" values in the spreadsheet. (Hint: They are the first +> numbers after the percentage efficiency in output of `nn_seff`). Make sure you have entered the values in the correct format and there is a tick next to each entry. ![Correctly entered data in spreadsheet.](../fig/correct-spreadsheet-entry.png) +> +> > ## Solution +> > +> > [spreadsheet]({{ site.exercise }}) +> {: .solution} +{: .challenge} + +{% include links.md %} diff --git a/_episodes/095-writing-good-code.md b/_episodes/095-writing-good-code.md new file mode 100644 index 00000000..e7e8d1ae --- /dev/null +++ b/_episodes/095-writing-good-code.md @@ -0,0 +1,269 @@ +--- +title: "Writing good code" +teaching: 20 +exercises: 10 +questions: +- "How do we write a good job script." +objectives: +- "Write a script that can be run serial or parallel." +- "Write a script that using SLURM environment variables." +- "Understand the limitations of random number generation." +keypoints: +- "Write your script in a way that is independent of data or environment. (elaborate)" +--- + +When talking about 'a script' we could be referring to multiple things. + +* Slurm/Bash script - Almost everyone will be using one of these to submit their Slurm jobs. +* Work script - If your work involves running another script (usually in a language other than Bash like Python, R or MATLAB) that will have to be invoked in your bash script. + +This section will cover best practice for both types of script. + + + +## Use environment variables + +In this lesson we will take a look at a few of the things to watch out for when writing scripts for use on the cluster. +This will be most relevant to people writing their own code, but covers general practices applicable to everyone. + +There is a lot of useful information contained within environment variable. + +> ## Slurm Environment +> +> For a small demo of the sort of useful info contained within env variables, run the command. +> +> ``` +> sbatch --output "slurm_env.out" --wrap "env | grep" +> ``` +> {: .language-bash} +> +> once the job has finished check the results with, +> +> ``` +> cat slurm_env.out +> ``` +> {: .language-bash} +> +> ``` +> SLURM_JOB_START_TIME=1695513911 +> SLURM_NODELIST=wbn098 +> SLURM_JOB_NAME=wrap +> SLURMD_NODENAME=wbn098 +> SLURM_TOPOLOGY_ADDR=top.s13.s7.wbn098 +> SLURM_PRIO_PROCESS=0 +> SLURM_NODE_ALIASES=(null) +> SLURM_JOB_QOS=staff +> SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node +> SLURM_JOB_END_TIME=1695514811 +> SLURM_MEM_PER_CPU=512 +> SLURM_NNODES=1 +> SLURM_JOBID=39572365 +> SLURM_TASKS_PER_NODE=2 +> SLURM_WORKING_CLUSTER=mahuika:hpcwslurmctrl01:6817:9984:109 +> SLURM_CONF=/etc/opt/slurm/slurm.conf +> SLURM_JOB_ID=39572365 +> SLURM_JOB_USER=cwal219 +> __LMOD_STACK_SLURM_I_MPI_PMI_LIBRARY=L29wdC9zbHVybS9saWI2NC9saWJwbWkyLnNv +> SLURM_JOB_UID=201333 +> SLURM_NODEID=0 +> SLURM_SUBMIT_DIR=/scale_wlg_persistent/filesets/home/cwal219 +> SLURM_TASK_PID=8747 +> SLURM_CPUS_ON_NODE=2 +> SLURM_PROCID=0 +> SLURM_JOB_NODELIST=wbn098 +> SLURM_LOCALID=0 +> SLURM_JOB_GID=201333 +> SLURM_JOB_CPUS_PER_NODE=2 +> SLURM_CLUSTER_NAME=mahuika +> SLURM_GTIDS=0 +> SLURM_SUBMIT_HOST=wbn003 +> SLURM_JOB_PARTITION=large +> SLURM_JOB_ACCOUNT=nesi99999 +> SLURM_JOB_NUM_NODES=1 +> SLURM_SCRIPT_CONTEXT=prolog_task +> ``` +> {: .output} +> +> Can you think of some examples as to how these variables could be used in your script? + +> > ## Solution +> > +> > * `SLURM_JOB_CPUS_PER_NODE` could be used to pass CPU numbers directly to any programs being used. +> > * Some other things. +> {: .solution} +{: .challenge} + +> ## Variables in Slurm Header +> +> Environment variables set by Slurm cannot be referenced in the Slurm header. +{: .callout} + +## Default values + +It is good practice to set default values when using environment variables when there is a chance they will be run in an environment where they may not be present. + +``` +FOO="${VARIABLE:-default}" +``` +{: .language-bash} + +`FOO` will be to to the value of `VARIABLE` if is set, otherwise it will be set to `default`. + +As a slight variation on the above example. (`:=` as opposed to `:-`). + +``` +FOO="${VARIABLE:=default}" +``` +{: .language-bash} + +`FOO` will be to to the value of `VARIABLE` if is set, otherwise it will be set to `default`, `VARIABLE` will also be set to `default`. + + + + +``` +num_cpus <- 2 +``` +{: .language-r} + +The number of CPU's being used is fixed in the script. We can save time and reduce chances for making mistakes by replacing this static value with an environment variable. +We can use the environment variable `SLURM_CPUS_PER_TASK`. + +``` +num_cpus <- strtoi(Sys.getenv('SLURM_CPUS_PER_TASK')) +``` +{: .language-r} + +Slurm sets many environment variables when starting a job, see [Slurm Documentation for the full list](https://slurm.schedmd.com/sbatch.html). + +The problem with this approach however, is our code will throw an error if we run it on the login node, or on our local machine or anywhere else that `SLURM_CPUS_PER_TASK` is not set. + +Generally it is best not to diverge your codebase especially if you don't have it under version control, so lets add some compatibility for those use cases. + +``` +num_cpus <- strtoi(Sys.getenv('SLURM_CPUS_PER_TASK', unset = "1")) +``` +{: .language-r} + +Now if `SLURM_CPUS_PER_TASK` variable is not set, 1 CPU will be used. You could also use some other method of detecting CPUs, like `detectCores()`. + +## Interoperability + +windows + mac + linux +headless + interactive + +## Verbose + +Having a printout of job progress is fine for an interactive terminal, but when you aren't seeing the updates in real time anyway, it's just bloat for your output files. + +Let's add an option to mute the updates. + +``` +print_progress <- FALSE +``` +{: .language-r} + + +``` +if (print_progress && percent_complete%%1==0){ + +``` +{: .language-r} + +## Reproduceability + +As this script uses [Pseudorandom number generation](https://en.wikipedia.org/wiki/Pseudorandom_number_generator) there are a few additional factors to consider. +It is desirable that our output be reproducible so we can confirm that changes to the code have not affected it. + +We can do this by setting the seed of the PRNG. That way we will get the same progression of 'random' numbers. + +We are using the environment variable `SLURM_ARRAY_TASK_ID` for reasons we will get to later. We also need to make sure a default seed is set for the occasions when `SLURM_ARRAY_TASK_ID` is not set. + +``` +seed <- strtoi(Sys.getenv('SLURM_ARRAY_TASK_ID', unset = "0")) +set.seed(seed) +``` +{: .language-r} + + +Now your script should look something like this; + +``` +{% include example_scripts/array_sum2.r %} +``` +{: .language-r} + +## Readability + +Comments! + +## Debugging + +``` +#!/bin/bash -e +``` +{: .language-bash} + +Exit bash script on error + +``` +#!/bin/bash -x +``` +{: .language-bash} + +Print environment. + +``` +env +``` +{: .language-bash} + +Print environment, if someone else has problems replicating the problem, it will likely come down to differences in your environment. + +``` +cat $0 +``` +{: .language-bash} + +Will print your input Slurm script to you output, this can help identify when changes in your submission script leads to errors. + +## Version control + +Version control is when changes to a document are tracked over time. + +In many cases you may be using the same piece of code across multiple environments, in these situations it can be difficult to keep track of changes made and your code can begin to diverge. Setting up version control like Git can save a lot of time. + +### Portability + + + +## Testing + +More often than not, problems come in the form of typos, or other small errors that become apparent within the first few seconds/minutes of script. + +Running on login node? + +Control + c to kill. + +{% include links.md %} diff --git a/_episodes/10-hpc-intro.md b/_episodes/10-hpc-intro.md deleted file mode 100644 index 0d054cde..00000000 --- a/_episodes/10-hpc-intro.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: "Why use a Cluster?" -teaching: 15 -exercises: 5 -questions: -- "Why would I be interested in High Performance Computing (HPC)?" -- "What can I expect to learn from this course?" -objectives: -- "Describe what an HPC system is" -- "Identify how an HPC system could benefit you." -keypoints: -- "High Performance Computing (HPC) typically involves connecting to very large - computing systems elsewhere in the world." -- "These other systems can be used to do work that would either be impossible - or much slower on smaller systems." -- "HPC resources are shared by multiple users." -- "The standard method of interacting with such systems is via a command line - interface." ---- - -Frequently, research problems that use computing can outgrow the capabilities -of the desktop or laptop computer where they started: - -* A statistics student wants to cross-validate a model. This involves running - the model 1000 times -- but each run takes an hour. Running the model on - a laptop will take over a month! In this research problem, final results are - calculated after all 1000 models have run, but typically only one model is - run at a time (in __serial__) on the laptop. Since each of the 1000 runs is - independent of all others, and given enough computers, it's theoretically - possible to run them all at once (in __parallel__). -* A genomics researcher has been using small datasets of sequence data, but - soon will be receiving a new type of sequencing data that is 10 times as - large. It's already challenging to open the datasets on a computer -- - analyzing these larger datasets will probably crash it. In this research - problem, the calculations required might be impossible to parallelize, but a - computer with __more memory__ would be required to analyze the much larger - future data set. -* An engineer is using a fluid dynamics package that has an option to run in - parallel. So far, this option was not used on a desktop. In going from 2D - to 3D simulations, the simulation time has more than tripled. It might be - useful to take advantage of that option or feature. In this research problem, - the calculations in each region of the simulation are largely independent of - calculations in other regions of the simulation. It's possible to run each - region's calculations simultaneously (in __parallel__), communicate selected - results to adjacent regions as needed, and repeat the calculations to - converge on a final set of results. In moving from a 2D to a 3D model, __both - the amount of data and the amount of calculations increases greatly__, and - it's theoretically possible to distribute the calculations across multiple - computers communicating over a shared network. - -In all these cases, access to more (and larger) computers is needed. Those -computers should be usable at the same time, __solving many researchers' -problems in parallel__. - -## Jargon Busting Presentation - -Open the [HPC Jargon Buster]({{ site.url }}{{ site.baseurl }}/files/jargon.html#p1) -in a new tab. To present the content, press `C` to open a **c**lone in a -separate window, then press `P` to toggle **p**resentation mode. - -> ## I've Never Used a Server, Have I? -> -> Take a minute and think about which of your daily interactions with a -> computer may require a remote server or even cluster to provide you with -> results. -> -> > ## Some Ideas -> > -> > * Checking email: your computer (possibly in your pocket) contacts a remote -> > machine, authenticates, and downloads a list of new messages; it also -> > uploads changes to message status, such as whether you read, marked as -> > junk, or deleted the message. Since yours is not the only account, the -> > mail server is probably one of many in a data center. -> > * Searching for a phrase online involves comparing your search term against -> > a massive database of all known sites, looking for matches. This "query" -> > operation can be straightforward, but building that database is a -> > [monumental task][mapreduce]! Servers are involved at every step. -> > * Searching for directions on a mapping website involves connecting your -> > (A) starting and (B) end points by [traversing a graph][dijkstra] in -> > search of the "shortest" path by distance, time, expense, or another -> > metric. Converting a map into the right form is relatively simple, but -> > calculating all the possible routes between A and B is expensive. -> > -> > Checking email could be serial: your machine connects to one server and -> > exchanges data. Searching by querying the database for your search term (or -> > endpoints) could also be serial, in that one machine receives your query -> > and returns the result. However, assembling and storing the full database -> > is far beyond the capability of any one machine. Therefore, these functions -> > are served in parallel by a large, ["hyperscale"][hyperscale] collection of -> > servers working together. -> {: .solution} -{: .challenge } - -{% include links.md %} - -[dijkstra]: https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm -[hyperscale]: https://en.wikipedia.org/wiki/Hyperscale_computing -[mapreduce]: https://en.wikipedia.org/wiki/MapReduce diff --git a/_episodes/11-connecting.md b/_episodes/11-connecting.md deleted file mode 100644 index 15f6992b..00000000 --- a/_episodes/11-connecting.md +++ /dev/null @@ -1,469 +0,0 @@ ---- -title: "Connecting to a remote HPC system" -teaching: 25 -exercises: 10 -questions: -- "How do I log in to a remote HPC system?" -objectives: -- "Configure secure access to a remote HPC system." -- "Connect to a remote HPC system." -keypoints: -- "An HPC system is a set of networked machines." -- "HPC systems typically provide login nodes and a set of worker nodes." -- "The resources found on independent (worker) nodes can vary in volume and - type (amount of RAM, processor architecture, availability of network mounted - filesystems, etc.)." -- "Files saved on one node are available on all nodes." ---- - -## Secure Connections - -The first step in using a cluster is to establish a connection from our laptop -to the cluster. When we are sitting at a computer (or standing, or holding it -in our hands or on our wrists), we have come to expect a visual display with -icons, widgets, and perhaps some windows or applications: a _graphical user -interface_, or GUI. Since computer clusters are remote resources that we -connect to over slow or intermittent interfaces (WiFi and VPNs especially), it -is more practical to use a _command-line interface_, or CLI, to send commands -as plain-text. If a command returns output, it is printed as plain text as -well. The commands we run today will not open a window to show graphical -results. - -If you have ever opened the Windows Command Prompt or macOS Terminal, you have -seen a CLI. If you have already taken The Carpentries' courses on the UNIX -Shell or Version Control, you have used the CLI on your _local machine_ -extensively. The only leap to be made here is to open a CLI on a _remote machine_, -while taking some precautions so that other folks on the network can't see (or -change) the commands you're running or the results the remote machine sends -back. We will use the Secure SHell protocol (or SSH) to open an encrypted -network connection between two machines, allowing you to send & receive text -and data without having to worry about prying eyes. - -{% include figure.html url="" max-width="50%" - file="/fig/connect-to-remote.svg" - alt="Connect to cluster" caption="" %} - -SSH clients are usually command-line tools, where you provide the remote -machine address as the only required argument. If your username on the remote -system differs from what you use locally, you must provide that as well. If -your SSH client has a graphical front-end, such as PuTTY or MobaXterm, you will -set these arguments before clicking "connect." From the terminal, you'll write -something like `ssh userName@hostname`, where the argument is just like an -email address: the "@" symbol is used to separate the personal ID from the -address of the remote machine. - -When logging in to a laptop, tablet, or other personal device, a username, -password, or pattern are normally required to prevent unauthorized access. In -these situations, the likelihood of somebody else intercepting your password is -low, since logging your keystrokes requires a malicious exploit or physical -access. For systems like {{ site.remote.host }} running an SSH server, anybody -on the network can log in, or try to. Since usernames are often public or easy -to guess, your password is often the weakest link in the security chain. Many -clusters therefore forbid password-based login, requiring instead that you -generate and configure a public-private key pair with a much stronger password. -Even if your cluster does not require it, the next section will guide you -through the use of SSH keys and an SSH agent to both strengthen your security -_and_ make it more convenient to log in to remote systems. - -### Better Security With SSH Keys - -The [Lesson Setup]({{ page.root }}/setup) provides instructions for installing -a shell application with SSH. If you have not done so already, please open that -shell application with a Unix-like command line interface to your system. - -SSH keys are an alternative method for authentication to obtain access to -remote computing systems. They can also be used for authentication when -transferring files or for accessing remote version control systems (such as -[GitHub][gh-ssh]). -In this section you will create a pair of SSH keys: - -* a private key which you keep on your own computer, and -* a public key which can be placed on any remote system you will access. - -> ## Private keys are your secure digital passport -> -> A private key that is visible to anyone but you should be considered -> compromised, and must be destroyed. This includes having improper permissions -> on the directory it (or a copy) is stored in, traversing any network that is -> not secure (encrypted), attachment on unencrypted email, and even displaying -> the key on your terminal window. -> -> Protect this key as if it unlocks your front door. In many ways, it does. -{: .caution} - -Regardless of the software or operating system you use, _please_ choose a -strong password or passphrase to act as another layer of protection for your -private SSH key. - -> ## Considerations for SSH Key Passwords -> -> When prompted, enter a strong password that you will remember. There are two -> common approaches to this: -> -> 1. Create a memorable passphrase with some punctuation and number-for-letter -> substitutions, 32 characters or longer. Street addresses work well; just -> be careful of social engineering or public records attacks. -> 2. Use a password manager and its built-in password generator with all -> character classes, 25 characters or longer. [KeePass][keepass] and -> [BitWarden][bitwarden] are two good options. -> 3. Nothing is _less_ secure than a private key with no password. If you -> skipped password entry by accident, go back and generate a new key pair -> _with_ a strong password. -{: .callout} - -#### SSH Keys on Linux, Mac, MobaXterm, and Windows Subsystem for Linux - -Once you have opened a terminal, check for existing SSH keys and filenames -since existing SSH keys are overwritten. - -``` -{{ site.local.prompt }} ls ~/.ssh/ -``` -{: .language-bash} - -If `~/.ssh/id_ed25519` already exists, you will need to specify -a different name for the new key-pair. - -Generate a new public-private key pair using the following command, which will -produce a stronger key than the `ssh-keygen` default by invoking these flags: - -* `-a` (default is 16): number of rounds of passphrase derivation; increase to - slow down brute force attacks. -* `-t` (default is [rsa][wiki-rsa]): specify the "type" or cryptographic - algorithm. `ed25519` specifies [EdDSA][wiki-dsa] with a 256-bit key; - it is faster than RSA with a comparable strength. -* `-f` (default is /home/user/.ssh/id_algorithm): filename to store your - private key. The public key filename will be identical, with a `.pub` - extension added. - -``` -{{ site.local.prompt }} ssh-keygen -a 100 -f ~/.ssh/id_ed25519 -t ed25519 -``` -{: .language-bash} - -When prompted, enter a strong password with the -[above considerations in mind](#considerations-for-ssh-key-passwords). -Note that the terminal will not appear to change while you type the password: -this is deliberate, for your security. You will be prompted to type it again, -so don't worry too much about typos. - -Take a look in `~/.ssh` (use `ls ~/.ssh`). You should see two new files: - -* your private key (`~/.ssh/id_ed25519`): _do not share with anyone!_ -* the shareable public key (`~/.ssh/id_ed25519.pub`): if a system administrator - asks for a key, this is the one to send. It is also safe to upload to - websites such as GitHub: it is meant to be seen. - -> ## Use RSA for Older Systems -> -> If key generation failed because ed25519 is not available, try using the older -> (but still strong and trustworthy) [RSA][wiki-rsa] cryptosystem. Again, first -> check for an existing key: -> -> ``` -> {{ site.local.prompt }} ls ~/.ssh/ -> ``` -> {: .language-bash} -> -> If `~/.ssh/id_rsa` already exists, you will need to specify choose a different -> name for the new key-pair. Generate it as above, with the following extra flags: -> -> * `-b` sets the number of bits in the key. The default is 2048. -> EdDSA uses a fixed key length, so this flag would have no effect. -> * `-o` (no default): use the OpenSSH key format, -> rather than PEM. -> -> ``` -> {{ site.local.prompt }} ssh-keygen -a 100 -b 4096 -f ~/.ssh/id_rsa -o -t rsa -> ``` -> {: .language-bash} -> -> When prompted, enter a strong password with the -> [above considerations in mind](#considerations-for-ssh-key-passwords). -> -> Take a look in `~/.ssh` (use `ls ~/.ssh`). You should see two new files: -> -> * your private key (`~/.ssh/id_rsa`): _do not share with anyone!_ -> * the shareable public key (`~/.ssh/id_rsa.pub`): if a system administrator -> asks for a key, this is the one to send. It is also safe to upload to -> websites such as GitHub: it is meant to be seen. -{: .callout} - -#### SSH Keys on PuTTY - -If you are using PuTTY on Windows, download and use `puttygen` to generate the -key pair. See the [PuTTY documentation][putty-gen] for details. - -* Select `EdDSA` as the key type. -* Select `255` as the key size or strength. -* Click on the "Generate" button. -* You do not need to enter a comment. -* When prompted, enter a strong password with the - [above considerations in mind](#considerations-for-ssh-key-passwords). -* Save the keys in a folder no other users of the system can read. - -Take a look in the folder you specified. You should see two new files: - -* your private key (`id_ed25519`): _do not share with anyone!_ -* the shareable public key (`id_ed25519.pub`): if a system administrator - asks for a key, this is the one to send. It is also safe to upload to - websites such as GitHub: it is meant to be seen. - -### SSH Agent for Easier Key Handling - -An SSH key is only as strong as the password used to unlock it, but on the -other hand, typing out a complex password every time you connect to a machine -is tedious and gets old very fast. This is where the [SSH Agent][ssh-agent] -comes in. - -Using an SSH Agent, you can type your password for the private key once, then -have the Agent remember it for some number of hours or until you log off. -Unless some nefarious actor has physical access to your machine, this keeps the -password safe, and removes the tedium of entering the password multiple times. - -Just remember your password, because once it expires in the Agent, you have to -type it in again. - -#### SSH Agents on Linux, macOS, and Windows - -Open your terminal application and check if an agent is running: - -``` -{{ site.local.prompt }} ssh-add -l -``` -{: .language-bash} - -* If you get an error like this one, - - ``` - Error connecting to agent: No such file or directory - ``` - {: .error} - - ... then you need to launch the agent as follows: - - ``` - {{ site.local.prompt }} eval $(ssh-agent) - ``` - {: .language-bash} - - > ## What's in a `$(...)`? - > - > The syntax of this SSH Agent command is unusual, based on what we've seen - > in the UNIX Shell lesson. This is because the `ssh-agent` command creates - > opens a connection that only you have access to, and prints a series of - > shell commands that can be used to reach it -- but _does not execute them!_ - > - > ``` - > {{ site.local.prompt }} ssh-agent - > ``` - > {: .language-bash} - > ``` - > SSH_AUTH_SOCK=/tmp/ssh-Zvvga2Y8kQZN/agent.131521; - > export SSH_AUTH_SOCK; - > SSH_AGENT_PID=131522; - > export SSH_AGENT_PID; - > echo Agent pid 131522; - > ``` - > {: .output} - > - > The `eval` command interprets this text output as commands and allows you - > to access the SSH Agent connection you just created. - > - > You could run each line of the `ssh-agent` output yourself, and - > achieve the same result. Using `eval` just makes this easier. - {: .callout} - -* Otherwise, your agent is already running: don't mess with it. - -Add your key to the agent, with session expiration after 8 hours: - -``` -{{ site.local.prompt }} ssh-add -t 8h ~/.ssh/id_ed25519 -``` -{: .language-bash} -``` -Enter passphrase for .ssh/id_ed25519: -Identity added: .ssh/id_ed25519 -Lifetime set to 86400 seconds -``` -{: .output} - -For the duration (8 hours), whenever you use that key, the SSH Agent will -provide the key on your behalf without you having to type a single keystroke. - -#### SSH Agent on PuTTY - -If you are using PuTTY on Windows, download and use `pageant` as the SSH agent. -See the [PuTTY documentation][putty-agent]. - -### Transfer Your Public Key - -{% if site.remote.portal %} -Visit [{{ site.remote.portal }}]({{ site.remote.portal }}) to upload your SSH -public key. (Remember, it's the one ending in `.pub`!) - -{% else %} -Use the **s**ecure **c**o**p**y tool to send your public key to the cluster. - -``` -{{ site.local.prompt }} scp ~/.ssh/id_ed25519.pub {{ site.remote.user }}@{{ site.remote.login }}:~/ -``` -{: .language-bash} -{% endif %} - -## Log In to the Cluster - -Go ahead and open your terminal or graphical SSH client, then log in to the -cluster. Replace `{{ site.remote.user }}` with your username or the one -supplied by the instructors. - -``` -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -``` -{: .language-bash} - -You may be asked for your password. Watch out: the characters you type after -the password prompt are not displayed on the screen. Normal output will resume -once you press `Enter`. - -You may have noticed that the prompt changed when you logged into the remote -system using the terminal (if you logged in using PuTTY this will not apply -because it does not offer a local terminal). This change is important because -it can help you distinguish on which system the commands you type will be run -when you pass them into the terminal. This change is also a small complication -that we will need to navigate throughout the workshop. Exactly what is displayed -as the prompt (which conventionally ends in `$`) in the terminal when it is -connected to the local system and the remote system will typically be different -for every user. We still need to indicate which system we are entering commands -on though so we will adopt the following convention: - -* `{{ site.local.prompt }}` when the command is to be entered on a terminal - connected to your local computer -* `{{ site.remote.prompt }}` when the command is to be entered on a - terminal connected to the remote system -* `$` when it really doesn't matter which system the terminal is connected to. - -## Looking Around Your Remote Home - -Very often, many users are tempted to think of a high-performance computing -installation as one giant, magical machine. Sometimes, people will assume that -the computer they've logged onto is the entire computing cluster. So what's -really happening? What computer have we logged on to? The name of the current -computer we are logged onto can be checked with the `hostname` command. (You -may also notice that the current hostname is also part of our prompt!) - -``` -{{ site.remote.prompt }} hostname -``` -{: .language-bash} - -``` -{{ site.remote.host }} -``` -{: .output} - -So, we're definitely on the remote machine. Next, let's find out where we are -by running `pwd` to **p**rint the **w**orking **d**irectory. - -``` -{{ site.remote.prompt }} pwd -``` -{: .language-bash} - -``` -{{ site.remote.homedir }}/{{ site.remote.user }} -``` -{: .output} - -Great, we know where we are! Let's see what's in our current directory: - -``` -{{ site.remote.prompt }} ls -``` -{: .language-bash} -``` -id_ed25519.pub -``` -{: .output} - -The system administrators may have configured your home directory with some -helpful files, folders, and links (shortcuts) to space reserved for you on -other filesystems. If they did not, your home directory may appear empty. To -double-check, include hidden files in your directory listing: - -``` -{{ site.remote.prompt }} ls -a -``` -{: .language-bash} -``` - . .bashrc id_ed25519.pub - .. .ssh -``` -{: .output} - -In the first column, `.` is a reference to the current directory and `..` a -reference to its parent (`{{ site.remote.homedir }}`). You may or may not see -the other files, or files like them: `.bashrc` is a shell configuration file, -which you can edit with your preferences; and `.ssh` is a directory storing SSH -keys and a record of authorized connections. - -{% unless site.remote.portal %} - -### Install Your SSH Key - -> ## There May Be a Better Way -> -> Policies and practices for handling SSH keys vary between HPC clusters: -> follow any guidance provided by the cluster administrators or -> documentation. In particular, if there is an online portal for managing SSH -> keys, use that instead of the directions outlined here. -{: .callout} - -If you transferred your SSH public key with `scp`, you should see -`id_ed25519.pub` in your home directory. To "install" this key, it must be -listed in a file named `authorized_keys` under the `.ssh` folder. - -If the `.ssh` folder was not listed above, then it does not yet -exist: create it. - -``` -{{ site.remote.prompt }} mkdir ~/.ssh -``` -{: .language-bash} - -Now, use `cat` to print your public key, but redirect the output, appending it -to the `authorized_keys` file: - -``` -{{ site.remote.prompt }} cat ~/id_ed25519.pub >> ~/.ssh/authorized_keys -``` -{: .language-bash} - -That's all! Disconnect, then try to log back into the remote: if your key and -agent have been configured correctly, you should not be prompted for the -password for your SSH key. - -``` -{{ site.remote.prompt }} logout -``` -{: .language-bash} - -``` -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -``` -{: .language-bash} -{% endunless %} - -{% include links.md %} - -[bitwarden]: https://bitwarden.com -[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard -[gh-ssh]: https://docs.github.com/en/authentication/connecting-to-github-with-ssh -[keepass]: https://keepass.info -[putty-gen]: https://tartarus.org/~simon/putty-prerel-snapshots/htmldoc/Chapter8.html#pubkey-puttygen -[putty-agent]: https://tartarus.org/~simon/putty-prerel-snapshots/htmldoc/Chapter9.html#pageant -[ssh-agent]: https://www.ssh.com/academy/ssh/agent -[ssh-flags]: https://stribika.github.io/2015/01/04/secure-secure-shell.html -[wiki-rsa]: https://en.wikipedia.org/wiki/RSA_(cryptosystem) -[wiki-dsa]: https://en.wikipedia.org/wiki/EdDSA diff --git a/_episodes/12-cluster.md b/_episodes/12-cluster.md deleted file mode 100644 index 3e5176d7..00000000 --- a/_episodes/12-cluster.md +++ /dev/null @@ -1,312 +0,0 @@ ---- -title: "Exploring Remote Resources" -teaching: 25 -exercises: 10 -questions: -- "How does my local computer compare to the remote systems?" -- "How does the login node compare to the compute nodes?" -- "Are all compute nodes alike?" -objectives: -- "Survey system resources using `nproc`, `free`, and the queuing system" -- "Compare & contrast resources on the local machine, login node, and worker -nodes" -- "Learn about the various filesystems on the cluster using `df`" -- "Find out `who` else is logged in" -- "Assess the number of idle and occupied nodes" -keypoints: -- "An HPC system is a set of networked machines." -- "HPC systems typically provide login nodes and a set of compute nodes." -- "The resources found on independent (worker) nodes can vary in volume and - type (amount of RAM, processor architecture, availability of network mounted - filesystems, etc.)." -- "Files saved on shared storage are available on all nodes." -- "The login node is a shared machine: be considerate of other users." ---- - -## Look Around the Remote System - -If you have not already connected to {{ site.remote.name }}, please do so now: - -``` -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -``` -{: .language-bash} - -Take a look at your home directory on the remote system: - -``` -{{ site.remote.prompt }} ls -``` -{: .language-bash} - -> ## What's different between your machine and the remote? -> -> Open a second terminal window on your local computer and run the `ls` command -> (without logging in to {{ site.remote.name }}). What differences do you see? -> -> > ## Solution -> > -> > You would likely see something more like this: -> > -> > ``` -> > {{ site.local.prompt }} ls -> > ``` -> > {: .language-bash} -> > ``` -> > Applications Documents Library Music Public -> > Desktop Downloads Movies Pictures -> > ``` -> > {: .output} -> > -> > The remote computer's home directory shares almost nothing in common with -> > the local computer: they are completely separate systems! -> {: .solution} -{: .discussion} - -Most high-performance computing systems run the Linux operating system, which -is built around the UNIX [Filesystem Hierarchy Standard][fshs]. Instead of -having a separate root for each hard drive or storage medium, all files and -devices are anchored to the "root" directory, which is `/`: - -``` -{{ site.remote.prompt }} ls / -``` -{: .language-bash} -``` -bin etc lib64 proc sbin sys var -boot {{ site.remote.homedir | replace: "/", "" }} mnt root scratch tmp working -dev lib opt run srv usr -``` -{: .output} - -The "{{ site.remote.homedir | replace: "/", "" }}" directory is the one where -we generally want to keep all of our files. Other folders on a UNIX OS contain -system files and change as you install new software or upgrade your OS. - -> ## Using HPC filesystems -> -> On HPC systems, you have a number of places where you can store your files. -> These differ in both the amount of space allocated and whether or not they -> are backed up. -> -> * __Home__ -- often a _network filesystem_, data stored here is available -> throughout the HPC system, and often backed up periodically. Files stored -> here are typically slower to access, the data is actually stored on another -> computer and is being transmitted and made available over the network! -> * __Scratch__ -- typically faster than the networked Home directory, but not -> usually backed up, and should not be used for long term storage. -> * __Work__ -- sometimes provided as an alternative to Scratch space, Work is -> a fast file system accessed over the network. Typically, this will have -> higher performance than your home directory, but lower performance than -> Scratch; it may not be backed up. It differs from Scratch space in that -> files in a work file system are not automatically deleted for you: you must -> manage the space yourself. -{: .callout} - -## Nodes - -Individual computers that compose a cluster are typically called _nodes_ -(although you will also hear people call them _servers_, _computers_ and -_machines_). On a cluster, there are different types of nodes for different -types of tasks. The node where you are right now is called the _login node_, -_head node_, _landing pad_, or _submit node_. A login node serves as an access -point to the cluster. - -As a gateway, the login node should not be used for time-consuming or -resource-intensive tasks. You should be alert to this, and check with your -site's operators or documentation for details of what is and isn't allowed. It -is well suited for uploading and downloading files, setting up software, and -running tests. Generally speaking, in these lessons, we will avoid running jobs -on the login node. - -Who else is logged in to the login node? - -``` -{{ site.remote.prompt }} who -``` -{: .language-bash} - -This may show only your user ID, but there are likely several other people -(including fellow learners) connected right now. - -> ## Dedicated Transfer Nodes -> -> If you want to transfer larger amounts of data to or from the cluster, some -> systems offer dedicated nodes for data transfers only. The motivation for -> this lies in the fact that larger data transfers should not obstruct -> operation of the login node for anybody else. Check with your cluster's -> documentation or its support team if such a transfer node is available. As a -> rule of thumb, consider all transfers of a volume larger than 500 MB to 1 GB -> as large. But these numbers change, e.g., depending on the network connection -> of yourself and of your cluster or other factors. -{: .callout} - -The real work on a cluster gets done by the _compute_ (or _worker_) _nodes_. -compute nodes come in many shapes and sizes, but generally are dedicated to long -or hard tasks that require a lot of computational resources. - -All interaction with the compute nodes is handled by a specialized piece of -software called a scheduler (the scheduler used in this lesson is called -{{ site.sched.name }}). We'll learn more about how to use the -scheduler to submit jobs next, but for now, it can also tell us more -information about the compute nodes. - -For example, we can view all of the compute nodes by running the command -`{{ site.sched.info }}`. - -``` -{{ site.remote.prompt }} {{ site.sched.info }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/cluster/queue-info.snip %} - -A lot of the nodes are busy running work for other users: we are not alone -here! - -There are also specialized machines used for managing disk storage, user -authentication, and other infrastructure-related tasks. Although we do not -typically logon to or interact with these machines directly, they enable a -number of key features like ensuring our user account and files are available -throughout the HPC system. - -## What's in a Node? - -All of the nodes in an HPC system have the same components as your own laptop -or desktop: _CPUs_ (sometimes also called _processors_ or _cores_), _memory_ -(or _RAM_), and _disk_ space. CPUs are a computer's tool for actually running -programs and calculations. Information about a current task is stored in the -computer's memory. Disk refers to all storage that can be accessed like a file -system. This is generally storage that can hold data permanently, i.e. data is -still there even if the computer has been restarted. While this storage can be -local (a hard drive installed inside of it), it is more common for nodes to -connect to a shared, remote fileserver or cluster of servers. - -{% include figure.html url="" max-width="40%" - file="/fig/node_anatomy.png" - alt="Node anatomy" caption="" %} - -> ## Explore Your Computer -> -> Try to find out the number of CPUs and amount of memory available on your -> personal computer. -> -> Note that, if you're logged in to the remote computer cluster, you need to -> log out first. To do so, type `Ctrl+d` or `exit`: -> -> ``` -> {{ site.remote.prompt }} exit -> {{ site.local.prompt }} -> ``` -> {: .language-bash} -> -> > ## Solution -> > -> > There are several ways to do this. Most operating systems have a graphical -> > system monitor, like the Windows Task Manager. More detailed information -> > can be found on the command line: -> > -> > * Run system utilities -> > ``` -> > {{ site.local.prompt }} nproc --all -> > {{ site.local.prompt }} free -m -> > ``` -> > {: .language-bash} -> > -> > * Read from `/proc` -> > ``` -> > {{ site.local.prompt }} cat /proc/cpuinfo -> > {{ site.local.prompt }} cat /proc/meminfo -> > ``` -> > {: .language-bash} -> > -> > * Run system monitor -> > ``` -> > {{ site.local.prompt }} htop -> > ``` -> > {: .language-bash} -> {: .solution} -{: .challenge} - -> ## Explore the Login Node -> -> Now compare the resources of your computer with those of the login node. -> -> > ## Solution -> > -> > ``` -> > {{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -> > {{ site.remote.prompt }} nproc --all -> > {{ site.remote.prompt }} free -m -> > ``` -> > {: .language-bash} -> > -> > You can get more information about the processors using `lscpu`, -> > and a lot of detail about the memory by reading the file `/proc/meminfo`: -> > -> > ``` -> > {{ site.remote.prompt }} less /proc/meminfo -> > ``` -> > {: .language-bash} -> > -> > You can also explore the available filesystems using `df` to show **d**isk -> > **f**ree space. The `-h` flag renders the sizes in a human-friendly format, -> > i.e., GB instead of B. The **t**ype flag `-T` shows what kind of filesystem -> > each resource is. -> > -> > ``` -> > {{ site.remote.prompt }} df -Th -> > ``` -> > {: .language-bash} -> > -> > > ## Different results from `df` -> > > -> > > * The local filesystems (ext, tmp, xfs, zfs) will depend on whether -> > > you're on the same login node (or compute node, later on). -> > > * Networked filesystems (beegfs, cifs, gpfs, nfs, pvfs) will be similar -> > > -- but may include {{ site.remote.user }}, depending on how it -> > > is [mounted][mount]. -> > {: .discussion} -> > -> > > ## Shared Filesystems -> > > -> > > This is an important point to remember: files saved on one node -> > > (computer) are often available everywhere on the cluster! -> > {: .callout} -> {: .solution} -{: .challenge} - -{% include {{ site.snippets }}/cluster/specific-node-info.snip %} - -> ## Compare Your Computer, the Login Node and the Compute Node -> -> Compare your laptop's number of processors and memory with the numbers you -> see on the cluster login node and compute node. What implications do -> you think the differences might have on running your research work on the -> different systems and nodes? -> -> > ## Solution -> > -> > Compute nodes are usually built with processors that have _higher -> > core-counts_ than the login node or personal computers in order to support -> > highly parallel tasks. Compute nodes usually also have substantially _more -> > memory (RAM)_ installed than a personal computer. More cores tends to help -> > jobs that depend on some work that is easy to perform in _parallel_, and -> > more, faster memory is key for large or _complex numerical tasks_. -> {: .solution} -{: .discussion} - -> ## Differences Between Nodes -> -> Many HPC clusters have a variety of nodes optimized for particular workloads. -> Some nodes may have larger amount of memory, or specialized resources such as -> Graphics Processing Units (GPUs or "video cards"). -{: .callout} - -With all of this in mind, we will now cover how to talk to the cluster's -scheduler, and use it to start running our scripts and programs! - -{% include links.md %} - -[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard -[mount]: https://en.wikipedia.org/wiki/Mount_(computing) diff --git a/_episodes/13-scheduler.md b/_episodes/13-scheduler.md deleted file mode 100644 index 22263304..00000000 --- a/_episodes/13-scheduler.md +++ /dev/null @@ -1,328 +0,0 @@ ---- -title: "Scheduler Fundamentals" -teaching: 45 -exercises: 30 -questions: -- "What is a scheduler and why does a cluster need one?" -- "How do I launch a program to run on a compute node in the cluster?" -- "How do I capture the output of a program that is run on a node in the - cluster?" -objectives: -- "Submit a simple script to the cluster." -- "Monitor the execution of jobs using command line tools." -- "Inspect the output and error files of your jobs." -- "Find the right place to put large datasets on the cluster." -keypoints: -- "The scheduler handles how compute resources are shared between users." -- "A job is just a shell script." -- "Request _slightly_ more resources than you will need." ---- - -## Job Scheduler - -An HPC system might have thousands of nodes and thousands of users. How do we -decide who gets what and when? How do we ensure that a task is run with the -resources it needs? This job is handled by a special piece of software called -the _scheduler_. On an HPC system, the scheduler manages which jobs run where -and when. - -The following illustration compares these tasks of a job scheduler to a waiter -in a restaurant. If you can relate to an instance where you had to wait for a -while in a queue to get in to a popular restaurant, then you may now understand -why sometimes your job do not start instantly as in your laptop. - -{% include figure.html max-width="75%" caption="" - file="/fig/restaurant_queue_manager.svg" - alt="Compare a job scheduler to a waiter in a restaurant" %} - -The scheduler used in this lesson is {{ site.sched.name }}. Although -{{ site.sched.name }} is not used everywhere, running jobs is quite similar -regardless of what software is being used. The exact syntax might change, but -the concepts remain the same. - -## Running a Batch Job - -The most basic use of the scheduler is to run a command non-interactively. Any -command (or series of commands) that you want to run on the cluster is called a -_job_, and the process of using a scheduler to run the job is called _batch job -submission_. - -In this case, the job we want to run is a shell script -- essentially a -text file containing a list of UNIX commands to be executed in a sequential -manner. Our shell script will have three parts: - -* On the very first line, add `{{ site.remote.bash_shebang }}`. The `#!` - (pronounced "hash-bang" or "shebang") tells the computer what program is - meant to process the contents of this file. In this case, we are telling it - that the commands that follow are written for the command-line shell (what - we've been doing everything in so far). -* Anywhere below the first line, we'll add an `echo` command with a friendly - greeting. When run, the shell script will print whatever comes after `echo` - in the terminal. - * `echo -n` will print everything that follows, _without_ ending - the line by printing the new-line character. -* On the last line, we'll invoke the `hostname` command, which will print the - name of the machine the script is run on. - -``` -{{ site.remote.prompt }} nano example-job.sh -``` -{: .language-bash} -``` -{{ site.remote.bash_shebang }} - -echo -n "This script is running on " -hostname -``` -{: .output} - -> ## Creating Our Test Job -> -> Run the script. Does it execute on the cluster or just our login node? -> -> > ## Solution -> > -> > ``` -> > {{ site.remote.prompt }} bash example-job.sh -> > ``` -> > {: .language-bash} -> > ``` -> > This script is running on {{ site.remote.host }} -> > ``` -> > {: .output} -> {: .solution} -{: .challenge} - -This script ran on the login node, but we want to take advantage of -the compute nodes: we need the scheduler to queue up `example-job.sh` -to run on a compute node. - -To submit this task to the scheduler, we use the -`{{ site.sched.submit.name }}` command. -This creates a _job_ which will run the _script_ when _dispatched_ to -a compute node which the queuing system has identified as being -available to perform the work. - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/basic-job-script.snip %} - -And that's all we need to do to submit a job. Our work is done -- now the -scheduler takes over and tries to run the job for us. While the job is waiting -to run, it goes into a list of jobs called the _queue_. To check on our job's -status, we check the queue using the command -`{{ site.sched.status }} {{ site.sched.flag.user }}`. - -``` -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/basic-job-status.snip %} - -> ## Where's the Output? -> -> On the login node, this script printed output to the terminal -- but -> now, when `{{ site.sched.status }}` shows the job has finished, -> nothing was printed to the terminal. -> -> Cluster job output is typically redirected to a file in the directory you -> launched it from. Use `ls` to find and `cat` to read the file. -{: .discussion} - -## Customising a Job - -The job we just ran used all of the scheduler's default options. In a -real-world scenario, that's probably not what we want. The default options -represent a reasonable minimum. Chances are, we will need more cores, more -memory, more time, among other special considerations. To get access to these -resources we must customize our job script. - -Comments in UNIX shell scripts (denoted by `#`) are typically ignored, but -there are exceptions. For instance the special `#!` comment at the beginning of -scripts specifies what program should be used to run it (you'll typically see -`{{ site.local.bash_shebang }}`). Schedulers like {{ site.sched.name }} also -have a special comment used to denote special scheduler-specific options. -Though these comments differ from scheduler to scheduler, -{{ site.sched.name }}'s special comment is `{{ site.sched.comment }}`. Anything -following the `{{ site.sched.comment }}` comment is interpreted as an -instruction to the scheduler. - -Let's illustrate this by example. By default, a job's name is the name of the -script, but the `{{ site.sched.flag.name }}` option can be used to change the -name of a job. Add an option to the script: - -``` -{{ site.remote.prompt }} cat example-job.sh -``` -{: .language-bash} - -``` -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} hello-world - -echo -n "This script is running on " -hostname -``` -{: .output} - -Submit the job and monitor its status: - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/job-with-name-status.snip %} - -Fantastic, we've successfully changed the name of our job! - -### Resource Requests - -What about more important changes, such as the number of cores and memory for -our jobs? One thing that is absolutely critical when working on an HPC system -is specifying the resources required to run a job. This allows the scheduler to -find the right time and place to schedule our job. If you do not specify -requirements (such as the amount of time you need), you will likely be stuck -with your site's default resources, which is probably not what you want. - -The following are several key resource requests: - -{% include {{ site.snippets }}/scheduler/option-flags-list.snip %} - -Note that just _requesting_ these resources does not make your job run faster, -nor does it necessarily mean that you will consume all of these resources. It -only means that these are made available to you. Your job may end up using less -memory, or less time, or fewer nodes than you have requested, and it will still -run. - -It's best if your requests accurately reflect your job's requirements. We'll -talk more about how to make sure that you're using resources effectively in a -later episode of this lesson. - -> ## Submitting Resource Requests -> -> Modify our `hostname` script so that it runs for a minute, then submit a job -> for it on the cluster. -> -> > ## Solution -> > -> > ``` -> > {{ site.remote.prompt }} cat example-job.sh -> > ``` -> > {: .language-bash} -> > -> > ``` -> > {{ site.remote.bash_shebang }} -> > {{ site.sched.comment }} {{ site.sched.flag.time }} 00:01 # timeout in HH:MM -> > -> > echo -n "This script is running on " -> > sleep 20 # time in seconds -> > hostname -> > ``` -> > {: .output} -> > -> > ``` -> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -> > ``` -> > {: .language-bash} -> > -> > Why are the {{ site.sched.name }} runtime and `sleep` time not identical? -> {: .solution} -{: .challenge} - -Resource requests are typically binding. If you exceed them, your job will be -killed. Let's use wall time as an example. We will request 1 minute of -wall time, and attempt to run a job for two minutes. - -``` -{{ site.remote.prompt }} cat example-job.sh -``` -{: .language-bash} - -``` -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} long_job -{{ site.sched.comment }} {{ site.sched.flag.time }} 00:01 # timeout in HH:MM - -echo "This script is running on ... " -sleep 240 # time in seconds -hostname -``` -{: .output} - -Submit the job and wait for it to finish. Once it is has finished, check the -log file. - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/runtime-exceeded-job.snip %} - -{% include {{ site.snippets }}/scheduler/runtime-exceeded-output.snip %} - -Our job was killed for exceeding the amount of resources it requested. Although -this appears harsh, this is actually a feature. Strict adherence to resource -requests allows the scheduler to find the best possible place for your jobs. -Even more importantly, it ensures that another user cannot use more resources -than they've been given. If another user messes up and accidentally attempts to -use all of the cores or memory on a node, {{ site.sched.name }} will either -restrain their job to the requested resources or kill the job outright. Other -jobs on the node will be unaffected. This means that one user cannot mess up -the experience of others, the only jobs affected by a mistake in scheduling -will be their own. - -## Cancelling a Job - -Sometimes we'll make a mistake and need to cancel a job. This can be done with -the `{{ site.sched.del }}` command. Let's submit a job and then cancel it using -its job number (remember to change the walltime so that it runs long enough for -you to cancel it before it is killed!). - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/terminate-job-begin.snip %} - -Now cancel the job with its job number (printed in your terminal). A clean -return of your command prompt indicates that the request to cancel the job was -successful. - -``` -{{ site.remote.prompt }} {{site.sched.del }} 38759 -# It might take a minute for the job to disappear from the queue... -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/scheduler/terminate-job-cancel.snip %} - -{% include {{ site.snippets }}/scheduler/terminate-multiple-jobs.snip %} - -## Other Types of Jobs - -Up to this point, we've focused on running jobs in batch mode. -{{ site.sched.name }} also provides the ability to start an interactive session. - -There are very frequently tasks that need to be done interactively. Creating an -entire job script might be overkill, but the amount of resources required is -too much for a login node to handle. A good example of this might be building a -genome index for alignment with a tool like [HISAT2][hisat]. Fortunately, we -can run these types of tasks as a one-off with `{{ site.sched.interactive }}`. - -{% include {{ site.snippets }}/scheduler/using-nodes-interactively.snip %} - -{% include links.md %} - -[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard -[hisat]: https://daehwankimlab.github.io/hisat2/ diff --git a/_episodes/15-modules.md b/_episodes/15-modules.md deleted file mode 100644 index 08a80f6a..00000000 --- a/_episodes/15-modules.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -title: "Accessing software via Modules" -teaching: 30 -exercises: 15 -questions: -- "How do we load and unload software packages?" -objectives: -- "Load and use a software package." -- "Explain how the shell environment changes when the module mechanism loads or unloads packages." -keypoints: -- "Load software with `module load softwareName`." -- "Unload software with `module unload`" -- "The module system handles software versioning and package conflicts for you - automatically." ---- - -On a high-performance computing system, it is seldom the case that the software -we want to use is available when we log in. It is installed, but we will need -to "load" it before it can run. - -Before we start using individual software packages, however, we should -understand the reasoning behind this approach. The three biggest factors are: - -- software incompatibilities -- versioning -- dependencies - -Software incompatibility is a major headache for programmers. Sometimes the -presence (or absence) of a software package will break others that depend on -it. Two well known examples are Python and C compiler versions. -Python 3 famously provides a `python` command that conflicts with that provided -by Python 2. Software compiled against a newer version of the C libraries and -then run on a machine that has older C libraries installed will result in a -nasty `'GLIBCXX_3.4.20' not found` error. - -Software versioning is another common issue. A team might depend on a certain -package version for their research project - if the software version was to -change (for instance, if a package was updated), it might affect their results. -Having access to multiple software versions allows a set of researchers to -prevent software versioning issues from affecting their results. - -Dependencies are where a particular software package (or even a particular -version) depends on having access to another software package (or even a -particular version of another software package). For example, the VASP -materials science software may depend on having a particular version of the -FFTW (Fastest Fourier Transform in the West) software library available for it -to work. - -## Environment Modules - -Environment modules are the solution to these problems. A _module_ is a -self-contained description of a software package -- it contains the -settings required to run a software package and, usually, encodes required -dependencies on other software packages. - -There are a number of different environment module implementations commonly -used on HPC systems: the two most common are _TCL modules_ and _Lmod_. Both of -these use similar syntax and the concepts are the same so learning to use one -will allow you to use whichever is installed on the system you are using. In -both implementations the `module` command is used to interact with environment -modules. An additional subcommand is usually added to the command to specify -what you want to do. For a list of subcommands you can use `module -h` or -`module help`. As for all commands, you can access the full help on the _man_ -pages with `man module`. - -On login you may start out with a default set of modules loaded or you may -start out with an empty environment; this depends on the setup of the system -you are using. - -### Listing Available Modules - -To see available software modules, use `module avail`: - -``` -{{ site.remote.prompt }} module avail -``` -{: .language-bash} - -{% include {{ site.snippets }}/modules/available-modules.snip %} - -### Listing Currently Loaded Modules - -You can use the `module list` command to see which modules you currently have -loaded in your environment. If you have no modules loaded, you will see a -message telling you so - -``` -{{ site.remote.prompt }} module list -``` -{: .language-bash} - -{% include {{ site.snippets }}/modules/default-modules.snip %} - -## Loading and Unloading Software - -To load a software module, use `module load`. In this example we will use -Python 3. - -Initially, Python 3 is not loaded. We can test this by using the `which` -command. `which` looks for programs the same way that Bash does, so we can use -it to tell us where a particular piece of software is stored. - -``` -{{ site.remote.prompt }} which python3 -``` -{: .language-bash} - -{% include {{ site.snippets }}/modules/missing-python.snip %} - -We can load the `python3` command with `module load`: - -{% include {{ site.snippets }}/modules/module-load-python.snip %} - -{% include {{ site.snippets }}/modules/python-executable-dir.snip %} - -So, what just happened? - -To understand the output, first we need to understand the nature of the `$PATH` -environment variable. `$PATH` is a special environment variable that controls -where a UNIX system looks for software. Specifically `$PATH` is a list of -directories (separated by `:`) that the OS searches through for a command -before giving up and telling us it can't find it. As with all environment -variables we can print it out using `echo`. - -``` -{{ site.remote.prompt }} echo $PATH -``` -{: .language-bash} - -{% include {{ site.snippets }}/modules/python-module-path.snip %} - -You'll notice a similarity to the output of the `which` command. In this case, -there's only one difference: the different directory at the beginning. When we -ran the `module load` command, it added a directory to the beginning of our -`$PATH`. Let's examine what's there: - -{% include {{ site.snippets }}/modules/python-ls-dir-command.snip %} - -{% include {{ site.snippets }}/modules/python-ls-dir-output.snip %} - -Taking this to its conclusion, `module load` will add software to your `$PATH`. -It "loads" software. A special note on this - depending on which version of the -`module` program that is installed at your site, `module load` will also load -required software dependencies. - -{% include {{ site.snippets }}/modules/software-dependencies.snip %} - -Note that this module loading process happens principally through -the manipulation of environment variables like `$PATH`. There -is usually little or no data transfer involved. - -The module loading process manipulates other special environment -variables as well, including variables that influence where the -system looks for software libraries, and sometimes variables which -tell commercial software packages where to find license servers. - -The module command also restores these shell environment variables -to their previous state when a module is unloaded. - -## Software Versioning - -So far, we've learned how to load and unload software packages. This is very -useful. However, we have not yet addressed the issue of software versioning. At -some point or other, you will run into issues where only one particular version -of some software will be suitable. Perhaps a key bugfix only happened in a -certain version, or version X broke compatibility with a file format you use. -In either of these example cases, it helps to be very specific about what -software is loaded. - -Let's examine the output of `module avail` more closely. - -``` -{{ site.remote.prompt }} module avail -``` -{: .language-bash} - -{% include {{ site.snippets }}/modules/available-modules.snip %} - -{% include {{ site.snippets }}/modules/wrong-gcc-version.snip %} - -> ## Using Software Modules in Scripts -> -> Create a job that is able to run `python3 --version`. Remember, no software -> is loaded by default! Running a job is just like logging on to the system -> (you should not assume a module loaded on the login node is loaded on a -> compute node). -> -> > ## Solution -> > -> > ``` -> > {{ site.remote.prompt }} nano python-module.sh -> > {{ site.remote.prompt }} cat python-module.sh -> > ``` -> > {: .language-bash} -> > -> > ``` -> > {{ site.remote.bash_shebang }} -> > {{ site.sched.comment }} {{ site.sched.flag.partition }}{% if site.sched.flag.qos %} -> > {{ site.sched.comment }} {{ site.sched.flag.qos }} -> > {% endif %}{{ site.sched.comment }} {{ site.sched.flag.time }} 00:00:30 -> > -> > module load {{ site.remote.module_python3 }} -> > -> > python3 --version -> > ``` -> > {: .output} -> > -> > ``` -> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}python-module.sh -> > ``` -> > {: .language-bash} -> {: .solution} -{: .challenge} - -{% include links.md %} diff --git a/_episodes/16-transferring-files.md b/_episodes/16-transferring-files.md deleted file mode 100644 index 5548dd6d..00000000 --- a/_episodes/16-transferring-files.md +++ /dev/null @@ -1,461 +0,0 @@ ---- -title: "Transferring files with remote computers" -teaching: 15 -exercises: 15 -questions: -- "How do I transfer files to (and from) the cluster?" -objectives: -- "Transfer files to and from a computing cluster." -keypoints: -- "`wget` and `curl -O` download a file from the internet." -- "`scp` and `rsync` transfer files to and from your computer." -- "You can use an SFTP client like FileZilla to transfer files through a GUI." ---- - -Performing work on a remote computer is not very useful if we cannot get files -to or from the cluster. There are several options for transferring data between -computing resources using CLI and GUI utilities, a few of which we will cover. - -## Download Lesson Files From the Internet - -One of the most straightforward ways to download files is to use either `curl` -or `wget`. One of these is usually installed in most Linux shells, on Mac OS -terminal and in GitBash. Any file that can be downloaded in your web browser -through a direct link can be downloaded using `curl` or `wget`. This is a -quick way to download datasets or source code. The syntax for these commands is - -* `wget [-O new_name] https://some/link/to/a/file` -* `curl [-o new_name] https://some/link/to/a/file` - -Try it out by downloading some material we'll use later on, from a terminal on -your local machine, using the URL of the current codebase: - - - -> ## Download the "Tarball" -> -> The word "tarball" in the above URL refers to a compressed archive format -> commonly used on Linux, which is the operating system the majority of HPC -> cluster machines run. -> A tarball is a lot like a `.zip` file. -> The actual file extension is `.tar.gz`, which reflects the two-stage process -> used to create the file: -> the files or folders are merged into a single file using `tar`, which is then -> compressed using `gzip`, so the file extension is "tar-dot-g-z." -> That's a mouthful, so people often say "the _xyz_ tarball" instead. -> -> You may also see the extension `.tgz`, which is just an abbreviation of -> `.tar.gz`. -> -> By default, `curl` and `wget` download files to the same name as the URL: -> in this case, `main`. -> Use one of the above commands to save the tarball as `amdahl.tar.gz`. -> -> > ## `wget` and `curl` Commands -> > -> > ``` -> > {{ site.local.prompt }} wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main -> > # or -> > {{ site.local.prompt }} curl -o amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main -> > ``` -> > {: .language-bash} -> {: .solution} -{: .challenge} - -After downloading the file, use `ls` to see it in your working directory: - -``` -{{ site.local.prompt }} ls -``` -{: .language-bash} - -## Archiving Files - -One of the biggest challenges we often face when transferring data between -remote HPC systems is that of large numbers of files. There is an overhead to -transferring each individual file and when we are transferring large numbers of -files these overheads combine to slow down our transfers to a large degree. - -The solution to this problem is to _archive_ multiple files into smaller -numbers of larger files before we transfer the data to improve our transfer -efficiency. -Sometimes we will combine archiving with _compression_ to reduce the amount of -data we have to transfer and so speed up the transfer. -The most common archiving command you will use on a (Linux) HPC cluster is -`tar`. - -`tar` can be used to combine files and folders into a single archive file and, -optionally, compress the result. -Let's look at the file we downloaded from the lesson site, `amdahl.tar.gz`. - -The `.gz` part stands for _gzip_, which is a compression library. -It's common (but not necessary!) that this kind of file can be interpreted by -reading its name: it appears somebody took files and folders relating to -something called "amdahl," wrapped them all up into a single file with `tar`, -then compressed that archive with `gzip` to save space. - -Let's see if that is the case, _without_ unpacking the file. -`tar` prints the "**t**able of contents" with the `-t` flag, for the file -specified with the `-f` flag followed by the filename. -Note that you can concatenate the two flags: writing `-t -f` is interchangeable -with writing `-tf` together. -However, the argument following `-f` must be a filename, so writing `-ft` will -_not_ work. - -``` -{{ site.local.prompt }} tar -tf amdahl.tar.gz -hpc-carpentry-amdahl-46c9b4b/ -hpc-carpentry-amdahl-46c9b4b/.github/ -hpc-carpentry-amdahl-46c9b4b/.github/workflows/ -hpc-carpentry-amdahl-46c9b4b/.github/workflows/python-publish.yml -hpc-carpentry-amdahl-46c9b4b/.gitignore -hpc-carpentry-amdahl-46c9b4b/LICENSE -hpc-carpentry-amdahl-46c9b4b/README.md -hpc-carpentry-amdahl-46c9b4b/amdahl/ -hpc-carpentry-amdahl-46c9b4b/amdahl/__init__.py -hpc-carpentry-amdahl-46c9b4b/amdahl/__main__.py -hpc-carpentry-amdahl-46c9b4b/amdahl/amdahl.py -hpc-carpentry-amdahl-46c9b4b/requirements.txt -hpc-carpentry-amdahl-46c9b4b/setup.py -``` -{: .language-bash} - -This example output shows a folder which contains a few files, where `46c9b4b` -is an 8-character [git][git-swc] commit hash that will change when the source -material is updated. - -Now let's unpack the archive. We'll run `tar` with a few common flags: - -* `-x` to e**x**tract the archive -* `-v` for **v**erbose output -* `-z` for g**z**ip compression -* `-f «tarball»` for the file to be unpacked - -> ## Extract the Archive -> -> Using the flags above, unpack the source code tarball into a new -> directory named "amdahl" using `tar`. -> -> ``` -> {{ site.local.prompt }} tar -xvzf amdahl.tar.gz -> ``` -> {: .language-bash} -> -> ``` -> hpc-carpentry-amdahl-46c9b4b/ -> hpc-carpentry-amdahl-46c9b4b/.github/ -> hpc-carpentry-amdahl-46c9b4b/.github/workflows/ -> hpc-carpentry-amdahl-46c9b4b/.github/workflows/python-publish.yml -> hpc-carpentry-amdahl-46c9b4b/.gitignore -> hpc-carpentry-amdahl-46c9b4b/LICENSE -> hpc-carpentry-amdahl-46c9b4b/README.md -> hpc-carpentry-amdahl-46c9b4b/amdahl/ -> hpc-carpentry-amdahl-46c9b4b/amdahl/__init__.py -> hpc-carpentry-amdahl-46c9b4b/amdahl/__main__.py -> hpc-carpentry-amdahl-46c9b4b/amdahl/amdahl.py -> hpc-carpentry-amdahl-46c9b4b/requirements.txt -> hpc-carpentry-amdahl-46c9b4b/setup.py -> ``` -> {: .output} -> -> Note that we did not need to type out `-x -v -z -f`, thanks to flag -> concatenation, though the command works identically either way -- -> so long as the concatenated list ends with `f`, because the next string -> must specify the name of the file to extract. -{: .discussion} - -The folder has an unfortunate name, so let's change that to something more -convenient. - -``` -{{ site.local.prompt }} mv hpc-carpentry-amdahl-46c9b4b amdahl -``` -{: .language-bash} - -Check the size of the extracted directory and compare to the compressed -file size, using `du` for "**d**isk **u**sage". - -``` -{{ site.local.prompt }} du -sh amdahl.tar.gz -8.0K amdahl.tar.gz -{{ site.local.prompt }} du -sh amdahl -48K amdahl -``` -{: .language-bash} - -Text files (including Python source code) compress nicely: -the "tarball" is one-sixth the total size of the raw data! - -If you want to reverse the process -- compressing raw data instead of -extracting it -- set a `c` flag instead of `x`, set the archive filename, -then provide a directory to compress: - -``` -{{ site.local.prompt }} tar -cvzf compressed_code.tar.gz amdahl -``` -{: .language-bash} -``` -amdahl/ -amdahl/.github/ -amdahl/.github/workflows/ -amdahl/.github/workflows/python-publish.yml -amdahl/.gitignore -amdahl/LICENSE -amdahl/README.md -amdahl/amdahl/ -amdahl/amdahl/__init__.py -amdahl/amdahl/__main__.py -amdahl/amdahl/amdahl.py -amdahl/requirements.txt -amdahl/setup.py -``` -{: .output} - -If you give `amdahl.tar.gz` as the filename in the above command, `tar` will -update the existing tarball with any changes you made to the files. -That would mean adding the new `amdahl` folder to the _existing_ folder -(`hpc-carpentry-amdahl-46c9b4b`) inside the tarball, doubling the size of the -archive! - -> ## Working with Windows -> -> When you transfer text files from a Windows system to a Unix system (Mac, -> Linux, BSD, Solaris, etc.) this can cause problems. Windows encodes its files -> slightly different than Unix, and adds an extra character to every line. -> -> On a Unix system, every line in a file ends with a `\n` (newline). On -> Windows, every line in a file ends with a `\r\n` (carriage return + newline). -> This causes problems sometimes. -> -> Though most modern programming languages and software handles this correctly, -> in some rare instances, you may run into an issue. The solution is to convert -> a file from Windows to Unix encoding with the `dos2unix` command. -> -> You can identify if a file has Windows line endings with `cat -A filename`. A -> file with Windows line endings will have `^M$` at the end of every line. A -> file with Unix line endings will have `$` at the end of a line. -> -> To convert the file, just run `dos2unix filename`. (Conversely, to convert -> back to Windows format, you can run `unix2dos filename`.) -{: .callout} - -## Transferring Single Files and Folders With `scp` - -To copy a single file to or from the cluster, we can use `scp` ("secure copy"). -The syntax can be a little complex for new users, but we'll break it down. -The `scp` command is a relative of the `ssh` command we used to -access the system, and can use the same public-key authentication -mechanism. - -To _upload to_ another computer, the template command is - -``` -{{ site.local.prompt }} scp local_file {{ site.remote.user }}@{{ site.remote.login }}:remote_destination -``` -{: .language-bash} - -in which `@` and `:` are field separators and `remote_destination` is a path -relative to your remote home directory, or a new filename if you wish to change -it, or both a relative path _and_ a new filename. -If you don't have a specific folder in mind you can omit the -`remote_destination` and the file will be copied to your home directory on the -remote computer (with its original name). -If you include a `remote_destination`, note that `scp` interprets this the same -way `cp` does when making local copies: -if it exists and is a folder, the file is copied inside the folder; if it -exists and is a file, the file is overwritten with the contents of -`local_file`; if it does not exist, it is assumed to be a destination filename -for `local_file`. - -Upload the lesson material to your remote home directory like so: - -``` -{{ site.local.prompt }} scp amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -``` -{: .language-bash} - -> ## Why Not Download on {{ site.remote.name }} Directly? -> -> Most computer clusters are protected from the open internet by a _firewall_. -> For enhanced security, some are configured to allow traffic _inbound_, but -> not _outbound_. -> This means that an authenticated user can send a file to a cluster machine, -> but a cluster machine cannot retrieve files from a user's machine or the -> open Internet. -> -> Try downloading the file directly. Note that it may well fail, and that's -> OK! -> -> > ## Commands -> > -> > ``` -> > {{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -> > {{ site.remote.prompt }} wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main -> > # or -> > {{ site.remote.prompt }} curl -o amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main -> > ``` -> > {: .language-bash} -> {: .solution} -> -> Did it work? If not, what does the terminal output tell you about what -> happened? -{: .challenge} - -## Transferring a Directory - -To transfer an entire directory, we add the `-r` flag for "**r**ecursive": -copy the item specified, and every item below it, and every item below those... -until it reaches the bottom of the directory tree rooted at the folder name you -provided. - -``` -{{ site.local.prompt }} scp -r amdahl {{ site.remote.user }}@{{ site.remote.login }}: -``` -{: .language-bash} - -> ## Caution -> -> For a large directory -- either in size or number of files -- -> copying with `-r` can take a long time to complete. -{: .callout} - -When using `scp`, you may have noticed that a `:` _always_ follows the remote -computer name. -A string _after_ the `:` specifies the remote directory you wish to transfer -the file or folder to, including a new name if you wish to rename the remote -material. -If you leave this field blank, `scp` defaults to your home directory and the -name of the local material to be transferred. - -On Linux computers, `/` is the separator in file or directory paths. -A path starting with a `/` is called _absolute_, since there can be nothing -above the root `/`. -A path that does not start with `/` is called _relative_, since it is not -anchored to the root. - -If you want to upload a file to a location inside your home directory -- -which is often the case -- then you don't need a _leading_ `/`. After the `:`, -you can type the destination path relative to your home directory. -If your home directory _is_ the destination, you can leave the destination -field blank, or type `~` -- the shorthand for your home directory -- for -completeness. - -With `scp`, a trailing slash on the target directory is optional, and has -no effect. It is important for other commands, like `rsync`. - -> ## A Note on `rsync` -> -> As you gain experience with transferring files, you may find the `scp` -> command limiting. The [rsync] utility provides -> advanced features for file transfer and is typically faster compared to both -> `scp` and `sftp` (see below). It is especially useful for transferring large -> and/or many files and for synchronizing folder contents between computers. -> -> The syntax is similar to `scp`. To transfer _to_ another computer with -> commonly used options: -> -> ``` -> {{ site.local.prompt }} rsync -avP amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -> ``` -> {: .language-bash} -> -> The options are: -> -> * `-a` (**a**rchive) to preserve file timestamps, permissions, and folders, -> among other things; implies recursion -> * `-v` (**v**erbose) to get verbose output to help monitor the transfer -> * `-P` (partial/progress) to preserve partially transferred files in case -> of an interruption and also displays the progress of the transfer. -> -> To recursively copy a directory, we can use the same options: -> -> ``` -> {{ site.local.prompt }} rsync -avP amdahl {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> -> As written, this will place the local directory and its contents under your -> home directory on the remote system. If the trailing slash is omitted on -> the destination, a new directory corresponding to the transferred directory -> will not be created, and the contents of the source -> directory will be copied directly into the destination directory. -> -> To download a file, we simply change the source and destination: -> -> ``` -> {{ site.local.prompt }} rsync -avP {{ site.remote.user }}@{{ site.remote.login }}:amdahl ./ -> ``` -> {: .language-bash} -{: .callout} - -File transfers using both `scp` and `rsync` use SSH to encrypt data sent through -the network. So, if you can connect via SSH, you will be able to transfer -files. By default, SSH uses network port 22. If a custom SSH port is in use, -you will have to specify it using the appropriate flag, often `-p`, `-P`, or -`--port`. Check `--help` or the `man` page if you're unsure. - -> ## Change the Rsync Port -> -> Say we have to connect `rsync` through port 768 instead of 22. How would we -> modify this command? -> -> ``` -> {{ site.local.prompt }} rsync amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -> ``` -> {: .language-bash} -> -> _Hint:_ check the `man` page or "help" for `rsync`. -> -> > ## Solution -> > -> > ``` -> > {{ site.local.prompt }} man rsync -> > {{ site.local.prompt }} rsync --help | grep port -> > --port=PORT specify double-colon alternate port number -> > See http://rsync.samba.org/ for updates, bug reports, and answers -> > {{ site.local.prompt }} rsync --port=768 amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -> > ``` -> > {: .language-bash} -> > -> > (Note that this command will fail, as the correct port in this case is the -> > default: 22.) -> {: .solution} -{: .challenge} - -## Transferring Files Interactively with FileZilla - -FileZilla is a cross-platform client for downloading and uploading files to and -from a remote computer. It is absolutely fool-proof and always works quite -well. It uses the `sftp` protocol. You can read more about using the `sftp` -protocol in the command line in the -[lesson discussion]({{ site.baseurl }}{% link _extras/discuss.md %}). - -Download and install the FileZilla client from . -After installing and opening the program, you should end up with a window with -a file browser of your local system on the left hand side of the screen. When -you connect to the cluster, your cluster files will appear on the right hand -side. - -To connect to the cluster, we'll just need to enter our credentials at the top -of the screen: - -* Host: `sftp://{{ site.remote.login }}` -* User: Your cluster username -* Password: Your cluster password -* Port: (leave blank to use the default port) - -Hit "Quickconnect" to connect. You should see your remote files appear on the -right hand side of the screen. You can drag-and-drop files between the left -(local) and right (remote) sides of the screen to transfer files. - -Finally, if you need to move large files (typically larger than a gigabyte) -from one remote computer to another remote computer, SSH in to the computer -hosting the files and use `scp` or `rsync` to transfer over to the other. This -will be more efficient than using FileZilla (or related applications) that -would copy from the source to your local machine, then to the destination -machine. - -{% include links.md %} - -[git-swc]: https://swcarpentry.github.io/git-novice/ -[rsync]: https://rsync.samba.org/ diff --git a/_episodes/17-parallel.md b/_episodes/17-parallel.md deleted file mode 100644 index dc5635e2..00000000 --- a/_episodes/17-parallel.md +++ /dev/null @@ -1,470 +0,0 @@ ---- -title: "Running a parallel job" -teaching: 30 -exercises: 60 -questions: -- "How do we execute a task in parallel?" -- "What benefits arise from parallel execution?" -- "What are the limits of gains from execution in parallel?" -objectives: -- "Install a Python package using `pip`" -- "Prepare a job submission script for the parallel executable." -- "Launch jobs with parallel execution." -- "Record and summarize the timing and accuracy of jobs." -- "Describe the relationship between job parallelism and performance." -keypoints: -- "Parallel programming allows applications to take advantage of - parallel hardware." -- "The queuing system facilitates executing parallel tasks." -- "Performance improvements from parallel execution do not scale linearly." ---- - -We now have the tools we need to run a multi-processor job. This is a very -important aspect of HPC systems, as parallelism is one of the primary tools -we have to improve the performance of computational tasks. - -If you disconnected, log back in to the cluster. - -``` -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -``` -{: .language-bash} - -## Install the Amdahl Program - -With the Amdahl source code on the cluster, we can install it, which will -provide access to the `amdahl` executable. -Move into the extracted directory, then use the Package Installer for Python, -or `pip`, to install it in your ("user") home directory: - -``` -{{ site.remote.prompt }} cd amdahl -{{ site.remote.prompt }} python3 -m pip install --user . -``` -{: .language-bash} - -> ## Amdahl is Python Code -> -> The Amdahl program is written in Python, and installing or using it requires -> locating the `python3` executable on the login node. -> If it can't be found, try listing available modules using `module avail`, -> load the appropriate one, and try the command again. -{: .callout} - -### MPI for Python - -The Amdahl code has one dependency: __mpi4py__. -If it hasn't already been installed on the cluster, `pip` will attempt to -collect mpi4py from the Internet and install it for you. -If this fails due to a one-way firewall, you must retrieve mpi4py on your -local machine and upload it, just as we did for Amdahl. - -> ## Retrieve and Upload `mpi4py` -> -> If installing Amdahl failed because mpi4py could not be installed, -> retrieve the tarball from -> then `rsync` it to the cluster, extract, and install: -> -> ``` -> {{ site.local.prompt }} wget -O mpi4py.tar.gz https://github.com/mpi4py/mpi4py/releases/download/3.1.4/mpi4py-3.1.4.tar.gz -> {{ site.local.prompt }} scp mpi4py.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -> # or -> {{ site.local.prompt }} rsync -avP mpi4py.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: -> ``` -> {: .language-bash} -> -> ``` -> {{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -> {{ site.remote.prompt }} tar -xvzf mpi4py.tar.gz # extract the archive -> {{ site.remote.prompt }} mv mpi4py* mpi4py # rename the directory -> {{ site.remote.prompt }} cd mpi4py -> {{ site.remote.prompt }} python3 -m pip install --user . -> {{ site.remote.prompt }} cd ../amdahl -> {{ site.remote.prompt }} python3 -m pip install --user . -> ``` -> {: .language-bash} -{: .discussion} - -> ## If `pip` Raises a Warning... -> -> `pip` may warn that your user package binaries are not in your PATH. -> -> ``` -> WARNING: The script amdahl is installed in "${HOME}/.local/bin" which is -> not on PATH. Consider adding this directory to PATH or, if you prefer to -> suppress this warning, use --no-warn-script-location. -> ``` -> {: .warning} -> -> To check whether this warning is a problem, use `which` to search for the -> `amdahl` program: -> -> ``` -> {{ site.remote.prompt }} which amdahl -> ``` -> {: .language-bash} -> -> If the command returns no output, displaying a new prompt, it means the file -> `amdahl` has not been found. You must update the environment variable named -> `PATH` to include the missing folder. -> Edit your shell configuration file as follows, then log off the cluster and -> back on again so it takes effect. -> -> ``` -> {{ site.remote.prompt }} nano ~/.bashrc -> {{ site.remote.prompt }} tail ~/.bashrc -> ``` -> {: .language-bash} -> ``` -> export PATH=${PATH}:${HOME}/.local/bin -> ``` -> {: .output} -> -> After logging back in to {{ site.remote.login }}, `which` should be able to -> find `amdahl` without difficulties. -> If you had to load a Python module, load it again. -{: .discussion} - -## Help! - -Many command-line programs include a "help" message. Try it with `amdahl`: - -``` -{{ site.remote.prompt }} amdahl --help -``` -{: .language-bash} - -``` -usage: amdahl [-h] [-p [PARALLEL_PROPORTION]] [-w [WORK_SECONDS]] [-t] [-e] [-j [JITTER_PROPORTION]] - -optional arguments: - -h, --help show this help message and exit - -p [PARALLEL_PROPORTION], --parallel-proportion [PARALLEL_PROPORTION] - Parallel proportion: a float between 0 and 1 - -w [WORK_SECONDS], --work-seconds [WORK_SECONDS] - Total seconds of workload: an integer greater than 0 - -t, --terse Format output as a machine-readable object for easier analysis - -e, --exact Exactly match requested timing by disabling random jitter - -j [JITTER_PROPORTION], --jitter-proportion [JITTER_PROPORTION] - Random jitter: a float between -1 and +1 -``` -{: .output} - -This message doesn't tell us much about what the program _does_, but it does -tell us the important flags we might want to use when launching it. - -## Running the Job on a Compute Node - -Create a submission file, requesting one task on a single node, then launch it. - -``` -{{ site.remote.prompt }} nano serial-job.sh -{{ site.remote.prompt }} cat serial-job.sh -``` -{: .language-bash} - -{% include {{ site.snippets }}/parallel/one-task-jobscript.snip %} - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} serial-job.sh -``` -{: .language-bash} - -As before, use the {{ site.sched.name }} status commands to check whether your job -is running and when it ends: - -``` -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} -``` -{: .language-bash} - -Use `ls` to locate the output file. The `-t` flag sorts in -reverse-chronological order: newest first. What was the output? - -> ## Read the Job Output -> -> The cluster output should be written to a file in the folder you launched the -> job from. For example, -> -> ``` -> {{ site.remote.prompt }} ls -t -> ``` -> {: .language-bash} -> ``` -> slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt -> ``` -> {: .output} -> ``` -> {{ site.remote.prompt }} cat slurm-347087.out -> ``` -> {: .language-bash} -> ``` -> Doing 30.000 seconds of 'work' on 1 processor, -> which should take 30.000 seconds with 0.850 parallel proportion of the workload. -> -> Hello, World! I am process 0 of 1 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. -> Hello, World! I am process 0 of 1 on {{ site.remote.node }}. I will do parallel 'work' for 25.500 seconds. -> -> Total execution time (according to rank 0): 30.033 seconds -> ``` -> {: .output} -{: .solution} - -As we saw before, two of the `amdahl` program flags set the amount of work and -the proportion of that work that is parallel in nature. Based on the output, we -can see that the code uses a default of 30 seconds of work that is 85% -parallel. The program ran for just over 30 seconds in total, and if we run the -numbers, it is true that 15% of it was marked 'serial' and 85% was 'parallel'. - -Since we only gave the job one CPU, this job wasn't really parallel: the same -processor performed the 'serial' work for 4.5 seconds, then the 'parallel' part -for 25.5 seconds, and no time was saved. The cluster can do better, if we ask. - -## Running the Parallel Job - -The `amdahl` program uses the Message Passing Interface (MPI) for parallelism --- this is a common tool on HPC systems. - -> ## What is MPI? -> -> The Message Passing Interface is a set of tools which allow multiple tasks -> running simultaneously to communicate with each other. -> Typically, a single executable is run multiple times, possibly on different -> machines, and the MPI tools are used to inform each instance of the -> executable about its sibling processes, and which instance it is. -> MPI also provides tools to allow communication between instances to -> coordinate work, exchange information about elements of the task, or to -> transfer data. -> An MPI instance typically has its own copy of all the local variables. -{: .callout} - -While MPI-aware executables can generally be run as stand-alone programs, in -order for them to run in parallel they must use an MPI _run-time environment_, -which is a specific implementation of the MPI _standard_. -To activate the MPI environment, the program should be started via a command -such as `mpiexec` (or `mpirun`, or `srun`, etc. depending on the MPI run-time -you need to use), which will ensure that the appropriate run-time support for -parallelism is included. - -> ## MPI Runtime Arguments -> -> On their own, commands such as `mpiexec` can take many arguments specifying -> how many machines will participate in the execution, -> and you might need these if you would like to run an MPI program on your -> own (for example, on your laptop). -> In the context of a queuing system, however, it is frequently the case that -> MPI run-time will obtain the necessary parameters from the queuing system, -> by examining the environment variables set when the job is launched. -{: .callout} - -Let's modify the job script to request more cores and use the MPI run-time. - -```bash -{{ site.remote.prompt }} cp serial-job.sh parallel-job.sh -{{ site.remote.prompt }} nano parallel-job.sh -{{ site.remote.prompt }} cat parallel-job.sh -``` -{: .language-bash} - -{% include {{ site.snippets }}/parallel/four-tasks-jobscript.snip %} - -Then submit your job. Note that the submission command has not really changed -from how we submitted the serial job: all the parallel settings are in the -batch file rather than the command line. - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} parallel-job.sh -``` -{: .language-bash} - -As before, use the status commands to check when your job runs. - -``` -{{ site.remote.prompt }} ls -t -``` -{: .language-bash} -``` -slurm-347178.out parallel-job.sh slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt -``` -{: .output} -``` -{{ site.remote.prompt }} cat slurm-347178.out -``` -{: .language-bash} -``` -Doing 30.000 seconds of 'work' on 4 processors, -which should take 10.875 seconds with 0.850 parallel proportion of the workload. - - Hello, World! I am process 0 of 4 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. - Hello, World! I am process 2 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 1 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 3 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 0 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - -Total execution time (according to rank 0): 10.888 seconds -``` -{: .output} - -> ## Is it 4× faster? -> -> The parallel job received 4× more processors than the serial job: -> does that mean it finished in ¼ the time? -> -> > ## Solution -> > -> > The parallel job did take _less_ time: 11 seconds is better than 30! -> > But it is only a 2.7× improvement, not 4×. -> > -> > Look at the job output: -> > -> > * While "process 0" did serial work, processes 1 through 3 did their -> > parallel work. -> > * While process 0 caught up on its parallel work, -> > the rest did nothing at all. -> > -> > Process 0 always has to finish its serial task before it can start on the -> > parallel work. This sets a lower limit on the amount of time this job will -> > take, no matter how many cores you throw at it. -> > -> > This is the basic principle behind [Amdahl's Law][amdahl], which is one way -> > of predicting improvements in execution time for a __fixed__ workload that -> > can be subdivided and run in parallel to some extent. -> {: .solution} -{: .challenge} - -## How Much Does Parallel Execution Improve Performance? - -In theory, dividing up a perfectly parallel calculation among _n_ MPI processes -should produce a decrease in total run time by a factor of _n_. -As we have just seen, real programs need some time for the MPI processes to -communicate and coordinate, and some types of calculations can't be subdivided: -they only run effectively on a single CPU. - -Additionally, if the MPI processes operate on different physical CPUs in the -computer, or across multiple compute nodes, even more time is required for -communication than it takes when all processes operate on a single CPU. - -In practice, it's common to evaluate the parallelism of an MPI program by - -* running the program across a range of CPU counts, -* recording the execution time on each run, -* comparing each execution time to the time when using a single CPU. - -Since "more is better" -- improvement is easier to interpret from increases in -some quantity than decreases -- comparisons are made using the speedup factor -_S_, which is calculated as the single-CPU execution time divided by the multi-CPU -execution time. For a perfectly parallel program, a plot of the speedup _S_ -versus the number of CPUs _n_ would give a straight line, _S_ = _n_. - -Let's run one more job, so we can see how close to a straight line our `amdahl` -code gets. - -```bash -{{ site.remote.prompt }} nano parallel-job.sh -{{ site.remote.prompt }} cat parallel-job.sh -``` - -{% include {{ site.snippets }}/parallel/eight-tasks-jobscript.snip %} - -Then submit your job. Note that the submission command has not really changed -from how we submitted the serial job: all the parallel settings are in the -batch file rather than the command line. - -``` -{{ site.remote.prompt }} {{ site.sched.submit.name }} parallel-job.sh -``` -{: .language-bash} - -As before, use the status commands to check when your job runs. - -``` -{{ site.remote.prompt }} ls -t -``` -{: .language-bash} -``` -slurm-347271.out parallel-job.sh slurm-347178.out slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt -``` -{: .output} -``` -{{ site.remote.prompt }} cat slurm-347178.out -``` -{: .language-bash} -``` -which should take 7.688 seconds with 0.850 parallel proportion of the workload. - - Hello, World! I am process 4 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 0 of 8 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. - Hello, World! I am process 2 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 1 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 3 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 5 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 6 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 7 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 0 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - -Total execution time (according to rank 0): 7.697 seconds -``` -{: .output} - -> ## Non-Linear Output -> -> When we ran the job with 4 parallel workers, the serial job wrote its output -> first, then the parallel processes wrote their output, with process 0 coming -> in first and last. -> -> With 8 workers, this is not the case: since the parallel workers take less -> time than the serial work, it is hard to say which process will write its -> output first, except that it will _not_ be process 0! -{: .discussion} - -Now, let's summarize the amount of time it took each job to run: - -| Number of CPUs | Runtime (sec) | -| --- | --- | -| 1 | 30.033 | -| 4 | 10.888 | -| 8 | 7.697 | - -Then, use the first row to compute speedups _S_, using Python as a command-line calculator: - -``` -{{ site.remote.prompt }} for n in 30.033 10.888 7.697; do python3 -c "print(30.033 / $n)"; done -``` -{: .language-bash} - -| Number of CPUs | Speedup | Ideal | -| --- | --- | --- | -| 1 | 1.0 | 1 | -| 4 | 2.75 | 4 | -| 8 | 3.90 | 8 | - -The job output files have been telling us that this program is performing 85% -of its work in parallel, leaving 15% to run in serial. This seems reasonably -high, but our quick study of speedup shows that in order to get a 4× speedup, -we have to use 8 or 9 processors in parallel. In real programs, the speedup -factor is influenced by - -* CPU design -* communication network between compute nodes -* MPI library implementations -* details of the MPI program itself - -Using Amdahl's Law, you can prove that with this program, it is _impossible_ -to reach 8× speedup, no matter how many processors you have on hand. Details of -that analysis, with results to back it up, are left for the next class in the -HPC Carpentry workshop, _HPC Workflows_. - -In an HPC environment, we try to reduce the execution time for all types of -jobs, and MPI is an extremely common way to combine dozens, hundreds, or -thousands of CPUs into solving a single problem. To learn more about -parallelization, see the [parallel novice lesson][parallel-novice] lesson. - -{% include links.md %} - -[amdahl]: https://en.wikipedia.org/wiki/Amdahl's_law -[cmd-line]: https://swcarpentry.github.io/python-novice-inflammation/12-cmdline/index.html -[inflammation]: https://swcarpentry.github.io/python-novice-inflammation/ -[np-dtype]: https://numpy.org/doc/stable/reference/generated/numpy.dtype.html -[parallel-novice]: http://www.hpc-carpentry.org/hpc-parallel-novice/ -[python-func]: https://swcarpentry.github.io/python-novice-inflammation/08-func/index.html -[units]: https://en.wikipedia.org/wiki/Byte#Multiple-byte_units diff --git a/_episodes/18-resources.md b/_episodes/18-resources.md deleted file mode 100644 index ffa4ed9a..00000000 --- a/_episodes/18-resources.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: "Using resources effectively" -teaching: 10 -exercises: 20 -questions: -- "How can I review past jobs?" -- "How can I use this knowledge to create a more accurate submission script?" -objectives: -- "Look up job statistics." -- "Make more accurate resource requests in job scripts based on data describing past performance." -keypoints: -- "Accurate job scripts help the queuing system efficiently allocate - shared resources." ---- - -We've touched on all the skills you need to interact with an HPC cluster: -logging in over SSH, loading software modules, submitting parallel jobs, and -finding the output. Let's learn about estimating resource usage and why it -might matter. - -## Estimating Required Resources Using the Scheduler - -Although we covered requesting resources from the scheduler earlier with the -π code, how do we know what type of resources the software will need in -the first place, and its demand for each? In general, unless the software -documentation or user testimonials provide some idea, we won't know how much -memory or compute time a program will need. - -> ## Read the Documentation -> -> Most HPC facilities maintain documentation as a wiki, a website, or a -> document sent along when you register for an account. Take a look at these -> resources, and search for the software you plan to use: somebody might have -> written up guidance for getting the most out of it. -{: .callout} - -A convenient way of figuring out the resources required for a job to run -successfully is to submit a test job, and then ask the scheduler about its -impact using `{{ site.sched.hist }}`. You can use this knowledge to set up the -next job with a closer estimate of its load on the system. A good general rule -is to ask the scheduler for 20% to 30% more time and memory than you expect the -job to need. This ensures that minor fluctuations in run time or memory use -will not result in your job being cancelled by the scheduler. Keep in mind that -if you ask for too much, your job may not run even though enough resources are -available, because the scheduler will be waiting for other people's jobs to -finish and free up the resources needed to match what you asked for. - -## Stats - -Since we already submitted `amdahl` to run on the cluster, we can query the -scheduler to see how long our job took and what resources were used. We will -use `{{ site.sched.hist }}` to get statistics about `parallel-job.sh`. - -``` -{{ site.remote.prompt }} {{ site.sched.hist }} -``` -{: .language-bash} - -{% include {{ site.snippets }}/resources/account-history.snip %} - -This shows all the jobs we ran today (note that there are multiple entries per -job). -To get info about a specific job (for example, 347087), we change command -slightly. - -``` -{{ site.remote.prompt }} {{ site.sched.hist }} {{ site.sched.flag.histdetail }} 347087 -``` -{: .language-bash} - -It will show a lot of info; in fact, every single piece of info collected on -your job by the scheduler will show up here. It may be useful to redirect this -information to `less` to make it easier to view (use the left and right arrow -keys to scroll through fields). - -``` -{{ site.remote.prompt }} {{ site.sched.hist }} {{ site.sched.flag.histdetail }} 347087 | less -S -``` -{: .language-bash} - -> ## Discussion -> -> This view can help compare the amount of time requested and actually -> used, duration of residence in the queue before launching, and memory -> footprint on the compute node(s). -> -> How accurate were our estimates? -{: .discussion} - -## Improving Resource Requests - -From the job history, we see that `amdahl` jobs finished executing in -at most a few minutes, once dispatched. The time estimate we provided -in the job script was far too long! This makes it harder for the -queuing system to accurately estimate when resources will become free -for other jobs. Practically, this means that the queuing system waits -to dispatch our `amdahl` job until the full requested time slot opens, -instead of "sneaking it in" a much shorter window where the job could -actually finish. Specifying the expected runtime in the submission -script more accurately will help alleviate cluster congestion and may -get your job dispatched earlier. - -> ## Narrow the Time Estimate -> -> Edit `parallel_job.sh` to set a better time estimate. How close can -> you get? -> -> Hint: use `{{ site.sched.flag.time }}`. -> -> > ## Solution -> > -> > The following line tells {{ site.sched.name }} that our job should -> > finish within 2 minutes: -> > -> > ``` -> > {{ site.sched.comment }} {{ site.sched.flag.time }}{% if site.sched.name == "Slurm" %} {% else %}={% endif %}00:02:00 -> > ``` -> > {: .language-bash} -> {: .solution} -{: .challenge} - -{% include links.md %} diff --git a/_episodes/19-responsibility.md b/_episodes/19-responsibility.md deleted file mode 100644 index e6239e1d..00000000 --- a/_episodes/19-responsibility.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -title: "Using shared resources responsibly" -teaching: 15 -exercises: 5 -questions: -- "How can I be a responsible user?" -- "How can I protect my data?" -- "How can I best get large amounts of data off an HPC system?" -objectives: -- "Describe how the actions of a single user can affect the experience of others on a shared system." -- "Discuss the behaviour of a considerate shared system citizen." -- "Explain the importance of backing up critical data." -- "Describe the challenges with transferring large amounts of data off HPC systems." -- "Convert many files to a single archive file using tar." -keypoints: -- "Be careful how you use the login node." -- "Your data on the system is your responsibility." -- "Plan and test large data transfers." -- "It is often best to convert many files to a single archive file before - transferring." ---- - -One of the major differences between using remote HPC resources and your own -system (e.g. your laptop) is that remote resources are shared. How many users -the resource is shared between at any one time varies from system to system, but -it is unlikely you will ever be the only user logged into or using such a -system. - -The widespread usage of scheduling systems where users submit jobs on HPC -resources is a natural outcome of the shared nature of these resources. There -are other things you, as an upstanding member of the community, need to -consider. - -## Be Kind to the Login Nodes - -The login node is often busy managing all of the logged in users, creating and -editing files and compiling software. If the machine runs out of memory or -processing capacity, it will become very slow and unusable for everyone. While -the machine is meant to be used, be sure to do so responsibly -- in ways -that will not adversely impact other users' experience. - -Login nodes are always the right place to launch jobs. Cluster policies vary, -but they may also be used for proving out workflows, and in some cases, may -host advanced cluster-specific debugging or development tools. The cluster may -have modules that need to be loaded, possibly in a certain order, and paths or -library versions that differ from your laptop, and doing an interactive test -run on the head node is a quick and reliable way to discover and fix these -issues. - -> ## Login Nodes Are a Shared Resource -> -> Remember, the login node is shared with all other users and your actions -> could cause issues for other people. Think carefully about the potential -> implications of issuing commands that may use large amounts of resource. -> -> Unsure? Ask your friendly systems administrator ("sysadmin") if the thing -> you're contemplating is suitable for the login node, or if there's another -> mechanism to get it done safely. -{: .callout} - -You can always use the commands `top` and `ps ux` to list the processes that -are running on the login node along with the amount of CPU and memory they are -using. If this check reveals that the login node is somewhat idle, you can -safely use it for your non-routine processing task. If something goes wrong --- the process takes too long, or doesn't respond -- you can use the -`kill` command along with the _PID_ to terminate the process. - -> ## Login Node Etiquette -> -> Which of these commands would be a routine task to run on the login node? -> -> 1. `python physics_sim.py` -> 2. `make` -> 3. `create_directories.sh` -> 4. `molecular_dynamics_2` -> 5. `tar -xzf R-3.3.0.tar.gz` -> -> > ## Solution -> > -> > Building software, creating directories, and unpacking software are common -> > and acceptable > tasks for the login node: options #2 (`make`), #3 -> > (`mkdir`), and #5 (`tar`) are probably OK. Note that script names do not -> > always reflect their contents: before launching #3, please -> > `less create_directories.sh` and make sure it's not a Trojan horse. -> > -> > Running resource-intensive applications is frowned upon. Unless you are -> > sure it will not affect other users, do not run jobs like #1 (`python`) -> > or #4 (custom MD code). If you're unsure, ask your friendly sysadmin for -> > advice. -> {: .solution} -{: .challenge} - -If you experience performance issues with a login node you should report it to -the system staff (usually via the helpdesk) for them to investigate. - -## Test Before Scaling - -Remember that you are generally charged for usage on shared systems. A simple -mistake in a job script can end up costing a large amount of resource budget. -Imagine a job script with a mistake that makes it sit doing nothing for 24 -hours on 1000 cores or one where you have requested 2000 cores by mistake and -only use 100 of them! This problem can be compounded when people write scripts -that automate job submission (for example, when running the same calculation or -analysis over lots of different parameters or files). When this happens it -hurts both you (as you waste lots of charged resource) and other users (who are -blocked from accessing the idle compute nodes). On very busy resources you may -wait many days in a queue for your job to fail within 10 seconds of starting due -to a trivial typo in the job script. This is extremely frustrating! - -Most systems provide dedicated resources for testing that have short wait times -to help you avoid this issue. - -> ## Test Job Submission Scripts That Use Large Amounts of Resources -> -> Before submitting a large run of jobs, submit one as a test first to make -> sure everything works as expected. -> -> Before submitting a very large or very long job submit a short truncated test -> to ensure that the job starts as expected. -{: .callout} - -## Have a Backup Plan - -Although many HPC systems keep backups, it does not always cover all the file -systems available and may only be for disaster recovery purposes (i.e. for -restoring the whole file system if lost rather than an individual file or -directory you have deleted by mistake). Protecting critical data from -corruption or deletion is primarily your responsibility: keep your own backup -copies. - -Version control systems (such as Git) often have free, cloud-based offerings -(e.g., GitHub and GitLab) that are generally used for storing source code. Even -if you are not writing your own programs, these can be very useful for storing -job scripts, analysis scripts and small input files. - -If you are building software, you may have a large amount of source code -that you compile to build your executable. Since this data can generally -be recovered by re-downloading the code, or re-running the checkout -operation from the source code repository, this data is also less -critical to protect. - -For larger amounts of data, especially important results from your runs, -which may be irreplaceable, you should make sure you have a robust system in -place for taking copies of data off the HPC system wherever possible -to backed-up storage. Tools such as `rsync` can be very useful for this. - -Your access to the shared HPC system will generally be time-limited so you -should ensure you have a plan for transferring your data off the system before -your access finishes. The time required to transfer large amounts of data -should not be underestimated and you should ensure you have planned for this -early enough (ideally, before you even start using the system for your -research). - -In all these cases, the helpdesk of the system you are using should be able to -provide useful guidance on your options for data transfer for the volumes of -data you will be using. - -> ## Your Data Is Your Responsibility -> -> Make sure you understand what the backup policy is on the file systems on the -> system you are using and what implications this has for your work if you lose -> your data on the system. Plan your backups of critical data and how you will -> transfer data off the system throughout the project. -{: .callout} - -## Transferring Data - -As mentioned above, many users run into the challenge of transferring large -amounts of data off HPC systems at some point (this is more often in -transferring data off than onto systems but the advice below applies in either -case). Data transfer speed may be limited by many different factors so the best -data transfer mechanism to use depends on the type of data being transferred -and where the data is going. - -The components between your data's source and destination have -varying levels of performance, and in particular, may have -different capabilities with respect to __bandwidth__ and __latency__. - -__Bandwidth__ is generally the raw amount of data per unit time a -device is capable of transmitting or receiving. It's a common -and generally well-understood metric. - -__Latency__ is a bit more subtle. For data transfers, it may be thought -of as the amount of time it takes to get data out of storage and into -a transmittable form. Latency issues are the reason it's advisable -to execute data transfers by moving a small number of large -files, rather than the converse. - -Some of the key components and their associated issues are: - -* __Disk speed__: File systems on HPC systems are often highly parallel, - consisting of a very large number of high performance disk drives. This - allows them to support a very high data bandwidth. Unless the remote system - has a similar parallel file system you may find your transfer speed limited - by disk performance at that end. -* __Meta-data performance__: _Meta-data operations_ such as opening and closing - files or listing the owner or size of a file are much less parallel than - read/write operations. If your data consists of a very large number of small - files you may find your transfer speed is limited by meta-data operations. - Meta-data operations performed by other users of the system can also interact - strongly with those you perform so reducing the number of such operations you - use (by combining multiple files into a single file) may reduce variability - in your transfer rates and increase transfer speeds. -* __Network speed__: Data transfer performance can be limited by network speed. - More importantly it is limited by the slowest section of the network between - source and destination. If you are transferring to your laptop/workstation, - this is likely to be its connection (either via LAN or WiFi). -* __Firewall speed__: Most modern networks are protected by some form of - firewall that filters out malicious traffic. This filtering has some overhead - and can result in a reduction in data transfer performance. The needs of a - general purpose network that hosts email/web-servers and desktop machines are - quite different from a research network that needs to support high volume - data transfers. If you are trying to transfer data to or from a host on a - general purpose network you may find the firewall for that network will limit - the transfer rate you can achieve. - -As mentioned above, if you have related data that consists of a large number of -small files it is strongly recommended to pack the files into a larger -_archive_ file for long term storage and transfer. A single large file makes -more efficient use of the file system and is easier to move, copy and transfer -because significantly fewer metadata operations are required. Archive files can -be created using tools like `tar` and `zip`. We have already met `tar` when we -talked about data transfer earlier. - -{% include figure.html url="" max-width="90%" - file="/fig/responsibility-bandwidth.svg" - alt="Schematic of network bandwidth" - caption="Schematic diagram of bandwidth and latency for disk and network - I/O. Each of the components on the figure is connected by a blue line of - width proportional to the interface bandwidth. The small mazes at the - link points illustrate the latency of the link, with more tortuous - mazes indicating higher latency." %} - -> ## Consider the Best Way to Transfer Data -> -> If you are transferring large amounts of data you will need to think about -> what may affect your transfer performance. It is always useful to run some -> tests that you can use to extrapolate how long it will take to transfer your -> data. -> -> Say you have a "data" folder containing 10,000 or so files, a healthy mix of -> small and large ASCII and binary data. Which of the following would be the -> best way to transfer them to {{ site.remote.name }}? -> -> 1. ``` -> {{ site.local.prompt }} scp -r data {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> 2. ``` -> {{ site.local.prompt }} rsync -ra data {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> 3. ``` -> {{ site.local.prompt }} rsync -raz data {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> 4. ``` -> {{ site.local.prompt }} tar -cvf data.tar data -> {{ site.local.prompt }} rsync -raz data.tar {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> 5. ``` -> {{ site.local.prompt }} tar -cvzf data.tar.gz data -> {{ site.local.prompt }} rsync -ra data.tar.gz {{ site.remote.user }}@{{ site.remote.login }}:~/ -> ``` -> {: .language-bash} -> -> > ## Solution -> > -> > 1. `scp` will recursively copy the directory. This works, but without -> > compression. -> > 2. `rsync -ra` works like `scp -r`, but preserves file information like -> > creation times. This is marginally better. -> > 3. `rsync -raz` adds compression, which will save some bandwidth. If you -> > have a strong CPU at both ends of the line, and you're on a slow -> > network, this is a good choice. -> > 4. This command first uses `tar` to merge everything into a single file, -> > then `rsync -z` to transfer it with compression. With this large -> > _number_ of files, metadata overhead can hamper your transfer, so this -> > is a good idea. -> > 5. This command uses `tar -z` to compress the archive, then `rsync` to -> > transfer it. This may perform similarly to #4, but in most cases (for -> > large datasets), it's the best combination of high throughput and low -> > latency (making the most of your time and network connection). -> {: .solution} -{: .challenge} diff --git a/_extras/discuss.md b/_extras/.discuss.md similarity index 100% rename from _extras/discuss.md rename to _extras/.discuss.md diff --git a/_extras/learner-prerequisites.md b/_extras/.learner-prerequisites.md similarity index 100% rename from _extras/learner-prerequisites.md rename to _extras/.learner-prerequisites.md diff --git a/_extras/learner-profiles.md b/_extras/.learner-profiles.md similarity index 100% rename from _extras/learner-profiles.md rename to _extras/.learner-profiles.md diff --git a/_extras/command-history.md b/_extras/command-history.md new file mode 100644 index 00000000..70862104 --- /dev/null +++ b/_extras/command-history.md @@ -0,0 +1,61 @@ +--- +layout: page +title: "Command History" +permalink: /command-history/ +--- + +{% include base_path.html %} + + + +{% for episode in site.episodes %} +
+{% endfor %} + +{% include links.md %} diff --git a/_extras/guide.md b/_extras/guide.md index 1f952e9c..ab136086 100644 --- a/_extras/guide.md +++ b/_extras/guide.md @@ -10,11 +10,11 @@ The following list captures terms that need to be added to this glossary. This is a great way to contribute. {:auto_ids} -[Accelerator](https://en.wikipedia.org/wiki/Hardware_acceleration) + [Central processing unit](https://en.wikipedia.org/wiki/CPU) : or simply *processor* is the (hardware) component of a computer that @@ -24,8 +24,8 @@ is a great way to contribute. **core** is a separate physical implementation of the electronic circuitry required to execute instructions. -[Cloud computing](https://en.wikipedia.org/wiki/Cloud_computing) -: *to be defined* +[Cloud (computing)](https://en.wikipedia.org/wiki/Cloud_computing) +: Virtualised remote computing resources that can be provisioned and deprovisioned as needed. [Cluster](https://en.wikipedia.org/wiki/Computer_cluster) : a collection of computers configured to enable collaboration on a common @@ -33,7 +33,7 @@ is a great way to contribute. and software (e.g. workload management). [Distributed memory](https://en.wikipedia.org/wiki/Distributed_memory) -: *to be defined* +: When multiple tasks are used. [Grid computing](https://en.wikipedia.org/wiki/Grid_computing) : *to be defined* @@ -60,7 +60,7 @@ is a great way to contribute. : *to be defined* [Shared memory](https://en.wikipedia.org/wiki/Shared_memory) -: *to be defined* +: Using multiple CPUs per task. [Slurm](https://en.wikipedia.org/wiki/Slurm_Workload_Manager) : *to be defined* @@ -76,3 +76,13 @@ is a great way to contribute. [Parallel File System](https://en.wikipedia.org/wiki/Distributed_file_systems) : *to be defined* + +[Bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) +: (Bourne Again Shell), is the default shell for many Linux distributions. + It is the primary tool we use for batch processing. + +[Shell](https://en.wikipedia.org/wiki/Shell_(computing)) +: Computer program that exposes an operating system's services to a human user or other programs. + +[Batch Programming](https://en.wikipedia.org/wiki/Batch_processing) +: Automating running of more than one command, usually through the use of a batch script (not to be confused with a Bash script). \ No newline at end of file diff --git a/_extras/jupyterhub.md b/_extras/jupyterhub.md new file mode 100644 index 00000000..330304b4 --- /dev/null +++ b/_extras/jupyterhub.md @@ -0,0 +1,67 @@ +--- +layout: page +title: "JupyterHub" +permalink: /jupyter/ +--- + +## NeSI JupyterHub Login + +The easiest method for accessing the NeSI cluster is to use our JupyterHub service. Below are the +login and troubleshooting instructions for NeSI JupyterHub: + +1. Follow this link: [https://jupyter.nesi.org.nz](https://jupyter.nesi.org.nz) +2. Enter your NeSI username, HPC password your 6 digit second factor token ![Login](/fig/Login_jupyterhubNeSI.png) +3. Choose server options: the session project code should be *NeSI Training ({{site.sched.project}})*, Number of CPUs and memory size will remain unchanged. However, select the appropriate **Wall time** based on the projected length of a session ![Options](/fig/ServerOptions_jupyterhubNeSI.png) +4. From Jupyter Launcher screen, choose Terminal (highlighted in red box) ![Terminal](/fig/jupyterLauncher.png) + +
+ + + +## Plugins + +### Rstudio + +### MATLAB + +### Virtual Desktop + +### Queue Manager + +## File transfer window + +![Terminal](/fig/jupyter_ftw.png) + +By default the file browser will be in a temporary hidden directory containing links to all of you projects and your home directory. Do-not create files here as they will be difficult to find later! + +Note, the file browser is not linked at all to your terminal, changing your directory in one will not affect the other. + +Files can be uploaded to the cluster by dragging from your local machine into the file transfer window. + +## Text Editor + +ctrl + s to save. \ No newline at end of file diff --git a/_includes/example_scripts/array_sum.py b/_includes/example_scripts/array_sum.py new file mode 100644 index 00000000..63454b00 --- /dev/null +++ b/_includes/example_scripts/array_sum.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from multiprocessing import Pool +import numpy as np +import os + + +def do_par(z): + p_complete = z * 100 / size_x + if p_complete % 1 == 0: + print("{0:.0f}% done...".format(p_complete)) + return sum(rng.random(size_y)) + + +num_cpus = int(os.getenv("SLURM_CPUS_PER_TASK", "1")) +size_x = 60000 # This on makes memorier +size_y = 40000 # This one to make longer + +# Time = (size_x/n) * size_y + c +# Mem = (size_x * n) * c1 + size_y * c2 + +print("Using {0} cpus to sum [ {1:e} x {2:e} ] matrix.".format(num_cpus, size_x, size_y)) + +rng = np.random.default_rng(seed=int(os.getenv('SLURM_ARRAY_TASK_ID', "0"))) + +with Pool(processes=num_cpus) as pool: + + results = pool.map(do_par, range(size_x)) + +print("Sum is '{0:f}'.".format(sum(results))) \ No newline at end of file diff --git a/_includes/example_scripts/array_sum.r b/_includes/example_scripts/array_sum.r new file mode 100644 index 00000000..2c28a6ee --- /dev/null +++ b/_includes/example_scripts/array_sum.r @@ -0,0 +1,19 @@ +#!/usr/bin/env {{ site.example.shell }} + +library(doParallel) + +num_cpus <- 2 # Use this value to request num cpus. +size_array <- 20000 + +registerDoParallel(num_cpus) + +sprintf("Using %i cpus to sum [ %e x %e ] matrix.",num_cpus,size_array,size_array) + +results <- foreach(z=0:size_array) %dopar% { + percent_complete= z*100/size_array + if (percent_complete%%1==0){ + cat(sprintf(" %i%% done...\r", percent_complete)) + } + sum(rnorm(size_array)) +} +sprintf("Sums to %f", Reduce("+",results)) diff --git a/_includes/example_scripts/array_sum2.r b/_includes/example_scripts/array_sum2.r new file mode 100644 index 00000000..932408c5 --- /dev/null +++ b/_includes/example_scripts/array_sum2.r @@ -0,0 +1,28 @@ +#!/usr/bin/env {{ site.example.shell }} + +library(doParallel) + +num_cpus <- strtoi(Sys.getenv('SLURM_CPUS_PER_TASK', unset = "1")) +size_x <-60000 # This on makes memorier +size_y <-20000 # This one to make longer + +# Time = (size_x/n) * size_y + c +# Mem = (size_x * n) * c1 + size_y * c2 + +print_progress <- interactive() # Whether to print progress or not. + +seed <- strtoi(Sys.getenv('SLURM_ARRAY_TASK_ID', unset = "0")) +set.seed(seed) + +registerDoParallel(num_cpus) + +sprintf("Using %i cpus to sum [ %e x %e ] matrix.",num_cpus,size_x,size_y) + +results <- foreach(z=0:size_x) %dopar% { + p_complete= z*100/size_x + if (print_progress && percent_complete%%1==0){ + cat(sprintf(" %i%% done...\r", percent_complete)) + } + sum(rnorm(size_y)) +} +sprintf("Seed '%s' sums to %f", seed, Reduce("+",results)) diff --git a/_includes/example_scripts/example-job.sh b/_includes/example_scripts/example-job.sh new file mode 100644 index 00000000..c93597b2 --- /dev/null +++ b/_includes/example_scripts/example-job.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +module purge +module load {{ site.example.module }} +{{ site.example.shell }} {{ site.example.script }} +echo "Done!" \ No newline at end of file diff --git a/_includes/example_scripts/example-job.sl.1 b/_includes/example_scripts/example-job.sl.1 new file mode 100644 index 00000000..b4b25c3b --- /dev/null +++ b/_includes/example_scripts/example-job.sl.1 @@ -0,0 +1,11 @@ +#!/bin/bash -e + +#SBATCH --job-name my_job +#SBATCH --account nesi99991 +#SBATCH --mem 300M +#SBATCH --time 00:15:00 + +module purge +module load {{ site.example.module }} +{{ site.example.shell }} {{ site.example.script }} +echo "Done!" diff --git a/_includes/example_scripts/example-job.sl.2 b/_includes/example_scripts/example-job.sl.2 new file mode 100644 index 00000000..7c4280b8 --- /dev/null +++ b/_includes/example_scripts/example-job.sl.2 @@ -0,0 +1,12 @@ +#!/bin/bash -e + +#SBATCH --job-name my_job +#SBATCH --account nesi99991 +#SBATCH --mem 300M +#SBATCH --time 00:15:00 +#SBATCH --cpus-per-task 4 + +module purge +module load R/4.3.1-gimkl-2022a +{{ site.example.shell }} {{ site.example.script }} +echo "Done!" diff --git a/_includes/example_scripts/example-job.sl.3 b/_includes/example_scripts/example-job.sl.3 new file mode 100644 index 00000000..04a3121a --- /dev/null +++ b/_includes/example_scripts/example-job.sl.3 @@ -0,0 +1,12 @@ +#!/bin/bash -e + +#SBATCH --job-name my_job +#SBATCH --account nesi99991 +#SBATCH --mem 600M +#SBATCH --time 00:10:00 +#SBATCH --cpus-per-task 4 + +module purge +module load R/4.3.1-gimkl-2022a +{{ site.example.shell }} {{ site.example.script }} +echo "Done!" \ No newline at end of file diff --git a/_includes/example_scripts/example_gpu.sl b/_includes/example_scripts/example_gpu.sl new file mode 100644 index 00000000..8f4f9974 --- /dev/null +++ b/_includes/example_scripts/example_gpu.sl @@ -0,0 +1,10 @@ +#!/bin/bash + +#SBATCH --job-name gpu +#SBATCH --account nesi99991 +#SBATCH --output %x_%a.out +#SBATCH --mem-per-cpu 2G +#SBATCH --gpu-per-node P100:1 + +module load CUDA +nvidia-smi \ No newline at end of file diff --git a/_includes/example_scripts/example_hybrid.sl b/_includes/example_scripts/example_hybrid.sl new file mode 100644 index 00000000..86d6a879 --- /dev/null +++ b/_includes/example_scripts/example_hybrid.sl @@ -0,0 +1,10 @@ +#!/bin/bash -e + +#SBATCH --job-name hybrid_job +#SBATCH --account nesi99991 +#SBATCH --output %x.out +#SBATCH --mem-per-cpu 500 +#SBATCH --ntasks 2 +#SBATCH --cpus-per-task 4 + +srun echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" diff --git a/_includes/example_scripts/example_jobarray.sl b/_includes/example_scripts/example_jobarray.sl new file mode 100644 index 00000000..6c4cde8b --- /dev/null +++ b/_includes/example_scripts/example_jobarray.sl @@ -0,0 +1,9 @@ +#!/bin/bash -e + +#SBATCH --job-name job_array +#SBATCH --account nesi99991 +#SBATCH --output %x_%a.out +#SBATCH --mem-per-cpu 500 +#SBATCH --array 0-3 + +srun echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" diff --git a/_includes/example_scripts/example_mpi.sl b/_includes/example_scripts/example_mpi.sl new file mode 100644 index 00000000..327c559a --- /dev/null +++ b/_includes/example_scripts/example_mpi.sl @@ -0,0 +1,8 @@ +#!/bin/bash -e + +#SBATCH --job-name mpi_job +#SBATCH --output %x.out +#SBATCH --mem-per-cpu 500 +#SBATCH --ntasks 4 + +srun bash whothis.sh diff --git a/_includes/example_scripts/example_smp.sl b/_includes/example_scripts/example_smp.sl new file mode 100644 index 00000000..5a1f9b87 --- /dev/null +++ b/_includes/example_scripts/example_smp.sl @@ -0,0 +1,9 @@ +#!/bin/bash -e + +#SBATCH --job-name smp_job +#SBATCH --account nesi99991 +#SBATCH --output %x.out +#SBATCH --mem-per-cpu 500 +#SBATCH --cpus-per-task 8 + +echo "I am task #${SLURM_PROCID} running on node '$(hostname)' with $(nproc) CPUs" diff --git a/_includes/example_scripts/shared-mem-job.sl b/_includes/example_scripts/shared-mem-job.sl new file mode 100644 index 00000000..9d144429 --- /dev/null +++ b/_includes/example_scripts/shared-mem-job.sl @@ -0,0 +1,10 @@ +#!/bin/bash -e + +#SBATCH --job-name my_job +#SBATCH --mem 300M +#SBATCH --time 00:15:00 +#SBATCH --cpus-per-task 4 + +module load R/4.3.1-gimkl-2022a +{{ site.example.shell }} {{ site.example.script }} +echo "Done!" diff --git a/_includes/late_start.sh b/_includes/late_start.sh new file mode 100644 index 00000000..ca1e279b --- /dev/null +++ b/_includes/late_start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Script for attendees joining after the bash intro. + +EXAMPLE_SCRIPT="{{ site.example.script }} " + +mkdir -vp "${USER}" +cp -v "${EXAMPLE_SCRIPT}" "${USER}" + +# May have do do more at some point. +echo "Done!" diff --git a/_includes/manual_episode_order.html b/_includes/manual_episode_order.html new file mode 100644 index 00000000..ffdff9a4 --- /dev/null +++ b/_includes/manual_episode_order.html @@ -0,0 +1,129 @@ +{% comment %} + This file enables manual episode ordering until + GitHub Pages switches to Jekyll that supports it + without any major hackery. Note, some logic will + be required even when this transition happens + but it will not be as involved as what we have to do + in this file. + + To order lesson episodes or extras manually + (instead of the default alpha-numerical order), + create array variables 'episode_order' and + 'extras_order' in `_config.yml` like so: + + episode_order: + - episodeA + - episodeB + + extras_order: + - extraA + - extraB + + Note that "Reference" page is currently always + added to "Extras" as the first item. + + The main outcomes of the code in this file are: + - 'lesson_episodes' variable that replaces + 'site.episodes' variable when manual episode + order is defined. + - 'lesson_extras' variable that replaces + 'site.extras' variable when manual ordering of + files in '_extras' is used + - 'previous_episode' and 'next_episode' objects + that replace 'page.previous' and 'page.next' variables, + correspondingly, and that have such properties + as 'url' and 'title' and that are used in + 'episode_navbar.html'. + + When episode order is specified manually, the 'lesson_episodes' + variable contains a list of episode names ("slugs", to be precise; + "slug" is the episode name without '.md'). Therefore, when we + iterate over 'lesson_episodes' (in navbar.html) , + we have to check whether we use manual episode ordering and, if so, + find the corresponding episode object. This is what we do with the + following code in every loop over 'lesson_episodes': + + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} +{% endcomment %} + +{% comment %} +Manual ordering of Episodes begins here +{% endcomment %} + +{% if site.episode_order %} + {% assign lesson_episodes = site.episode_order %} +{% else %} + {% assign lesson_episodes = site.episodes %} +{% endif %} + + +{% comment %} + If 'episode_order' is defined, we need to determine + - previous episode object ('previous_episode') + - and next episode object ('next_episode') +{% endcomment %} + + +{% if site.episode_order %} + {% for lesson_episode in lesson_episodes %} + + {% comment %} + We iterate over the specified lesson episodes using + a 'for' loop because we can use + 'forloop.first', 'forloop.last', and 'forloop.index0'. + {% endcomment %} + + {% unless lesson_episode == page.slug %} {% continue %} {% endunless %} + + {% if forloop.first %} + {% assign previous_episode = nil %} + {% else %} + {% assign p_idx = forloop.index0 | minus: 1 %} + {% assign p_name = lesson_episodes[p_idx] %} + {% assign previous_episode = site.episodes | where: "slug", p_name | first %} + {% comment %} + Pretty gross, but what can ya do. + {% endcomment %} + {% if previous_episode.break %} + {% assign p_idx = forloop.index0 | minus: 2 %} + {% assign p_name = lesson_episodes[p_idx] %} + {% assign previous_episode = site.episodes | where: "slug", p_name | first %} + {% endif %} + {% endif %} + + {% if forloop.last == true %} + {% assign next_episode = nil %} + {% else %} + {% assign n_idx = forloop.index0 | plus: 1 %} + {% assign n_name = lesson_episodes[n_idx] %} + {% assign next_episode = site.episodes | where: "slug", n_name | first %} + {% if next_episode.break %} + {% assign n_idx = forloop.index0 | plus: 2 %} + {% assign n_name = lesson_episodes[n_idx] %} + {% assign next_episode = site.episodes | where: "slug", n_name | first %} + {% endif %} + {% endif %} + {% endfor %} +{% else %} + {% assign previous_episode = page.previous %} + {% assign next_episode = page.next %} +{% endif %} + + +{% comment %} +Manual ordering of Extras begins here +{% endcomment %} + +{% if site.extras_order %} + {% assign lesson_extras = site.extras_order %} +{% else %} + {% assign lesson_extras = site.extras %} +{% endif %} + +{% comment %} + We do not need to determine "previous" or "next" extra. +{% endcomment %} diff --git a/_includes/navbar.html b/_includes/navbar.html new file mode 100644 index 00000000..c32b34ff --- /dev/null +++ b/_includes/navbar.html @@ -0,0 +1,126 @@ +{% comment %} +Lesson navigation bar. +{% endcomment %} + +{% include gh_variables.html %} +{% include manual_episode_order.html %} + + \ No newline at end of file diff --git a/_includes/snippets_library/NIST_CTCMS_slurm/_config_options.yml b/_includes/snippets_library/NIST_CTCMS_slurm/_config_options.yml index 0533c725..3c35dab5 100644 --- a/_includes/snippets_library/NIST_CTCMS_slurm/_config_options.yml +++ b/_includes/snippets_library/NIST_CTCMS_slurm/_config_options.yml @@ -46,7 +46,7 @@ sched: flag: user: "-u yourUsername" interactive: "" - histdetail: "--format=JobName,Submit,Start,State,ReqCPUS,Reserved,Elapsed,MaxRSS -j" + histdetail: "--format=JobID,JobName,Submit,Start,State,ReqCPUS,Reserved,Elapsed,MaxRSS" name: "-J" time: "-t" queue: "-p" diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/_config_options.yml b/_includes/snippets_library/NeSI_Mahuika_slurm/_config_options.yml new file mode 100644 index 00000000..5254c129 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/_config_options.yml @@ -0,0 +1,175 @@ +#------------------------------------------------------------ +# NeSI: Mahuika + SLURM +#------------------------------------------------------------ +# If your cluster is not represented in the library, please +# copy an existing folder, rename it, and customize for your +# installation. Remember to keep the leading slash on the +# `snippets` variable below! + +snippets: "/snippets_library/NeSI_Mahuika_slurm" + +local: + prompt: "[user@laptop ~]$" + bash_shebang: "#!/usr/bin/env bash" + +remote: + name: "mahuika" + login: "nesi.org.nz" + host: "mahuika02" + node: "mahuika02" + location: "Greta Point Datacenter" + homedir: "/home" + user: "yourUsername" + prompt: "[yourUsername@mahuika ~]$" + bash_shebang: "#!/bin/bash -e" + +sched: + name: "Slurm" + submit: + name: "sbatch" + options: "" + queue: + debug: "debug" + status: "squeue" + flag: + user: "-u yourUsername" + me: "--me" + interactive: "" + histdetail: "-l -j" + name: "--job-name" + time: "--time" + queue: "--partition" + del: "scancel" + interactive: "srun" + info: "sinfo" + comment: "#SBATCH" + hist: "sacct" + efficiency: "nn_seff" + projectcode: "nesi99991" + +episode_order: + - 01-cluster + - 02-filedir + - 03-break1 + - 04-modules + - 05-scheduler + - 06-lunch + - 064-parallel + - 07-resources + - 08-break2 + - 09-scaling + - 095-writing-good-code + + +#------------------------------------------------------------ +# Values for this lesson +#------------------------------------------------------------ + +# Which carpentry is this? +# "swc": Software Carpentry +# "dc": Data Carpentry +# "lc": Library Carpentry +# "cp": Carpentries (e.g., instructor traning) +carpentry: "incubator" + +# Overall title for pages. +title: "Introduction to High-Performance Computing" + +# Email address, no mailto: +email: "support@nesi.org.nz" + +# Life cycle stage of the lesson ("pre-alpha", "alpha", "beta", "stable")? +life_cycle: "stable" + +#------------------------------------------------------------ +# Generic settings (should not need to change). +#------------------------------------------------------------ + +# What kind of thing is this ("workshop" or "lesson")? +kind: "lesson" + +# Workshop working directory. +working_dir: + - /nesi/nobackup/nesi99991 + - ASP_winterschool2023 + +# Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). +# 600 is 10am +start_time: 600 +# Start time for people wanting to skip bash content +hpc_start_time: 765 +# Lesson to start at. +hpc_start_lesson: 01-cluster +# Script to run if skipping bash lessons. Should create neccicery files to get started. +hpc_start_script: late_start.sh + +# Magic to make URLs resolve both locally and on GitHub. Please don't change. +repository: / + +# Invoke the Carpentries theme rather than copying into our repository. +remote_theme: carpentries/carpentries-theme + +# Sites. +amy_site: "https://amy.carpentries.org/" +carpentries_github: "https://github.com/carpentries" +carpentries_pages: "https://carpentries.github.io" +carpentries_site: "https://carpentries.org/" +dc_site: "https://datacarpentry.org" +example_repo: "https://github.com/carpentries/lesson-example" +example_site: "https://carpentries.github.io/lesson-example" +lc_site: "https://librarycarpentry.org/" +swc_github: "https://github.com/swcarpentry" +swc_pages: "https://swcarpentry.github.io" +swc_site: "https://software-carpentry.org" +template_repo: "https://github.com/carpentries/styles" +training_site: "https://carpentries.github.io/instructor-training" +workshop_repo: "https://github.com/carpentries/workshop-template" +workshop_site: "https://carpentries.github.io/workshop-template" +cc_by_human: "https://creativecommons.org/licenses/by/4.0/" +support_docs: "https://support.nesi.org.nz/hc/en-gb" +exercise: "https://docs.google.com/spreadsheets/d/1D5PnhE6iJOB3ZKkDCiBHnk5CNZlhmj_gS-IXKGkkVoI/edit?usp=sharing" + + + +# Specify that things in the Episodes and Extras collections should be output. +collections: + episodes: + output: true + permalink: /:path/index.html + extras: + output: true + permalink: /:path/index.html + +# Set the default layout for things in the episodes collection. +defaults: + - values: + root: . + layout: page + - scope: + path: "" + type: episodes + values: + root: .. + layout: episode + - scope: + path: "" + type: extras + values: + root: .. + layout: page + +# Files and directories that are not to be copied. +exclude: + - Makefile + - bin/ + - .Rproj.user/ + - .vendor/ + - vendor/ + - .docker-vendor/ + +# Turn on built-in syntax highlighting. +highlighter: rouge + +plugins: + - jekyll-redirect-from + - jekyll-paginate diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/queue-info.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/queue-info.snip new file mode 100644 index 00000000..5610d2c8 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/queue-info.snip @@ -0,0 +1,14 @@ +``` +PARTITION AVAIL TIMELIMIT S:C:T MEMORY NODES(A/I) NODELIST +maintenance up 3-00:00:00 2+:1+:1+ 108000+ 244/3 vgpuwbg005,wbg[001-004],wbh001,wbl[001-005,008-011],wbn[003-006,009-064,066-120,122-176,178-236,238],wch001,wcl[001-002] +interactive up 3-00:00:00 2:18:2 108000 2/2 wbn[003-006] +large up 3-00:00:00 2:18:2 108000 226/0 wbn[009-064,066-120,122-176,178-236,238] +long up 21-00:00:0 2:18:2 108000 81/0 wbn[011-062,067-092,235-236,238] +gpu up 3-00:00:00 2:18:2 474000 4/0 wbg[001-004] +infill up 3-00:00:00 2:18:2 474000 4/0 wbg[001-004] +igpu up 3-00:00:00 8:1:1 108000 0/1 vgpuwbg005 +prepost up 3:00:00 2:18:2 474000 8/0 wbl[001-005,008-011] +bigmem up 7-00:00:00 2:18:2 474000 8/0 wbl[001-005,008-011] +hugemem up 7-00:00:00 2+:16+:2 1500000+ 4/0 wbh001,wch001,wcl[001-002] +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/specific-node-info.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/specific-node-info.snip new file mode 100644 index 00000000..6c08e9c2 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/cluster/specific-node-info.snip @@ -0,0 +1,10 @@ +> ## Explore a Worker Node +> +> Finally, let's look at the resources available on the worker nodes where your +> jobs will actually run:: +> +> ``` +> {{ site.remote.prompt }} sinfo +> ``` +> {: .language-bash} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.python.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.python.snip new file mode 100644 index 00000000..584119f4 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.python.snip @@ -0,0 +1,4 @@ +``` +{% include example_scripts/{{ site.example.script }} %} +``` +{: .language-python} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.r.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.r.snip new file mode 100644 index 00000000..ccab3502 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/code/include.r.snip @@ -0,0 +1,4 @@ +``` +{% include example_scripts/{{ site.example.script }} %} +``` +{: .language-r} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents1.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents1.snip new file mode 100644 index 00000000..4eaa3be8 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents1.snip @@ -0,0 +1,4 @@ +``` + {{ site.example.script }} usr123 usr345 +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents2.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents2.snip new file mode 100644 index 00000000..fd07a73c --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents2.snip @@ -0,0 +1,8 @@ +``` +-rw-r-----+ 1 usr9999 {{site.sched.projectcode}} 460 Nov 18 17:03 +-rw-r-----+ 1 usr9999 {{site.sched.projectcode}} 460 Nov 18 17:03 {{ site.example.script }} +drwxr-sr-x+ 3 usr9999 {{site.sched.projectcode}} 4096 22 Sep 08:40 birds +drwxrws---+ 2 usr123 {{site.sched.projectcode}} 4096 Nov 15 09:01 usr123 +drwxrws---+ 2 usr345 {{site.sched.projectcode}} 4096 Nov 15 09:01 usr345 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents3.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents3.snip new file mode 100644 index 00000000..f1df34db --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/dir-contents3.snip @@ -0,0 +1,8 @@ +``` +drwxrws---+ 4 usr001 {{site.sched.projectcode}} 4096 Nov 15 09:00 . +drwxrws---+ 12 root {{site.sched.projectcode}} 262144 Nov 15 09:23 .. +-rw-r-----+ 1 cwal219 {{site.sched.projectcode}} 460 Nov 18 17:03 {{ site.example.script }} +drwxrws---+ 2 usr123 {{site.sched.projectcode}} 4096 Nov 15 09:01 usr123 +drwxrws---+ 2 usr345 {{site.sched.projectcode}} 4096 Nov 15 09:01 usr345 +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/sinfo.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/sinfo.snip new file mode 100644 index 00000000..98166f19 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/sinfo.snip @@ -0,0 +1,9 @@ + +``` +$ nn_storage_quota +Quota Location Available Used Use% State Inodes IUsed IUse% IState +home_johndoe 20G 14.51G 72.57% OK 1000000 112084 11.21% OK +project_{{site.sched.projectcode}} 100G 101G 101.00% LOCKED 100000 194 0.19% OK +nobackup_{{site.sched.projectcode}} 10T 0 0.00% OK 1000000 14 0.00% OK +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/working-dir.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/working-dir.snip new file mode 100644 index 00000000..d0d7d27d --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/filedir/working-dir.snip @@ -0,0 +1,4 @@ +``` +{{ site.example.script }} ernz2021 introhpc202209 ml101_20210928 ml101_20220811 ml102_tensorflow_dataset slurm-31060451.out submit.sh +Bowtie2 example-job.sl ml101_20210329``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/available-modules.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/available-modules.snip new file mode 100644 index 00000000..cb3a6241 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/available-modules.snip @@ -0,0 +1,41 @@ +``` +-----------------/opt/nesi/CS400_centos7_bdw/modules/all ------------------ + Flye/2.9-gimkl-2020a-Python-3.8.2 (D) PyQt/5.10.1-gimkl-2018b-Python-3.7.3 + fmlrc/1.0.0-GCC-9.2.0 PyQt/5.12.1-gimkl-2018b-Python-2.7.16 + fmt/7.1.3-GCCcore-9.2.0 PyQt/5.12.1-gimkl-2020a-Python-3.8.2 (D) + fmt/8.0.1 (D) pyspoa/0.0.8-gimkl-2018b-Python-3.8.1 + fontconfig/2.12.1-gimkl-2017a Python-Geo/2.7.14-gimkl-2017a + fontconfig/2.13.1-GCCcore-7.4.0 Python-Geo/2.7.16-gimkl-2018b + fontconfig/2.13.1-GCCcore-9.2.0 (D) Python-Geo/3.6.3-gimkl-2017a + forge/19.0 Python-Geo/3.7.3-gimkl-2018b + forge/20.0.2 (D) Python-Geo/3.8.2-gimkl-2020a + FoX/4.1.2-intel-2018b Python-Geo/3.9.5-gimkl-2020a (D) + FragGeneScan/1.31-gimkl-2018b Python-GPU/3.6.3-gimkl-2017a + FreeBayes/1.1.0-gimkl-2017a Python/2.7.14-gimkl-2017a + FreeBayes/1.3.1-GCC-7.4.0 Python/2.7.16-gimkl-2018b + FreeBayes/1.3.2-GCC-9.2.0 (D) Python/2.7.16-intel-2018b + freetype/2.7.1-gimkl-2017a Python/2.7.18-gimkl-2020a + freetype/2.9.1-GCCcore-7.4.0 Python/3.6.3-gimkl-2017a + freetype/2.10.1-GCCcore-9.2.0 (D) Python/3.7.3-gimkl-2018b + FreeXL/1.0.2-gimkl-2017a Python/3.8.1-gimkl-2018b + FreeXL/1.0.5-GCCcore-7.4.0 (D) Python/3.8.2-gimkl-2020a (D) + FreeXL/1.0.5-GCCcore-9.2.0 Python/3.9.5-gimkl-2020a + FriBidi/1.0.10-GCCcore-9.2.0 qcat/1.1.0-gimkl-2020a-Python-3.8.2 + +[removed most of the output here for clarity] + +----------------------------------- /cm/local/modulefiles ----------------------------------- + cluster-tools/8.0 freeipmi/1.5.5 module-git openmpi/mlnx/gcc/64/2.1.2a1 + cmd gcc/6.3.0 module-info shared + cuda-dcgm/1.3.3.1 ipmitool/1.8.18 null + dot lua/5.3.4 openldap + + Where: + D: Default Module + +Use "module spider" to find all possible modules. +Use "module keyword key1 key2 ..." to search for all possible modules matching any of the +"keys". + +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/env-output.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/env-output.snip new file mode 100644 index 00000000..98c1f731 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/env-output.snip @@ -0,0 +1,18 @@ +``` +[removed some lines for clarity] +EBROOTTCL=/opt/nesi/CS400_centos7_bdw/Tcl/8.6.10-GCCcore-9.2.0 +CPUARCH_STRING=bdw +TERM=xterm-256color +SHELL=/bin/bash +EBROOTGCCCORE=/opt/nesi/CS400_centos7_bdw/GCCcore/9.2.0 +EBDEVELFREETYPE=/opt/nesi/CS400_centos7_bdw/freetype/2.10.1-GCCcore-9.2.0/easybuild/freetype-2.10.1-GCCcore-9.2.0-easybuild-devel +HISTSIZE=10000 +XALT_EXECUTABLE_TRACKING=yes +MODULEPATH_ROOT=/usr/share/modulefiles +LMOD_SYSTEM_DEFAULT_MODULES=NeSI +SSH_CLIENT=192.168.94.65 45946 22 +EBDEVELMETIS=/opt/nesi/CS400_centos7_bdw/METIS/5.1.0-GCCcore-9.2.0/easybuild/METIS-5.1.0-GCCcore-9.2.0-easybuild-devel +XALT_DIR=/opt/nesi/CS400_centos7_bdw/XALT/current +LMOD_PACKAGE_PATH=/opt/nesi/share/etc/lmod +``` +{: .output } \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/missing-r.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/missing-r.snip new file mode 100644 index 00000000..6abfecdc --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/missing-r.snip @@ -0,0 +1,4 @@ +``` +/usr/bin/which: no R in (/opt/nesi/CS400_centos7_bdw/XALT/current/bin:/opt/nesi/CS400_centos7_bdw/Python/3.10.5-gimkl-2022a/bin:/opt/nesi/CS400_centos7_bdw/OpenSSL/1.1.1k-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/Tk/8.6.10-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/Tcl/8.6.10-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/SQLite/3.36.0-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/netCDF/4.8.1-gimpi-2022a/bin:/opt/nesi/CS400_centos7_bdw/cURL/7.83.1-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/libxslt/1.1.34-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/libxml2/2.9.10-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/ncurses/6.2-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/libjpeg-turbo/2.1.3-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/HDF5/1.12.2-gimpi-2022a/bin:/opt/nesi/CS400_centos7_bdw/freetype/2.11.1-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/libpng/1.6.37-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/XZ/5.2.5-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/bzip2/1.0.8-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/impi/2021.5.1-GCC-11.3.0/mpi/2021.5.1/libfabric/bin:/opt/nesi/CS400_centos7_bdw/impi/2021.5.1-GCC-11.3.0/mpi/2021.5.1/bin:/opt/nesi/CS400_centos7_bdw/UCX/1.12.1-GCC-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/numactl/2.0.14-GCC-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/binutils/2.38-GCCcore-11.3.0/bin:/opt/nesi/CS400_centos7_bdw/GCCcore/11.3.0/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/nesi/share/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin) +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-list-default.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-list-default.snip new file mode 100644 index 00000000..9af85c48 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-list-default.snip @@ -0,0 +1,7 @@ +``` + +Currently Loaded Modules: + 1) XALT/minimal 2) slurm 3) NeSI (S) + +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-load-r.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-load-r.snip new file mode 100644 index 00000000..e923ede2 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/module-load-r.snip @@ -0,0 +1,10 @@ +``` +{{ site.remote.prompt }} module load R +{{ site.remote.prompt }} which R +``` +{: .language-bash} + +``` +/opt/nesi/CS400_centos7_bdw/R/4.2.1-gimkl-2022a/bin/R +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-ls-dir-command.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-ls-dir-command.snip new file mode 100644 index 00000000..27148e5a --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-ls-dir-command.snip @@ -0,0 +1,9 @@ +``` +{{ site.remote.prompt }} ls /opt/nesi/CS400_centos7_bdw/R/4.2.1-gimkl-2022a/bin +``` +{: .language-bash} + +``` +R {{ site.example.shell }} +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-module-path.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-module-path.snip new file mode 100644 index 00000000..b368211c --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/r-module-path.snip @@ -0,0 +1,58 @@ +``` +{{ site.remote.prompt }} echo $PATH +``` +{: .language-bash} + +``` +/opt/nesi/CS400_centos7_bdw/XALT/current/bin:/opt/nesi/CS400_centos7_bdw/Python/3.8.2-gimkl-2020a/bin:/opt/nesi/CS400_centos7_bdw/Tk/8.6.10-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/Tcl/8.6.10-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/SuiteSparse/5.6.0-gimkl-2020a-METIS-5.1.0/bin:/opt/nesi/CS400_centos7_bdw/METIS/5.1.0-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/SQLite/3.31.1-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/netCDF/4.7.3-gimpi-2020a/bin:/opt/nesi/CS400_centos7_bdw/PCRE/8.43-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/cURL/7.64.0-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/libxslt/1.1.34-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/libxml2/2.9.10-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/ncurses/6.1-GCCcore-9.2.0/bin:/opt/nesi/mahuika/libjpeg-turbo/2.0.2-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/HDF5/1.10.5-gimpi-2020a/bin:/opt/nesi/CS400_centos7_bdw/freetype/2.10.1-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/libpng/1.6.37-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/XZ/5.2.4-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/bzip2/1.0.8-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/impi/2019.6.166-GCC-9.2.0/intel64/libfabric/bin:/opt/nesi/CS400_centos7_bdw/impi/2019.6.166-GCC-9.2.0/intel64/bin:/opt/nesi/CS400_centos7_bdw/binutils/2.32-GCCcore-9.2.0/bin:/opt/nesi/CS400_centos7_bdw/GCCcore/9.2.0/bin:/home/harrellw/bin:/home/harrellw/.local/bin:/home/harrellw/apps/bin:/usr/lpp/mmfs/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/nesi/share/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/nesi/bin:/opt/ibutils/bin +``` +{: .output} + +We can improve the readability of this command slightly by replacing the colon delimiters (`:`) with newline (`\n`) characters. + +``` +{{ site.remote.prompt }} echo $PATH | tr ":" "\n" +``` +{: .language-bash} + +``` +/opt/nesi/CS400_centos7_bdw/XALT/current/bin +/opt/nesi/CS400_centos7_bdw/R/4.2.1-gimkl-2022a/bin +/opt/nesi/CS400_centos7_bdw/nodejs/16.15.1-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/Java/17 +/opt/nesi/CS400_centos7_bdw/Java/17/bin +/opt/nesi/CS400_centos7_bdw/PCRE2/10.40-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/Python/3.10.5-gimkl-2022a/bin +/opt/nesi/CS400_centos7_bdw/OpenSSL/1.1.1k-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/Tk/8.6.10-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/Tcl/8.6.10-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/SQLite/3.36.0-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/netCDF/4.8.1-gimpi-2022a/bin +/opt/nesi/CS400_centos7_bdw/cURL/7.83.1-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/libxslt/1.1.34-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/libxml2/2.9.10-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/ncurses/6.2-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/libjpeg-turbo/2.1.3-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/HDF5/1.12.2-gimpi-2022a/bin +/opt/nesi/CS400_centos7_bdw/freetype/2.11.1-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/libpng/1.6.37-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/XZ/5.2.5-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/bzip2/1.0.8-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/impi/2021.5.1-GCC-11.3.0/mpi/2021.5.1/libfabric/bin +/opt/nesi/CS400_centos7_bdw/impi/2021.5.1-GCC-11.3.0/mpi/2021.5.1/bin +/opt/nesi/CS400_centos7_bdw/UCX/1.12.1-GCC-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/numactl/2.0.14-GCC-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/binutils/2.38-GCCcore-11.3.0/bin +/opt/nesi/CS400_centos7_bdw/GCCcore/11.3.0/bin +/opt/slurm/sbin +/opt/slurm/bin +/opt/nesi/share/bin +/usr/local/bin +/usr/bin +/usr/local/sbin +/usr/sbin +/opt/ibutils/bin +/opt/nesi/vdt +/opt/nesi/bin +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/software-dependencies.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/software-dependencies.snip new file mode 100644 index 00000000..f2bd61f4 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/software-dependencies.snip @@ -0,0 +1,20 @@ +To demonstrate, let's use `module list`. + +``` +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` +Currently Loaded Modules: + 1) XALT/minimal 8) GCC/11.3.0 15) gimpi/2022a 22) libreadline/8.1-GCCcore 29) Java/17 + 2) slurm 9) libpmi/2-slurm 16) imkl-FFTW/2022.0.2-gimpi-2022a 23) libpng/1.6.37-GCCcore 30) nodejs/16.15.1-GCCcore-11.3.0 + 3) NeSI (S) 10) numactl/2.0.14-GCC-11.3.0 17) gimkl/2022a 24) libxml2/2.9.10-GCCcore-11.3.0 31) OpenSSL/1.1.1k-GCCcore-11.3.0 + 4) LegacySystemLibs/7 11) UCX/1.12.1-GCC-11.3.0 18) bzip2/1.0.8-GCCcore-11.3.0 25) SQLite/3.36.0-GCCcore-11.3.0 32) R/4.2.1-gimkl-2022a + 5) GCCcore/11.3.0 12) impi/2021.5.1-GCC-11.3.0 19) XZ/5.2.5-GCCcore-11.3.0 26) cURL/7.83.1-GCCcore-11.3.0 + 6) zlib/1.2.11-GCCcore-11.3.0 13) AlwaysIntelMKL/1.0 20) PCRE2/10.40-GCCcore-11.3.0 27) NLopt/2.7.0-GCC-11.3.0 + 7) binutils/2.38-GCCcore-11.3.0 14) imkl/2022.0.2 21) ncurses/6.2-GCCcore-11.3.0 28) GMP/6.2.1-GCCcore-11.3.0 +``` +{: .output} + +Notice that our initial list of modules has increased by 30. When we loaded R, it also loaded all of it's dependencies along with all the dependencies of those modules. diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-gcc-version.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-gcc-version.snip new file mode 100644 index 00000000..941b2050 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-gcc-version.snip @@ -0,0 +1,71 @@ +Let's take a closer look at the `GCC` module. GCC is a widely used +C/C++/Fortran compiler. Tons of software is dependent on the GCC version, and +might not compile or run if the wrong version is loaded. In this case, there +are many different versions: `GCC/5.4.0`, `GCC/6.3.0` through `GCC/9.2.0`. +How do we load each copy and which copy is the default? + +In this case, `GCC/9.2.0` has a `(D)` next to it. This indicates that it is the +default — if we type `module load GCC`, this is the copy that will be +loaded. + +``` +{{ site.remote.prompt }} module load GCC +{{ site.remote.prompt }} gcc --version +``` +{: .language-bash} + +``` +gcc (GCC) 9.2.0 +Copyright (C) 2019 Free Software Foundation, Inc. +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +``` +{: .output} + +So how do we load the non-default copy of a software package? In this case, the +only change we need to make is be more specific about the module we are +loading. There are many other GCC versions. To load a +non-default module, the only change we need to make to our `module load` +command is to add the version number after the `/`. + +``` +{{ site.remote.prompt }} module load GCC/5.4.0 +``` +{: .language-bash} + +``` +The following have been reloaded with a version change: + 1) GCC/9.2.0 => GCC/5.4.0 2) GCCcore/9.2.0 => GCCcore/5.4.0 3) binutils/2.32-GCCcore-9.2.0 => binutils/2.26-GCCcore-5.4.0 + +``` +{: .output} + +Notice how the module command has swapped out versions of the GCC module as well as it's dependencies. This is quite helpful as it reconciles all of the software requirements with one command. +And now we test which version we are using: + +``` +{{ site.remote.prompt }} gcc --version +``` +{: .language-bash} + +``` +gcc (GCC) 5.4.0 +Copyright (C) 2015 Free Software Foundation, Inc. +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +``` +{: .output} + +We are now left with only those module required to do our work for this project. + +``` +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` +Currently Loaded Modules: + 1) XALT/minimal 3) NeSI (S) 5) GCCcore/5.4.0 7) GCC/5.4.0 + 2) slurm 4) zlib/1.2.11-GCCcore-9.2.0 6) binutils/2.26-GCCcore-5.4.0 +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-python-version.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-python-version.snip new file mode 100644 index 00000000..5ca39b4e --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/modules/wrong-python-version.snip @@ -0,0 +1,81 @@ +Let's take a closer look at the `Python` modules. There are many applications +that are run using python and may fail to run if the wrong version is loaded. +In this case, there are many different versions: `Python/3.6.3-gimkl-2017a`, +`Python/3.7.3-gimkl-2018b` through to the newest versions. + +How do we load each copy and which copy is the default? + +In this case, `Python/3.8.2-gimkl-2020a` has a `(D)` next to it. This indicates that it is the +default — if we type `module load Python`, as we did above, this is the copy that will be +loaded. + +``` +{{ site.remote.prompt }} module load Python +{{ site.remote.prompt }} python3 --version +``` +{: .language-bash} + +``` +Python 3.8.2 +``` +{: .output} + +So how do we load the non-default copy of a software package? In this case, the +only change we need to make is be more specific about the module we are +loading. There are many other Python versions. To load a +non-default module, the only change we need to make to our `module load` +command is to add the version number after the `/`. + +``` +{{ site.remote.prompt }} module load Python/3.9.5-gimkl-2020a +``` +{: .language-bash} + +``` +The following have been reloaded with a version change: + 1) Python/3.8.2-gimkl-2020a => Python/3.9.5-gimkl-2020a +``` +{: .output} + +Notice how the module command has swapped out versions of the Python module. +And now we test which version we are using: + +``` +{{ site.remote.prompt }} python3 --version +``` +{: .language-bash} + +``` +Python 3.9.5 +``` +{: .output} + +We are now left with only those module required to do our work for this project. + +``` +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` + 1) XALT/minimal 20) HDF5/1.10.5-gimpi-2020a + 2) slurm 21) libjpeg-turbo/2.0.2-GCCcore-9.2.0 + 3) NeSI (S) 22) ncurses/6.1-GCCcore-9.2.0 + 4) craype-broadwell 23) libreadline/8.0-GCCcore-9.2.0 + 5) craype-network-infiniband 24) libxml2/2.9.10-GCCcore-9.2.0 + 6) GCCcore/9.2.0 25) libxslt/1.1.34-GCCcore-9.2.0 + 7) zlib/1.2.11-GCCcore-9.2.0 26) cURL/7.64.0-GCCcore-9.2.0 + 8) binutils/2.32-GCCcore-9.2.0 27) PCRE/8.43-GCCcore-9.2.0 + 9) GCC/9.2.0 28) netCDF/4.7.3-gimpi-2020a + 10) libpmi 29) SQLite/3.31.1-GCCcore-9.2.0 + 11) impi/2019.6.166-GCC-9.2.0 30) METIS/5.1.0-GCCcore-9.2.0 + 12) gimpi/2020a 31) tbb/2019_U9-GCCcore-9.2.0 + 13) imkl/2020.0.166-gimpi-2020a 32) SuiteSparse/5.6.0-gimkl-2020a-METIS-5.1.0 + 14) gimkl/2020a 33) Tcl/8.6.10-GCCcore-9.2.0 + 15) bzip2/1.0.8-GCCcore-9.2.0 34) Tk/8.6.10-GCCcore-9.2.0 + 16) XZ/5.2.4-GCCcore-9.2.0 35) LLVM/10.0.1-GCCcore-9.2.0 + 17) libpng/1.6.37-GCCcore-9.2.0 36) OpenSSL/1.1.1k-GCCcore-9.2.0 + 18) freetype/2.10.1-GCCcore-9.2.0 37) Python/3.9.5-gimkl-2020a + 19) Szip/2.1.1-GCCcore-9.2.0 +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/four-tasks-jobscript.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/four-tasks-jobscript.snip new file mode 100644 index 00000000..2629f51e --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/four-tasks-jobscript.snip @@ -0,0 +1,15 @@ +``` +{{ site.remote.bash_shebang }} +{{ site.sched.comment }} {{ site.sched.flag.name }} parallel-pi +{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} +{{ site.sched.comment }} --nodes 1 +{{ site.sched.comment }} --ntasks 4 +{{ site.sched.comment }} --mem=3G + +# Load the computing environment we need +module load Python/3.8.2-gimkl-2020a + +# Execute the task +srun python pi.py 100000000 +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/one-task-with-memory-jobscript.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/one-task-with-memory-jobscript.snip new file mode 100644 index 00000000..cde991af --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/parallel/one-task-with-memory-jobscript.snip @@ -0,0 +1,13 @@ +``` +{{ site.remote.bash_shebang }} +{{ site.sched.comment }} {{ site.sched.flag.name }} serial-pi +{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} +{{ site.sched.comment }} --mem=3G + +# Load the computing environment we need +module load Python/3.8.2-gimkl-2020a + +# Execute the task +python pi.py 100000000 +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/resources/account-history.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/account-history.snip new file mode 100644 index 00000000..f791ee96 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/account-history.snip @@ -0,0 +1,8 @@ +``` + JobID JobName Alloc Elapsed TotalCPU ReqMem MaxRSS State +--------------- ---------------- ----- ----------- ------------ ------- -------- ---------- +19284436 bash 8 09:54:53 3-04:47:11 300Gn COMPLETED +19284436.batch batch 2 09:54:53 3-04:47:11 300Gn COMPLETED +19284436.extern extern 8 09:54:53 3-04:47:11 300Gn 1363913k COMPLETED +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/resources/get-job-node.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/get-job-node.snip new file mode 100644 index 00000000..c0f3c025 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/get-job-node.snip @@ -0,0 +1,5 @@ +``` +JOBID USER ACCOUNT NAME CPUS MIN_MEM PARTITI START_TIME TIME_LEFT STATE NODELIST(REASON) +26763045 cwal219 {{site.sched.projectcode}} test 2 512M large May 11 11:35 14:46 RUNNING wbn144 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/resources/monitor-processes-top.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/monitor-processes-top.snip new file mode 100644 index 00000000..12685735 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/monitor-processes-top.snip @@ -0,0 +1,19 @@ +``` +top - 21:00:19 up 3:07, 1 user, load average: 1.06, 1.05, 0.96 +Tasks: 311 total, 1 running, 222 sleeping, 0 stopped, 0 zombie +%Cpu(s): 7.2 us, 3.2 sy, 0.0 ni, 89.0 id, 0.0 wa, 0.2 hi, 0.2 si, 0.0 st +KiB Mem : 16303428 total, 8454704 free, 3194668 used, 4654056 buff/cache +KiB Swap: 8220668 total, 8220668 free, 0 used. 11628168 avail Mem + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1693 jeff 20 0 4270580 346944 171372 S 29.8 2.1 9:31.89 gnome-shell + 3140 jeff 20 0 3142044 928972 389716 S 27.5 5.7 13:30.29 Web Content + 3057 jeff 20 0 3115900 521368 231288 S 18.9 3.2 10:27.71 firefox + 6007 jeff 20 0 813992 112336 75592 S 4.3 0.7 0:28.25 tilix + 1742 jeff 20 0 975080 164508 130624 S 2.0 1.0 3:29.83 Xwayland + 1 root 20 0 230484 11924 7544 S 0.3 0.1 0:06.08 systemd + 68 root 20 0 0 0 0 I 0.3 0.0 0:01.25 kworker/4:1 + 2913 jeff 20 0 965620 47892 37432 S 0.3 0.3 0:11.76 code + 2 root 20 0 0 0 0 S 0.0 0.0 0:00.02 kthreadd +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/resources/seff.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/seff.snip new file mode 100644 index 00000000..97336acd --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/seff.snip @@ -0,0 +1,13 @@ +``` +Job ID: 27323570 +Cluster: mahuika +User/Group: username/username +State: COMPLETED (exit code 0) +Cores: 1 +Tasks: 1 +Nodes: 1 +Job Wall-time: 5.11% 00:00:46 of 00:15:00 time limit +CPU Efficiency: 141.30% 00:01:05 of 00:00:46 core-walltime +Mem Efficiency: 93.31% 233.29 MB of 250.00 MB +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/resources/system-memory-free.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/system-memory-free.snip new file mode 100644 index 00000000..ec4c0d3f --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/resources/system-memory-free.snip @@ -0,0 +1,6 @@ +``` +total used free shared buff/cache available +Mem: 3.8G 1.5G 678M 327M 1.6G 1.6G +Swap: 3.9G 170M 3.7G +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/OOM.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/OOM.snip new file mode 100644 index 00000000..cb6595a0 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/OOM.snip @@ -0,0 +1,8 @@ +```out +JobID JobName Partition Account AllocCPUS State ExitCode +------------ ---------- ---------- ---------- ---------- ---------- -------- +27323464 my_job large {{site.sched.projectcode}} 4 OUT_OF_ME+ 0:125 +27323464.ba+ batch {{site.sched.projectcode}} 4 OUT_OF_ME+ 0:125 +27323464.ex+ extern {{site.sched.projectcode}} 4 COMPLETED 0:0 +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/array-example.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/array-example.snip new file mode 100644 index 00000000..74ef67e4 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/array-example.snip @@ -0,0 +1,8 @@ +#!/bin/bash -e +#SBATCH --job-name=ArrayJob #job name +#SBATCH --time=00:01:00 #walltime +#SBATCH --mem=3000 #memory in MB +#SBATCH --array=1-2 #array size + +pwd #prints working directory +echo "${SLURM_ARRAY_TASK_ID}" #prints SLURM_ARRAY_TASK_ID \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/distibuted-mem-example.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/distibuted-mem-example.snip new file mode 100644 index 00000000..b0ba9973 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/distibuted-mem-example.snip @@ -0,0 +1,7 @@ +#!/bin/bash -e +#SBATCH --job-name=MPIJob #job name +#SBATCH --time=00:01:00 #walltime +#SBATCH --mem-per-cpu=1500 #memory per logical CPU core in MB +#SBATCH --ntasks=2 #number of tasks (e.g. MPI) + +srun pwd #executes pwd on each task \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/shared-mem-example.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/shared-mem-example.snip new file mode 100644 index 00000000..14e79f50 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scaling/shared-mem-example.snip @@ -0,0 +1,7 @@ +#!/bin/bash -e +#SBATCH --job-name=OpenMPJob #job name +#SBATCH --time=00:01:00 #walltime +#SBATCH --mem=3000 #memory in MB +#SBATCH --cpus-per-task=4 #cores per task (e.g. OpenMP) + +taskset -c -p $$ #prints PID and accessible CPUs \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-script.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-script.snip new file mode 100644 index 00000000..1ba1cd40 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-script.snip @@ -0,0 +1,4 @@ +``` +Submitted batch job 23137702 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status-sacct.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status-sacct.snip new file mode 100644 index 00000000..3f4b1fa1 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status-sacct.snip @@ -0,0 +1,8 @@ +``` +JobID JobName Alloc Elapsed TotalCPU ReqMem MaxRSS State +--------------- ---------------- ----- ----------- ------------ ------- -------- ---------- +31060451 example-job.sl 2 00:00:48 00:33.548 1G CANCELLED +31060451.batch batch 2 00:00:48 00:33.547 102048K CANCELLED +31060451.extern extern 2 00:00:48 00:00:00 0 CANCELLED +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status.snip new file mode 100644 index 00000000..02d04f1e --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/basic-job-status.snip @@ -0,0 +1,5 @@ +``` +JOBID USER ACCOUNT NAME CPUS MIN_MEM PARTITI START_TIME TIME_LEFT STATE NODELIST(REASON) +231964 yourUsername {{site.sched.projectcode}} example-job.sl 1 512M large N/A 1:00 PENDING (Priority) +``` +{: .output} \ No newline at end of file diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/job-with-name-status.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/job-with-name-status.snip new file mode 100644 index 00000000..dc1fda51 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/job-with-name-status.snip @@ -0,0 +1,5 @@ +``` +JOBID USER ACCOUNT NAME ST REASON START_TIME TIME TIME_LEF... +38191 yourUsername yourAccount new_name PD Priority N/A 0:00 1:00:00 ... +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/option-flags-list.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/option-flags-list.snip new file mode 100644 index 00000000..ee893031 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/option-flags-list.snip @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--job-name#SBATCH --job-name=MyJobThe name that will appear when using squeue or sacct
--account#SBATCH --account={{site.sched.projectcode}}The account your core hours will be 'charged' to.
--time#SBATCH --time=DD-HH:MM:SSJob max walltime
--mem#SBATCH --mem=1500MMemory required per node.
--output#SBATCH --output=%j_output.outPath and name of standard output file.
--ntasks#SBATCH --ntasks=2Will start 2 MPI tasks.
--cpus-per-task#SBATCH --cpus-per-task=10 +

Will request 10 logical CPUs per task.

+

See Hyperthreading.

+
--partition#SBATCH --partition=milan +

Requests that your script run on a specific subsection of the cluster.

+

The scheduler will generally try to determine this for you based on the resources requested, but you may need to set this manually.

+
+ + + + diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/print-sched-variables.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/print-sched-variables.snip new file mode 100644 index 00000000..a5e8972c --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/print-sched-variables.snip @@ -0,0 +1,30 @@ +> ## Job environment variables +> +> When {{ site.sched.name }} runs a job, it sets a number of environment +> variables for the job. One of these will let us check what directory our job +> script was submitted from. The `SLURM_SUBMIT_DIR` variable is set to the +> directory from which our job was submitted. Using the `SLURM_SUBMIT_DIR` +> variable, modify your job so that it prints out the location from which the +> job was submitted. +> +> > ## Solution +> > +> > ``` +> > {{ site.remote.prompt }} nano example-job.sh +> > {{ site.remote.prompt }} cat example-job.sh +> > ``` +> > {: .language-bash} +> > +> > ``` +> > {{ site.remote.bash_shebang }} +> > #SBATCH --time 00:00:30 +> > +> > echo -n "This script is running on " +> > hostname +> > +> > echo "This job was launched in the following directory:" +> > echo ${SLURM_SUBMIT_DIR} +> > ``` +> > {: .language-bash} +> {: .solution} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-job.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-job.snip new file mode 100644 index 00000000..875df1c3 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-job.snip @@ -0,0 +1,4 @@ +``` +{{ site.remote.prompt }} cat slurm-38193.out +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-output.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-output.snip new file mode 100644 index 00000000..453dc9fd --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/runtime-exceeded-output.snip @@ -0,0 +1,7 @@ +``` +This job is running on: +wbn003 +slurmstepd: error: *** JOB 38193 ON wbn003 CANCELLED AT 2021-11-17T16:35:48 +DUE TO TIME LIMIT *** +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-begin.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-begin.snip new file mode 100644 index 00000000..dcf8bd11 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-begin.snip @@ -0,0 +1,5 @@ +``` +JOBID USER ACCOUNT NAME CPUS MIN_MEM PARTITI START_TIME TIME_LEFT STATE NODELIST(REASON) +231972 yourUsername {{site.sched.projectcode}} example-job.sl 1 512M large N/A 00:30 PENDING (Priority) +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-cancel.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-cancel.snip new file mode 100644 index 00000000..0d19b7de --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-job-cancel.snip @@ -0,0 +1,4 @@ +``` +JOBID USER ACCOUNT NAME CPUS MIN_MEM PARTITI START_TIME TIME_LEFT STATE NODELIST(REASON) +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-multiple-jobs.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-multiple-jobs.snip new file mode 100644 index 00000000..9fe5831a --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/terminate-multiple-jobs.snip @@ -0,0 +1,27 @@ +> ## Cancelling multiple jobs +> +> We can also cancel all of our jobs at once using the `-u` option. This will +> delete all jobs for a specific user (in this case, yourself). Note that you +> can only delete your own jobs. +> +> Try submitting multiple jobs and then cancelling them all. +> +> > ## Solution +> > +> > First, submit a trio of jobs: +> > +> > ``` +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sl +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sl +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sl +> > ``` +> > {: .language-bash} +> > +> > Then, cancel them all: +> > +> > ``` +> > {{ site.remote.prompt }} {{ site.sched.del }} --user yourUsername +> > ``` +> > {: .language-bash} +> {: .solution} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/using-nodes-interactively.snip b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/using-nodes-interactively.snip new file mode 100644 index 00000000..b9103808 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/scheduler/using-nodes-interactively.snip @@ -0,0 +1,51 @@ +`{{ site.sched.interactive }}` runs a single command on the cluster and then +exits. Let's demonstrate this by running the `hostname` command with `{{ +site.sched.interactive }}`. (We can cancel an `{{ site.sched.interactive }}` +job with `Ctrl-c`.) + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} hostname +``` +{: .language-bash} + +``` +wbn003 +``` +{: .output} + +`{{ site.sched.interactive }}` accepts all of the same options as `{{ +site.sched.submit.name }}`. However, instead of specifying these in a script, +these options are specified on the command-line when starting a job. To submit +a job that uses 2 CPUs for instance, we could use the following command: + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} --ntasks 2 echo "This job will use 2 CPUs." +``` +{: .language-bash} + +``` +This job will use 2 CPUs. +This job will use 2 CPUs. +``` +{: .output} + +Typically, the resulting shell environment will be the same as that for +`{{ site.sched.submit.name }}`. + +### Interactive jobs + +Sometimes, you will need a lot of resource for interactive use. Perhaps it's +our first time running an analysis or we are attempting to debug something that +went wrong with a previous job. Fortunately, {{ site.sched.name }} makes it +easy to start an interactive job with `{{ site.sched.interactive }}`: + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} --pty bash +``` +{: .language-bash} + +You should be presented with a bash prompt. Note that the prompt will likely +change to reflect your new location, in this case the compute node we are +logged on. You can also verify this with `hostname`. + +When you are done with the interactive job, type `exit` to quit your session. diff --git a/_includes/snippets_library/NeSI_Mahuika_slurm/setup.md b/_includes/snippets_library/NeSI_Mahuika_slurm/setup.md new file mode 100644 index 00000000..d9af1023 --- /dev/null +++ b/_includes/snippets_library/NeSI_Mahuika_slurm/setup.md @@ -0,0 +1,132 @@ +# BIOSCI701-NeSI_Jupyter_Login +Login and troubleshooting instructions for NeSI Jupyter Services. + +>**WARNING**- We **do not recommend** using Internet Explorer to access [NeSI JupyterHub](https://jupyter.nesi.org.nz/hub/login) + +1. Follow https://jupyter.nesi.org.nz/hub/login +2.

Enter NeSI username (same as UoA UPI), HPC password and 6 digit second factor token

drawing

+3.

Choose server options as below OR as required for the session +>Project code should be **uoa03265** (select from drop down list), Number of CPUs and memory size will remain unchanged. However, select the appropriate **Wall time** based on the projected length of a session + +


drawing

+## Where to Type Commands: How to Open a New Shell + +4.

Jupyter Launcher screen + +

drawing

+Some computers include a default Unix Shell program. The steps below describe +some methods for identifying and opening a Unix Shell program if you already +have one installed. There are also options for identifying and downloading a +Unix Shell program, a Linux/UNIX emulator, or a program to access a Unix Shell +on a server. + +### Windows + +Computers with Windows operating systems do not automatically have a Unix Shell +program installed. In this lesson, we encourage you to use an emulator included +in Git for Windows, which gives you access to both Bash shell commands and Git. +If you have attended a Software Carpentry workshop session, it is likely you +have already received instructions on how to install Git for Windows. + +Once installed, you can open a terminal by running the program Git Bash from +the Windows start menu. + +#### Shell Programs for Windows + +* [Git for Windows](https://gitforwindows.org/) — *Recommended* +* [Windows Subsystem for Linux]( + https://docs.microsoft.com/en-us/windows/wsl/install-win10) + — advanced option for Windows 10 + +> ## Alternatives to Git for Windows +> +> Other solutions are available for running Bash commands on Windows. There is +> now a Bash shell command-line tool available for Windows 10. Additionally, +> you can run Bash commands on a remote computer or server that already has a +> Unix Shell, from your Windows machine. This can usually be done through a +> Secure Shell (SSH) client. One such client available for free for Windows +> computers is PuTTY. See the reference below for information on installing and +> using PuTTY, using the Windows 10 command-line tool, or installing and using +> a Unix/Linux emulator. +> +> For advanced users, you may choose one of the following alternatives: +> +> * Install the [Windows Subsystem for +> Linux][microsoft-wsl] +> * Use the Windows [Powershell][microsoft-powershell] +> * Read up on [Using a Unix/Linux emulator][unix-emulator] (Cygwin) or Secure +> Shell (SSH) client (Putty) +> +> > ## Warning +> > +> > Commands in the Windows Subsystem for Linux (WSL), Powershell, or Cygwin +> > may differ slightly from those shown in the lesson or presented in the +> > workshop. Please ask if you encounter such a mismatch — you're +> > probably not alone. +> {: .challenge} +{: .discussion} + +### macOS + +On macOS, the default Unix Shell is accessible by running the Terminal program +from the `/Application/Utilities` folder in Finder. + +To open Terminal, try one or both of the following: + +* In Finder, select the Go menu, then select Utilities. Locate Terminal in the + Utilities folder and open it. +* Use the Mac ‘Spotlight’ computer search function. Search for: `Terminal` and + press Return. + +For an introduction, see [How to Use Terminal on a Mac][mac-terminal]. + +### Linux + +On most versions of Linux, the default Unix Shell is accessible by running the +[(Gnome) Terminal](https://help.gnome.org/users/gnome-terminal/stable/) or +[(KDE) Konsole](https://konsole.kde.org/) or +[xterm](https://en.wikipedia.org/wiki/Xterm), which can be found via the +applications menu or the search bar. + +### Special Cases + +If none of the options above address your circumstances, try an online search +for: `Unix shell [your operating system]`. + +## SSH for Secure Connections + +All students should have an SSH client installed. SSH is a tool that allows us +to connect to and use a remote computer as our own. + +### SSH for Windows + +Git for Windows comes with SSH preinstalled: you do not have to do anything. + +> ## GUI Support for Windows +> +> If you know that the software you will be running on the cluster requires a +> graphical user interface (a GUI window needs to open for the application to +> run properly), please install [MobaXterm](https://mobaxterm.mobatek.net) Home +> Edition. +{: .discussion} + +### SSH for macOS + +macOS comes with SSH pre-installed: you do not have to do anything. + +> ## GUI Support for macOS +> +> If you know that the software you will be running requires a graphical user +> interface, please install [XQuartz](https://www.xquartz.org). +{: .discussion} + +### SSH for Linux + +Linux comes with SSH and X window support preinstalled: you do not have to do +anything. + + +[mac-terminal]: https://www.macworld.co.uk/feature/mac-software/how-use-terminal-on-mac-3608274/ +[microsoft-wsl]: https://docs.microsoft.com/en-us/windows/wsl/install-win10 +[microsoft-powershell]: https://docs.microsoft.com/en-us/powershell/scripting/learn/remoting/ssh-remoting-in-powershell-core?view=powershell-7 +[unix-emulator]: https://faculty.smu.edu/reynolds/unixtut/windows.html diff --git a/_includes/snippets_library/NeSI_Maui_slurm/_config_options.yml b/_includes/snippets_library/NeSI_Maui_slurm/_config_options.yml new file mode 100644 index 00000000..5655c3c6 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/_config_options.yml @@ -0,0 +1,67 @@ +#------------------------------------------------------------ +# NeSI: Maui + SLURM +#------------------------------------------------------------ + +# Cluster host and scheduler options: the defaults come from +# Graham at Compute Canada, running SLURM. Other options can +# be found in the library of snippets, +# `_includes/snippets_library`. To use one, replace options +# below with those in `_config_options.yml` from the +# library. E.g, to customise for Cirrus at EPCC, running +# PBS, we could replace the options below with those from +# +# _includes/snippets_library/EPCC_Cirrus_pbs/_config_options.yml +# +# If your cluster is not represented in the library, please +# copy an existing folder, rename it, and customize for your +# installation. Remember to keep the leading slash on the +# `snippets` variable below! + +snippets: "/snippets_library/NeSI_Maui_slurm" + +local: + prompt: "[user@laptop ~]$" + bash_shebang: "#!/usr/bin/env bash" + +remote: + name: "maui" + login: "nesi.org.nz" + host: "maui02" + node: "maui02" + location: "Greta Point Datacenter" + homedir: "/home" + user: "yourUsername" + prompt: "[yourUsername@maui ~]$" + bash_shebang: "#!/usr/bin/env bash" + +sched: + name: "SLURM" + submit: + name: "sbatch" + options: "" + queue: + debug: "debug" + status: "squeue" + flag: + user: "-u yourUsername" + interactive: "" + histdetail: "-l -j" + name: "-J" + time: "-t" + queue: "-p" + del: "scancel" + interactive: "srun" + info: "sinfo" + comment: "#SBATCH" + hist: "sacct" + +episode_order: + - 11-hpc-intro + - 12-cluster + - 13-scheduler + - 14-modules + - 15-transferring-files + - 16-parallel + - 17-resources + - 18-responsibility + diff --git a/_includes/snippets_library/NeSI_Maui_slurm/cluster/queue-info.snip b/_includes/snippets_library/NeSI_Maui_slurm/cluster/queue-info.snip new file mode 100644 index 00000000..5610d2c8 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/cluster/queue-info.snip @@ -0,0 +1,14 @@ +``` +PARTITION AVAIL TIMELIMIT S:C:T MEMORY NODES(A/I) NODELIST +maintenance up 3-00:00:00 2+:1+:1+ 108000+ 244/3 vgpuwbg005,wbg[001-004],wbh001,wbl[001-005,008-011],wbn[003-006,009-064,066-120,122-176,178-236,238],wch001,wcl[001-002] +interactive up 3-00:00:00 2:18:2 108000 2/2 wbn[003-006] +large up 3-00:00:00 2:18:2 108000 226/0 wbn[009-064,066-120,122-176,178-236,238] +long up 21-00:00:0 2:18:2 108000 81/0 wbn[011-062,067-092,235-236,238] +gpu up 3-00:00:00 2:18:2 474000 4/0 wbg[001-004] +infill up 3-00:00:00 2:18:2 474000 4/0 wbg[001-004] +igpu up 3-00:00:00 8:1:1 108000 0/1 vgpuwbg005 +prepost up 3:00:00 2:18:2 474000 8/0 wbl[001-005,008-011] +bigmem up 7-00:00:00 2:18:2 474000 8/0 wbl[001-005,008-011] +hugemem up 7-00:00:00 2+:16+:2 1500000+ 4/0 wbh001,wch001,wcl[001-002] +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/cluster/specific-node-info.snip b/_includes/snippets_library/NeSI_Maui_slurm/cluster/specific-node-info.snip new file mode 100644 index 00000000..26ddff16 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/cluster/specific-node-info.snip @@ -0,0 +1,11 @@ +> ## Explore a Worker Node +> +> Finally, let's look at the resources available on the worker nodes where your +> jobs will actually run. Try running this command to see the name, CPUs and +> memory (in MegaBytes) available on the worker nodes: +> +> ``` +> {{ site.remote.prompt }} sinfo +> ``` +> {: .language-bash} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/available-modules.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/available-modules.snip new file mode 100644 index 00000000..288bf1cb --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/available-modules.snip @@ -0,0 +1,33 @@ +``` +-----------------/opt/nesi/CS400_centos7_bdw/modules/all ------------------ + ABAQUS/6.14.2 LMDB/0.9.24-GCCcore-9.2.0 (D) + ABAQUS/2017 Loki/0.1.7-gimkl-2017a + ABAQUS/2018 lp_solve/5.5.2.5-gimkl-2017a + ABAQUS/2019 lp_solve/5.5.2.5-gimkl-2018b + ABAQUS/2020 lp_solve/5.5.2.5-gimkl-2020a (D) + ABAQUS/2021 (D) LZO/2.10-GCCcore-7.4.0 + ABySS/2.0.2-gimkl-2017a LZO/2.10-GCCcore-9.2.0 (D) + ABySS/2.2.5-gimpi-2020a (D) M4/1.4.18-GCCcore-7.4.0 + ACTC/1.1-GCCcore-9.2.0 MAFFT/7.310-gimkl-2017a + ADAM/0.30.0 MAFFT/7.429-gimkl-2018b + AdapterRemoval/2.3.1-GCCcore-7.4.0 MAFFT/7.429-gimkl-2020a (D) + Advisor/2020_update1 MAGMA/1.07b-GCCcore-9.2.0 + AGE/2017-GCC-7.4.0-serial MAGMA/1.08-GCCcore-9.2.0 (D) + +[removed most of the output here for clarity] + +----------------------------------- /cm/local/modulefiles ----------------------------------- + cluster-tools/8.0 freeipmi/1.5.5 module-git openmpi/mlnx/gcc/64/2.1.2a1 + cmd gcc/6.3.0 module-info shared + cuda-dcgm/1.3.3.1 ipmitool/1.8.18 null + dot lua/5.3.4 openldap + + Where: + D: Default Module + +Use "module spider" to find all possible modules. +Use "module keyword key1 key2 ..." to search for all possible modules matching any of the +"keys". + +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/missing-python.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/missing-python.snip new file mode 100644 index 00000000..d841efc4 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/missing-python.snip @@ -0,0 +1,9 @@ +``` +/usr/bin/which: no python3 in ( +/opt/software/slurm/16.05.9/bin: +/cvmfs/soft.computecanada.ca/easybuild/software/2017/Core/imkl/11.3.4.258/bin: +/opt/software/bin:/opt/puppetlabs/puppet/bin:/opt/software/slurm/current/bin: +/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin: +/home/yourUsername/.local/bin:/home/yourUsername/bin) +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/module-load-python.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/module-load-python.snip new file mode 100644 index 00000000..f309f030 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/module-load-python.snip @@ -0,0 +1,5 @@ +``` +{{ site.remote.prompt }} module load python +{{ site.remote.prompt }} which python3 +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/python-executable-dir.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-executable-dir.snip new file mode 100644 index 00000000..0f77b6b8 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-executable-dir.snip @@ -0,0 +1,4 @@ +``` +/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/python-3.5.2/bin/python3 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-command.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-command.snip new file mode 100644 index 00000000..3e81e367 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-command.snip @@ -0,0 +1,4 @@ +``` +{{ site.remote.prompt }} ls /cvmfs/soft.computecanada.ca/nix/var/nix/profiles/python-3.5.2/bin +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-output.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-output.snip new file mode 100644 index 00000000..0ec6b7d7 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-ls-dir-output.snip @@ -0,0 +1,8 @@ +``` +2to3 idle3.5 pydoc3.5 python3.5m virtualenv +2to3-3.5 pip python python3.5m-config wheel +easy_install pip3 python3 python3-config +easy_install-3.5 pip3.5 python3.5 pyvenv +idle3 pydoc3 python3.5-config pyvenv-3.5 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/python-module-path.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-module-path.snip new file mode 100644 index 00000000..00c53fbc --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/python-module-path.snip @@ -0,0 +1,4 @@ +``` +/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/python-3.5.2/bin:/opt/software/slurm/16.05.9/bin:/cvmfs/soft.computecanada.ca/easybuild/software/2017/avx2/Compiler/intel2016.4/openmpi/2.1.1/bin:/cvmfs/soft.computecanada.ca/easybuild/software/2017/Core/imkl/11.3.4.258/mkl/bin:/cvmfs/soft.computecanada.ca/easybuild/software/2017/Core/imkl/11.3.4.258/bin:/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/gcc-5.4.0/bin:/opt/software/bin:/opt/puppetlabs/puppet/bin:/opt/software/slurm/current/bin:/opt/software/slurm/bin:/cvmfs/soft.computecanada.ca/easybuild/bin:/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/16.09/bin:/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/16.09/sbin:/cvmfs/soft.computecanada.ca/custom/bin:/opt/software/slurm/current/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/yourUsername/.local/bin:/home/yourUsername/bin +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/software-dependencies.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/software-dependencies.snip new file mode 100644 index 00000000..adb026f2 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/software-dependencies.snip @@ -0,0 +1,94 @@ +To demonstrate, let's use `module list`. `module list` shows all loaded +software modules. + +``` +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` +Currently Loaded Modules: + 1) nixpkgs/.16.09 (H,S) 5) intel/2016.4 (t) + 2) icc/.2016.4.258 (H) 6) openmpi/2.1.1 (m) + 3) gcccore/.5.4.0 (H) 7) StdEnv/2016.4 (S) + 4) ifort/.2016.4.258 (H) 8) python/3.5.2 (t) + + Where: + S: Module is Sticky, requires --force to unload or purge + m: MPI implementations / Implémentations MPI + t: Tools for development / Outils de développement + H: Hidden Module +``` +{: .output} + +``` +{{ site.remote.prompt }} module load beast +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` +Currently Loaded Modules: + 1) nixpkgs/.16.09 (H,S) 5) intel/2016.4 (t) 9) java/1.8.0_121 (t) + 2) icc/.2016.4.258 (H) 6) openmpi/2.1.1 (m) 10) beagle-lib/2.1.2 (bio) + 3) gcccore/.5.4.0 (H) 7) StdEnv/2016.4 (S) 11) beast/2.4.0 (chem) + 4) ifort/.2016.4.258 (H) 8) python/3.5.2 (t) + + Where: + S: Module is Sticky, requires --force to unload or purge + bio: Bioinformatic libraries/apps / Logiciels de bioinformatique + m: MPI implementations / Implémentations MPI + t: Tools for development / Outils de développement + chem: Chemistry libraries/apps / Logiciels de chimie + H: Hidden Module +``` +{: .output} + +So in this case, loading the `beast` module (a bioinformatics software +package), also loaded `java/1.8.0_121` and `beagle-lib/2.1.2` as well. Let's +try unloading the `beast` package. + +``` +{{ site.remote.prompt }} module unload beast +{{ site.remote.prompt }} module list +``` +{: .language-bash} + +``` +Currently Loaded Modules: + 1) nixpkgs/.16.09 (H,S) 5) intel/2016.4 (t) + 2) icc/.2016.4.258 (H) 6) openmpi/2.1.1 (m) + 3) gcccore/.5.4.0 (H) 7) StdEnv/2016.4 (S) + 4) ifort/.2016.4.258 (H) 8) python/3.5.2 (t) + + Where: + S: Module is Sticky, requires --force to unload or purge + m: MPI implementations / Implémentations MPI + t: Tools for development / Outils de développement + H: Hidden Module +``` +{: .output} + +So using `module unload` "un-loads" a module along with its dependencies. If we +wanted to unload everything at once, we could run `module purge` (unloads +everything). + +``` +{{ site.remote.prompt }} module purge +``` +{: .language-bash} + +``` +The following modules were not unloaded: + (Use "module --force purge" to unload all): + + 1) StdEnv/2016.4 5) ifort/.2016.4.258 + 2) nixpkgs/.16.09 6) intel/2016.4 + 3) icc/.2016.4.258 7) imkl/11.3.4.258 + 4) gcccore/.5.4.0 8) openmpi/2.1.1 +``` +{: .output} + +Note that `module purge` is informative. It lets us know that all but a default +set of packages have been unloaded (and how to actually unload these if we +truly so desired). diff --git a/_includes/snippets_library/NeSI_Maui_slurm/modules/wrong-gcc-version.snip b/_includes/snippets_library/NeSI_Maui_slurm/modules/wrong-gcc-version.snip new file mode 100644 index 00000000..f63927c6 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/modules/wrong-gcc-version.snip @@ -0,0 +1,79 @@ +Let's take a closer look at the `gcc` module. GCC is an extremely widely used +C/C++/Fortran compiler. Tons of software is dependent on the GCC version, and +might not compile or run if the wrong version is loaded. In this case, there +are two different versions: `gcc/4.8.5` and `gcc/5.4.0`. How do we load each +copy and which copy is the default? + +In this case, `gcc/5.4.0` has a `(D)` next to it. This indicates that it is the +default — if we type `module load gcc`, this is the copy that will be +loaded. + +``` +{{ site.remote.prompt }} module load gcc +{{ site.remote.prompt }} gcc --version +``` +{: .language-bash} + +``` +Lmod is automatically replacing "intel/2016.4" with "gcc/5.4.0". + +Due to MODULEPATH changes, the following have been reloaded: + 1) openmpi/2.1.1 + +gcc (GCC) 5.4.0 +Copyright (C) 2015 Free Software Foundation, Inc. +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +``` +{: .output} + +Note that three things happened: the default copy of GCC was loaded (version +5.4.0), the Intel compilers (which conflict with GCC) were unloaded, and +software that is dependent on compiler (OpenMPI) was reloaded. The `module` +system turned what might be a super-complex operation into a single command. + +So how do we load the non-default copy of a software package? In this case, the +only change we need to make is be more specific about the module we are +loading. There are two GCC modules: `gcc/5.4.0` and `gcc/4.8.5`. To load a +non-default module, the only change we need to make to our `module load` +command is to leave in the version number after the `/`. + +``` +{{ site.remote.prompt }} module load gcc/4.8.5 +{{ site.remote.prompt }} gcc --version +``` +{: .language-bash} + +``` +Inactive Modules: + 1) openmpi + +The following have been reloaded with a version change: + 1) gcc/5.4.0 => gcc/4.8.5 + +gcc (GCC) 4.8.5 +Copyright (C) 2015 Free Software Foundation, Inc. +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +``` +{: .output} + +We now have successfully switched from GCC 5.4.0 to GCC 4.8.5. It is also +important to note that there was no compatible OpenMPI module available for GCC +4.8.5. Because of this, the `module` program has "inactivated" the module. All +this means for us is that if we re-load GCC 5.4.0, `module` will remember +OpenMPI used to be loaded and load that module as well. + +``` +{{ site.remote.prompt }} module load gcc/5.4.0 +``` +{: .language-bash} + +``` +Activating Modules: + 1) openmpi/2.1.1 + +The following have been reloaded with a version change: + 1) gcc/4.8.5 => gcc/5.4.0 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/parallel/four-tasks-jobscript.snip b/_includes/snippets_library/NeSI_Maui_slurm/parallel/four-tasks-jobscript.snip new file mode 100644 index 00000000..2629f51e --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/parallel/four-tasks-jobscript.snip @@ -0,0 +1,15 @@ +``` +{{ site.remote.bash_shebang }} +{{ site.sched.comment }} {{ site.sched.flag.name }} parallel-pi +{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} +{{ site.sched.comment }} --nodes 1 +{{ site.sched.comment }} --ntasks 4 +{{ site.sched.comment }} --mem=3G + +# Load the computing environment we need +module load Python/3.8.2-gimkl-2020a + +# Execute the task +srun python pi.py 100000000 +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/parallel/one-task-with-memory-jobscript.snip b/_includes/snippets_library/NeSI_Maui_slurm/parallel/one-task-with-memory-jobscript.snip new file mode 100644 index 00000000..cde991af --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/parallel/one-task-with-memory-jobscript.snip @@ -0,0 +1,13 @@ +``` +{{ site.remote.bash_shebang }} +{{ site.sched.comment }} {{ site.sched.flag.name }} serial-pi +{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} +{{ site.sched.comment }} --mem=3G + +# Load the computing environment we need +module load Python/3.8.2-gimkl-2020a + +# Execute the task +python pi.py 100000000 +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/resources/account-history.snip b/_includes/snippets_library/NeSI_Maui_slurm/resources/account-history.snip new file mode 100644 index 00000000..81b8e0b4 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/resources/account-history.snip @@ -0,0 +1,11 @@ +``` +JobID JobName Partition Account AllocCPUS State ExitCode +------------ ---------- ---------- ---------- ---------- ---------- -------- +1964 bash standard default 1 COMPLETED 0:0 +1964.extern extern default 1 COMPLETED 0:0 +1964.0 bash default 1 COMPLETED 0:0 +1965 build-ind+ summer-sc+ default 1 COMPLETED 0:0 +1965.batch batch default 1 COMPLETED 0:0 +1965.extern extern default 1 COMPLETED 0:0 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/resources/monitor-processes-top.snip b/_includes/snippets_library/NeSI_Maui_slurm/resources/monitor-processes-top.snip new file mode 100644 index 00000000..12685735 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/resources/monitor-processes-top.snip @@ -0,0 +1,19 @@ +``` +top - 21:00:19 up 3:07, 1 user, load average: 1.06, 1.05, 0.96 +Tasks: 311 total, 1 running, 222 sleeping, 0 stopped, 0 zombie +%Cpu(s): 7.2 us, 3.2 sy, 0.0 ni, 89.0 id, 0.0 wa, 0.2 hi, 0.2 si, 0.0 st +KiB Mem : 16303428 total, 8454704 free, 3194668 used, 4654056 buff/cache +KiB Swap: 8220668 total, 8220668 free, 0 used. 11628168 avail Mem + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1693 jeff 20 0 4270580 346944 171372 S 29.8 2.1 9:31.89 gnome-shell + 3140 jeff 20 0 3142044 928972 389716 S 27.5 5.7 13:30.29 Web Content + 3057 jeff 20 0 3115900 521368 231288 S 18.9 3.2 10:27.71 firefox + 6007 jeff 20 0 813992 112336 75592 S 4.3 0.7 0:28.25 tilix + 1742 jeff 20 0 975080 164508 130624 S 2.0 1.0 3:29.83 Xwayland + 1 root 20 0 230484 11924 7544 S 0.3 0.1 0:06.08 systemd + 68 root 20 0 0 0 0 I 0.3 0.0 0:01.25 kworker/4:1 + 2913 jeff 20 0 965620 47892 37432 S 0.3 0.3 0:11.76 code + 2 root 20 0 0 0 0 S 0.0 0.0 0:00.02 kthreadd +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/resources/system-memory-free.snip b/_includes/snippets_library/NeSI_Maui_slurm/resources/system-memory-free.snip new file mode 100644 index 00000000..ec4c0d3f --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/resources/system-memory-free.snip @@ -0,0 +1,6 @@ +``` +total used free shared buff/cache available +Mem: 3.8G 1.5G 678M 327M 1.6G 1.6G +Swap: 3.9G 170M 3.7G +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-script.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-script.snip new file mode 100644 index 00000000..e31c6fc0 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-script.snip @@ -0,0 +1,4 @@ +``` +Submitted batch job 36855 +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-status.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-status.snip new file mode 100644 index 00000000..9d407e42 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/basic-job-status.snip @@ -0,0 +1,9 @@ +``` +JOBID USER ACCOUNT NAME ST REASON START_TIME T... +36856 yourUsername yourAccount example-job.sh R None 2017-07-01T16:47:02 ... +``` +{: .output} + +We can see all the details of our job, most importantly that it is in the `R` +or `RUNNING` state. Sometimes our jobs might need to wait in a queue +(`PENDING`) or have an error (`E`). diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/job-with-name-status.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/job-with-name-status.snip new file mode 100644 index 00000000..dc1fda51 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/job-with-name-status.snip @@ -0,0 +1,5 @@ +``` +JOBID USER ACCOUNT NAME ST REASON START_TIME TIME TIME_LEF... +38191 yourUsername yourAccount new_name PD Priority N/A 0:00 1:00:00 ... +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/option-flags-list.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/option-flags-list.snip new file mode 100644 index 00000000..5e80b164 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/option-flags-list.snip @@ -0,0 +1,15 @@ +* `--ntasks=` or `-n `: How many CPU cores does your job need, + in total? + +* `--time ` or `-t `: + How much real-world time (walltime) will your job take to run? The `` + part can be omitted. + +* `--mem=`: How much memory on a node does your job need in + megabytes? You can also specify gigabytes using by adding a little "g" + afterwards (example: `--mem=5g`) + +* `--nodes=` or `-N `: How many separate machines does your job + need to run on? Note that if you set `ntasks` to a number greater than what + one machine can offer, {{ site.sched.name }} will set this value + automatically. diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/print-sched-variables.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/print-sched-variables.snip new file mode 100644 index 00000000..e2f68c29 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/print-sched-variables.snip @@ -0,0 +1,30 @@ +> ## Job environment variables +> +> When {{ site.sched.name }} runs a job, it sets a number of environment +> variables for the job. One of these will let us check what directory our job +> script was submitted from. The `SLURM_SUBMIT_DIR` variable is set to the +> directory from which our job was submitted. Using the `SLURM_SUBMIT_DIR` +> variable, modify your job so that it prints out the location from which the +> job was submitted. +> +> > ## Solution +> > +> > ``` +> > {{ site.remote.prompt }} nano example-job.sh +> > {{ site.remote.prompt }} cat example-job.sh +> > ``` +> > {: .language-bash} +> > +> > ``` +> > {{ site.remote.bash_shebang }} +> > #SBATCH --time 00:00:30 +> > +> > echo -n "This script is running on " +> > hostname +> > +> > echo "This job was launched in the following directory:" +> > echo ${SLURM_SUBMIT_DIR} +> > ``` +> > {: .output} +> {: .solution} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-job.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-job.snip new file mode 100644 index 00000000..875df1c3 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-job.snip @@ -0,0 +1,4 @@ +``` +{{ site.remote.prompt }} cat slurm-38193.out +``` +{: .language-bash} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-output.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-output.snip new file mode 100644 index 00000000..b82e79f5 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/runtime-exceeded-output.snip @@ -0,0 +1,7 @@ +``` +This job is running on: +gra533 +slurmstepd: error: *** JOB 38193 ON gra533 CANCELLED AT 2017-07-02T16:35:48 +DUE TO TIME LIMIT *** +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-begin.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-begin.snip new file mode 100644 index 00000000..6abcea80 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-begin.snip @@ -0,0 +1,7 @@ +``` +Submitted batch job 38759 + +JOBID USER ACCOUNT NAME ST REASON TIME TIME_LEFT NOD... +38759 yourUsername yourAccount example-job.sh PD Priority 0:00 1:00 1 ... +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-cancel.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-cancel.snip new file mode 100644 index 00000000..96caec62 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-job-cancel.snip @@ -0,0 +1,4 @@ +``` +JOBID USER ACCOUNT NAME ST REASON START_TIME TIME TIME_LEFT NODES CPUS +``` +{: .output} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-multiple-jobs.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-multiple-jobs.snip new file mode 100644 index 00000000..3fd18f83 --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/terminate-multiple-jobs.snip @@ -0,0 +1,27 @@ +> ## Cancelling multiple jobs +> +> We can also cancel all of our jobs at once using the `-u` option. This will +> delete all jobs for a specific user (in this case, yourself). Note that you +> can only delete your own jobs. +> +> Try submitting multiple jobs and then cancelling them all. +> +> > ## Solution +> > +> > First, submit a trio of jobs: +> > +> > ``` +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sh +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sh +> > {{ site.remote.prompt }} {{ site.sched.submit.name }} {{ site.sched.submit.options }} example-job.sh +> > ``` +> > {: .language-bash} +> > +> > Then, cancel them all: +> > +> > ``` +> > {{ site.remote.prompt }} {{ site.sched.del }} --user yourUsername +> > ``` +> > {: .language-bash} +> {: .solution} +{: .challenge} diff --git a/_includes/snippets_library/NeSI_Maui_slurm/scheduler/using-nodes-interactively.snip b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/using-nodes-interactively.snip new file mode 100644 index 00000000..153b70fe --- /dev/null +++ b/_includes/snippets_library/NeSI_Maui_slurm/scheduler/using-nodes-interactively.snip @@ -0,0 +1,69 @@ +`{{ site.sched.interactive }}` runs a single command on the cluster and then +exits. Let's demonstrate this by running the `hostname` command with `{{ +site.sched.interactive }}`. (We can cancel an `{{ site.sched.interactive }}` +job with `Ctrl-c`.) + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} hostname +``` +{: .language-bash} + +``` +gra752 +``` +{: .output} + +`{{ site.sched.interactive }}` accepts all of the same options as `{{ +site.sched.submit.name }}`. However, instead of specifying these in a script, +these options are specified on the command-line when starting a job. To submit +a job that uses 2 CPUs for instance, we could use the following command: + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} --ntasks 2 echo "This job will use 2 CPUs." +``` +{: .language-bash} + +``` +This job will use 2 CPUs. +This job will use 2 CPUs. +``` +{: .output} + +Typically, the resulting shell environment will be the same as that for +`{{ site.sched.submit.name }}`. + +### Interactive jobs + +Sometimes, you will need a lot of resource for interactive use. Perhaps it's +our first time running an analysis or we are attempting to debug something that +went wrong with a previous job. Fortunately, {{ site.sched.name }} makes it +easy to start an interactive job with `{{ site.sched.interactive }}`: + +``` +{{ site.remote.prompt }} {{ site.sched.interactive }} --pty bash +``` +{: .language-bash} + +You should be presented with a bash prompt. Note that the prompt will likely +change to reflect your new location, in this case the compute node we are +logged on. You can also verify this with `hostname`. + +> ## Creating remote graphics +> +> To see graphical output inside your jobs, you need to use X11 forwarding. To +> connect with this feature enabled, use the `-Y` option when you login with +> the `ssh` command, e.g., `ssh -Y yourUsername@{{ site.remote.login }}`. +> +> To demonstrate what happens when you create a graphics window on the remote +> node, use the `xeyes` command. A relatively adorable pair of eyes should pop +> up (press `Ctrl-C` to stop). If you are using a Mac, you must have installed +> XQuartz (and restarted your computer) for this to work. +> +> If your cluster has the +> [slurm-spank-x11](https://github.com/hautreux/slurm-spank-x11) plugin +> installed, you can ensure X11 forwarding within interactive jobs by using the +> `--x11` option for `{{ site.sched.interactive }}` with the command +> `{{ site.sched.interactive }} --x11 --pty bash`. +{: .callout} + +When you are done with the interactive job, type `exit` to quit your session. diff --git a/_includes/syllabus.html b/_includes/syllabus.html new file mode 100644 index 00000000..e7f08b4c --- /dev/null +++ b/_includes/syllabus.html @@ -0,0 +1,120 @@ +{% include base_path.html %} + +{% comment %} +Display syllabus in tabular form. +Days are displayed if at least one episode has 'start = true'. +{% endcomment %} + +{% include manual_episode_order.html %} + +
+

Schedule

+ + {% assign lesson_number = 0 %} + {% assign day = 0 %} + {% assign multiday = false %} + {% for lesson_episode in lesson_episodes %} + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} + {% if episode.start %}{% assign multiday = true %}{% break %}{% endif %} + {% endfor %} + {% assign current = site.start_time %} + + + {% if site.instructor_training %} + + {% if multiday %}{% endif %} + + + + + {% endif %} + + {% for lesson_episode in lesson_episodes %} + {% if site.episode_order %} + {% assign episode = site.episodes | where: "slug", lesson_episode | first %} + {% else %} + {% assign episode = lesson_episode %} + {% endif %} + {% if episode.start %} {% comment %} Starting a new day? {% endcomment %} + {% assign day = day | plus: 1 %} + {% if day > 1 %} {% comment %} If about to start day 2 or later, show finishing time for previous day {% + endcomment %} + {% assign hours = current | divided_by: 60 %} + {% assign minutes = current | modulo: 60 %} + + {% if multiday %}{% endif %} + + + + + {% endif %} + {% assign current = site.start_time %} {% comment %}Re-set start time of this episode to general daily start + time {% endcomment %} + {% endif %} + {% assign hours = current | divided_by: 60 %} + {% assign minutes = current | modulo: 60 %} + + {% if multiday %}{% endif %} + + + + + {% assign current = current | plus: episode.teaching | plus: episode.exercises | plus: episode.break %} + {% endfor %} + {% assign hours = current | divided_by: 60 %} + {% assign minutes = current | modulo: 60 %} + {% if site.instructor_training %} + + {% if multiday %}{% endif %} + + + + + {% assign current = current | plus: 15 %} + {% assign hours = current | divided_by: 60 %} + {% assign minutes = current | modulo: 60 %} + {% endif %} + + {% if multiday %}{% endif %} + + + + +
Pre-training + surveyPlease fill out our pre-training survey before the start of the course.
Setup
{% if hours < 10 %}0{% endif %}{{ hours + }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
{% if episode.start %}Day {{ day }}{% endif %}{% if hours < 10 %}0{% endif %}{{ hours + }}:{% if minutes < 10 %}0{% endif %}{{ minutes }} + {% if episode.break %} + {{ episode.title }} + {% else %} + {% assign lesson_number = lesson_number | plus: 1 %} + {{ lesson_number }}. {{ episode.title }} + {% endif %} + + {% if episode.break %} + Break + {% else %} + {% if episode.questions %} + {% for question in episode.questions %} + {{question|markdownify|strip_html}} + {% unless forloop.last %} +
+ {% endunless %} + {% endfor %} + {% endif %} + {% endif %} +
{% if hours < 10 %}0{% endif %}{{ hours + }}:{% if minutes < 10 %}0{% endif +++ %}{{ minutes }}Post-training survey + Please fill out our post-training survey after the course.
{% if hours < 10 %}0{% endif %}{{ hours + }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
+ +

+ The actual schedule may vary slightly depending on the topics and exercises chosen by the instructor. +

+ +
\ No newline at end of file diff --git a/_make.sh b/_make.sh new file mode 100755 index 00000000..dd5e593a --- /dev/null +++ b/_make.sh @@ -0,0 +1 @@ +HPC_JEKYLL_CONFIG=_includes/snippets_library/NeSI_Mahuika_slurm/_config_options.yml make serve diff --git a/bin/lesson_check.py b/bin/lesson_check.py index fa9a1c73..fc7e4e27 100755 --- a/bin/lesson_check.py +++ b/bin/lesson_check.py @@ -35,7 +35,6 @@ 'LICENSE.md': True, 'MAINTENANCE.md': False, 'README.md': False, - os.path.join('_extras', 'discuss.md'): True, os.path.join('_extras', 'guide.md'): True, 'index.md': True, 'reference.md': True, diff --git a/bin/lesson_initialize.py b/bin/lesson_initialize.py index 65079c1f..a29e2a9d 100755 --- a/bin/lesson_initialize.py +++ b/bin/lesson_initialize.py @@ -20,6 +20,7 @@ '_extras/discuss.md', '_extras/figures.md', '_extras/guide.md', + '_extras/command-history.md', 'aio.md', 'index.md', 'reference.md', diff --git a/fig/883px-Jupyter_logo.svg.png b/fig/883px-Jupyter_logo.svg.png new file mode 100644 index 00000000..74326498 Binary files /dev/null and b/fig/883px-Jupyter_logo.svg.png differ diff --git a/fig/AmdahlsLaw.svg b/fig/AmdahlsLaw.svg new file mode 100644 index 00000000..ed506d13 --- /dev/null +++ b/fig/AmdahlsLaw.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/AmdahlsLaw2.svg b/fig/AmdahlsLaw2.svg new file mode 100644 index 00000000..c0f64c13 --- /dev/null +++ b/fig/AmdahlsLaw2.svg @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +20 +18 +16 +14 +12 +10 +8 +6 +4 +2 +0 +Amdahl's Law +Parallel portion +Number of processors +50% +75% +90% +95% + + +Speedup +1 +2 +4 +8 +16 +32 +64 +128 +256 +512 +1024 +2048 +4096 +8192 +16384 +32768 +65536 + + + diff --git a/fig/DimReturns.svg b/fig/DimReturns.svg new file mode 100644 index 00000000..3af91b77 --- /dev/null +++ b/fig/DimReturns.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/DimReturns2.png b/fig/DimReturns2.png new file mode 100644 index 00000000..14446dd0 Binary files /dev/null and b/fig/DimReturns2.png differ diff --git a/fig/Login_jupyterhubNeSI.png b/fig/Login_jupyterhubNeSI.png new file mode 100644 index 00000000..d0ca0542 Binary files /dev/null and b/fig/Login_jupyterhubNeSI.png differ diff --git a/fig/NesiFiletree.svg b/fig/NesiFiletree.svg new file mode 100644 index 00000000..ffaaee58 --- /dev/null +++ b/fig/NesiFiletree.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/ServerOptions_jupyterhubNeSI.png b/fig/ServerOptions_jupyterhubNeSI.png new file mode 100644 index 00000000..9d5dbfa1 Binary files /dev/null and b/fig/ServerOptions_jupyterhubNeSI.png differ diff --git a/fig/Unix_Command_Struc.svg b/fig/Unix_Command_Struc.svg new file mode 100644 index 00000000..8b118e6c --- /dev/null +++ b/fig/Unix_Command_Struc.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/UsingJupyterHub1.svg b/fig/UsingJupyterHub1.svg new file mode 100644 index 00000000..1a895db7 --- /dev/null +++ b/fig/UsingJupyterHub1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/UsingJupyterHub2.svg b/fig/UsingJupyterHub2.svg new file mode 100644 index 00000000..3abf6001 --- /dev/null +++ b/fig/UsingJupyterHub2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/UsingJupyterHub3.svg b/fig/UsingJupyterHub3.svg new file mode 100644 index 00000000..d879f41c --- /dev/null +++ b/fig/UsingJupyterHub3.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/clusterDiagram.png b/fig/clusterDiagram.png new file mode 100644 index 00000000..a7efa81b Binary files /dev/null and b/fig/clusterDiagram.png differ diff --git a/fig/correct-spreadsheet-entry.png b/fig/correct-spreadsheet-entry.png new file mode 100644 index 00000000..6a69be13 Binary files /dev/null and b/fig/correct-spreadsheet-entry.png differ diff --git a/fig/cpu_eff.svg b/fig/cpu_eff.svg new file mode 100644 index 00000000..fbeb16a5 --- /dev/null +++ b/fig/cpu_eff.svg @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + Compute Efficiency = + 100 + TotalCPU + Alloc + Elapsed + + x + + + x + + + + diff --git a/fig/jupyterLauncher.png b/fig/jupyterLauncher.png new file mode 100644 index 00000000..ecaeb614 Binary files /dev/null and b/fig/jupyterLauncher.png differ diff --git a/fig/jupyter_ftw.png b/fig/jupyter_ftw.png new file mode 100644 index 00000000..e09eb0b3 Binary files /dev/null and b/fig/jupyter_ftw.png differ diff --git a/fig/mem_eff.svg b/fig/mem_eff.svg new file mode 100644 index 00000000..8a4e85ba --- /dev/null +++ b/fig/mem_eff.svg @@ -0,0 +1,135 @@ + + + + + + + + + + + + Memory Efficiency = + 100 + MaxRSS + ReqMem + + + x + + + diff --git a/fig/parts_slurm_script.svg b/fig/parts_slurm_script.svg new file mode 100644 index 00000000..13ac868b --- /dev/null +++ b/fig/parts_slurm_script.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/fig/shell_command_syntax.svg b/fig/shell_command_syntax.svg new file mode 100644 index 00000000..a32b8f92 --- /dev/null +++ b/fig/shell_command_syntax.svg @@ -0,0 +1,47 @@ + + + + + + + diff --git a/fig/time_eff.svg b/fig/time_eff.svg new file mode 100644 index 00000000..1b195df5 --- /dev/null +++ b/fig/time_eff.svg @@ -0,0 +1,135 @@ + + + + + + + + + + + + Time Efficiency = + 100 + TimeLimit + Elapsed + + + x + + + diff --git a/files/jargon.html b/files/jargon.html index 11d35eeb..6a47ef45 100644 --- a/files/jargon.html +++ b/files/jargon.html @@ -70,7 +70,7 @@ # Shared Computing Resources .center[ -![An HPC resource (img: [Julian Herzog](https://commons.wikimedia.org/wiki/File:High_Performance_Computing_Center_Stuttgart_HLRS_2015_07_Cray_XC40_Hazel_Hen_IO.jpg))](../fig/HPCCStuttgart_Hazel_Hen_XC40.png) +![An HPC resource (img: [Julian Herzog](https://commons.wikimedia.org/wiki/File:High_Performance_Computing_Center_Stuttgart_HLRS_2015_07_Cray_XC40_Hazel_Hen_IO.jpg))](/fig/HPCCStuttgart_Hazel_Hen_XC40.png) ] @@ -146,10 +146,14 @@ ??? -A collection of standalone computers that are networked together. They will -frequently have software installed that allow the coordinated running of other -software across all of these computers. This allows these networked computers -to work together to accomplish computing tasks faster. +Cloud systems physically resemble clusters or supercomputers, but +with greater heterogeneity among components, and less coupling, often +without an interconnect at all. Cloud servers are generally accessed +by connecting to their services over the network. Multiple virtual +machines may reside on individual physical machines. Typical tasks +are driven by user connections from the outside to the various "front +end" services (e.g. web sites) which pass traffic on to the "back +end" servers (e.g. databases) to present results to users. --- diff --git a/index.md b/index.md index 449087eb..30b7dd0f 100644 --- a/index.md +++ b/index.md @@ -4,44 +4,39 @@ root: . --- {% include gh_variables.html %} +{% assign hours = site.hpc_start_time | divided_by: 60 %} +{% assign minutes = site.hpc_start_time | modulo: 60 %} +{% assign late_start_episode = site.episodes | where: "slug", site.hpc_start_lesson | first %} +The first half of this workshop will cover basics of command line (in an HPC environment). -This workshop is an introduction to using high-performance computing systems -effectively. We can't cover every case or give an exhaustive course on parallel -programming in just two days' teaching time. Instead, this workshop is intended -to give students a good introduction and overview of the tools available and -how to use them effectively. +The second half of this workshop will provide an introduction and overview to the tools available on high-performance computing systems and how to use them effectively. + +[//]: If you are already comfortable with command line and wish to skip the first half you may join for the [{{late_start_episode.title}}]({{site.url}}/{{site.hpc_start_lesson}}) at {% if hours < 10 %}0{% endif %}{{ hours}}:{% if minutes < 10 %}0{% endif %}{{ minutes }} > ## Prerequisites > -> Command line experience is necessary for this lesson. We recommend the -> participants to go through -> [shell-novice](https://swcarpentry.github.io/shell-novice/), if new to the -> command line (also known as terminal or shell). +> While we cover the bash essentials, we still recommend checking out the +> [shell-novice](https://swcarpentry.github.io/shell-novice/) lesson for a better foundation in bash. {: .prereq} By the end of this workshop, students will know how to: * Identify problems a cluster can help solve -* Use the UNIX shell (also known as terminal or command line) to - connect to a cluster. -* Transfer files onto a cluster. -* Submit and manage jobs on a cluster using a scheduler. -* Observe the benefits and limitations of parallel execution. +* Use the UNIX shell (also known as terminal or command line) to operate a + computer, connect to a cluster, and write simple shell scripts. +* Submit and manage jobs on a cluster using a scheduler, transfer files, and +* Use software through environment modules. +* Review and optimise job resource usage. > ## Getting Started > > To get started, follow the directions in the "[Setup]( -> {{ page.root }}/setup.html)" tab to download data to your computer and follow -> any installation instructions. +> {{ page.root }}/setup.html)" tab and follow any installation instructions. {: .callout} Note that this is the draft HPC Carpentry release. Comments and feedback are welcome. -> ## For Instructors -> -> If you are teaching this lesson in a workshop, please see the -> [Instructor notes](guide/). -{: .callout} +This is a hands on workshop, excersises will generally depend on the previous steps having been completed, so it is important to keep up! {% include links.md %} diff --git a/setup.md b/setup.md index 35aa55dd..e1123f8d 100644 --- a/setup.md +++ b/setup.md @@ -4,139 +4,45 @@ title: Setup root: . --- -There are several pieces of software you will wish to install before the -workshop. Though installation help will be provided at the workshop, we -recommend that these tools are installed (or at least downloaded) beforehand. - -1. [A terminal application or command-line interface]( - #where-to-type-commands-how-to-open-a-new-shell) -2. [A Secure Shell application](#ssh-for-secure-connections) - -> ## Bash and SSH -> -> This lesson requires a terminal application (`bash`, `zsh`, or others) with -> the ability to securely connect to a remote machine (`ssh`). -{: .prereq} - -## Where to Type Commands: How to Open a New Shell - -The shell is a program that enables us to send commands to the computer and -receive output. It is also referred to as the terminal or command line. - -Some computers include a default Unix Shell program. The steps below describe -some methods for identifying and opening a Unix Shell program if you already -have one installed. There are also options for identifying and downloading a -Unix Shell program, a Linux/UNIX emulator, or a program to access a Unix Shell -on a server. - -### Unix Shells on Windows -Computers with Windows operating systems do not automatically have a Unix Shell -program installed. In this lesson, we encourage you to use an emulator included -in Git for Windows, which gives you access to both Bash shell commands and Git. -If you have attended a Software Carpentry workshop session, it is likely you -have already received instructions on how to install Git for Windows. - -Once installed, you can open a terminal by running the program Git Bash from -the Windows start menu. - -#### Shell Programs for Windows - -* [Git for Windows][git4win] -- *Recommended* -* [Windows Subsystem for Linux][ms-wsl] -- advanced option for Windows 10 - -> ## Alternatives to Git for Windows -> -> Other solutions are available for running Bash commands on Windows. There is -> now a Bash shell command-line tool available for Windows 10. Additionally, -> you can run Bash commands on a remote computer or server that already has a -> Unix Shell, from your Windows machine. This can usually be done through a -> Secure Shell (SSH) client. One such client available for free for Windows -> computers is [PuTTY][putty]. See the reference below for information on -> installing and using PuTTY, using the Windows 10 command-line tool, or -> installing and using a Unix/Linux emulator. -> -> For advanced users, you may choose one of the following alternatives: -> -> * Install the [Windows Subsystem for Linux][ms-wsl] -> * Use the Windows [PowerShell][ms-shell] -> * Read up on [Using a Unix/Linux emulator][unix-emulator] (Cygwin) or Secure -> Shell (SSH) client (PuTTY) -> -> > ## Warning -> > -> > Commands in the Windows Subsystem for Linux (WSL), PowerShell, or Cygwin -> > may differ slightly from those shown in the lesson or presented in the -> > workshop. Please ask if you encounter such a mismatch -- you're -> > probably not alone. -> {: .challenge} -{: .discussion} - -### Unix Shell on macOS - -On macOS, the default Unix Shell is accessible by running the Terminal program -from the `/Application/Utilities` folder in Finder. - -To open Terminal, try one or both of the following: - -* In Finder, select the Go menu, then select Utilities. Locate Terminal in the - Utilities folder and open it. -* Use the Mac ‘Spotlight’ computer search function. Search for: `Terminal` and - press Return. - -For an introduction, see [How to Use Terminal on a Mac][mac-terminal]. - -### Unix Shell on Linux - -On most versions of Linux, the default Unix Shell is accessible by running the -[(Gnome) Terminal](https://help.gnome.org/users/gnome-terminal/stable/) or -[(KDE) Konsole](https://konsole.kde.org/) or -[xterm](https://en.wikipedia.org/wiki/Xterm), which can be found via the -applications menu or the search bar. - -### Special Cases - -If none of the options above address your circumstances, try an online search -for: `Unix shell [your operating system]`. - -## SSH for Secure Connections - -All students should have an SSH client installed. SSH is a tool that allows us -to connect to and use a remote computer as our own. - -### SSH for Windows - -Git for Windows comes with SSH preinstalled: you do not have to do anything. - -> ## GUI Support for Windows -> -> If you know that the software you will be running on the cluster requires a -> graphical user interface (a GUI window needs to open for the application to -> run properly), please install [MobaXterm](https://mobaxterm.mobatek.net) Home -> Edition. -{: .discussion} - -### SSH for macOS - -macOS comes with SSH pre-installed: you do not have to do anything. - -> ## GUI Support for macOS -> -> If you know that the software you will be running requires a graphical user -> interface, please install [XQuartz](https://www.xquartz.org). -{: .discussion} - -### SSH for Linux - -Linux comes with SSH and X window support preinstalled: you do not have to do -anything. - - -[git4win]: https://gitforwindows.org/ -[mac-terminal]: https://www.macworld.co.uk/feature/mac-software/how-use-terminal-on-mac-3608274/ -[ms-wsl]: https://docs.microsoft.com/en-us/windows/wsl/install-win10 -[ms-shell]: https://docs.microsoft.com/en-us/powershell/scripting/learn/remoting/ssh-remoting-in-powershell-core?view=powershell-7 -[mobax-gen]: https://mobaxterm.mobatek.net/documentation.html -[putty]: https://www.chiark.greenend.org.uk/~sgtatham/putty/ -[unix-emulator]: https://www.cygwin.com/ -[wsl]: https://docs.microsoft.com/en-us/windows/wsl/install-win10 +## NeSI JupyterHub Login + +The easiest method for accessing the NeSI cluster is to use our JupyterHub service. Below are the +login and troubleshooting instructions for NeSI JupyterHub: + +1. Follow this link: [https://jupyter.nesi.org.nz](https://jupyter.nesi.org.nz) +2. Enter your NeSI username, HPC password your 6 digit second factor token ![Login](/fig/Login_jupyterhubNeSI.png) +3. Choose server options: the session project code should be *NeSI Training ({{site.sched.project}})*, Number of CPUs and memory size will remain unchanged. However, select the appropriate **Wall time** based on the projected length of a session ![Options](/fig/ServerOptions_jupyterhubNeSI.png) +4. From Jupyter Launcher screen, choose Terminal (highlighted in red box) ![Terminal](/fig/jupyterLauncher.png) + +
+ +## Accessing the Cluster and Running Jobs + +### Jupyter for Interactive work. + +![Terminal](/fig/UsingJupyterHub2.svg) +In a web browser, navigate to [https://jupyter.nesi.org.nz](https://jupyter.nesi.org.nz), select the resource requirements *for your job*. Jobs are run interactively. + +**Best For:** New Users, Job and code tests and Jobs using < 4 CPUs and < 128GB + +**Advantages:** Interactive, file explorer, no local setup required. + +### Jupyter and SBATCH + + +![Terminal](/fig/UsingJupyterHub3.svg) +In a web browser, navigate to [https://jupyter.nesi.org.nz](https://jupyter.nesi.org.nz), select the resource requirements *for your session* (should only need minimal memory and CPU). Jobs scripts are submitted using the `sbatch` command (non-interactive). + +**Best For:** New Users, Windows Users. + +**Advantages:** File explorer, no local setup required. + +### SSH and SBATCH + +![Terminal](/fig/UsingJupyterHub1.svg) +From your local computer, using an SSH client to connect to a shell session (interactive), running on the NeSI login Node. Jobs scripts are submitted using the `sbatch` command (non-interactive). Instructions for SSH and command-line setup can be found in our documentation: [Accessing the HPCs](https://support.nesi.org.nz/hc/en-gb/sections/360000034315) + +**Best For:** Users familiar with command line, Linux/Mac users. + +**Advantages:** Most flexible.