module load anaconda3/2022.10

source activate TorchA100

Minimal docker images are kept here:
/sw/Containers/docker

mkdir -p /export/home/snumber/sw/Containers
cd /export/home/snumber/sw/Containers
Convert the tarball to a Singularity image. 
module load singularity/4.1.3 
singularity build --sandbox pytorch docker-archive:///sw/Containers/docker/pytorchnew.tar
(it takes quite sometimes to create the sandbox. Please be patient. You only have to do this once!)

for testing:
singularity shell -e -B /scratch/snumber:/scratch/snumber -B /export/home/snumber:/export/home/snumber /export/home/snumber/sw/Containers/pytorch

e.g:
Singularity> python
Python 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.cuda.is_available()
True

So we see it works!

So now you need to integrate this into a slurm or pbs script. 
PBS examples are here



Slurm would be like this:

create a run script:
mkdir /export/home/snumber/slurm/data
create a file /export/home/snumber/slurm/data/myrun1.sh

cat  /export/home/snumber/slurm/data/myrun1.sh
>>>>>>>>>
cd /export/home/snumber/slurm/data
python /export/home/snumber/slurm/data/isCuda.py
>>>>>>>>>>

make it executable:
chmod +x  /export/home/snumber/slurm/data/myrun1.sh

Here is the content of the slurm.01 file

>>>>cat slurm.02 >>>>>>>>>>>>> 
#!/bin/bash 

####SBATCH --account=def-yoursnumber 

#SBATCH --job-name=helloWord 

#SBATCH --cpus-per-task=1 

#SBATCH --mem-per-cpu=1500MB 

#SBATCH --gres=gpu:a100:1 

###SBATCH --qos=work 

#SBATCH --qos=work 

###SBATCH --mem=4000M               # memory per node 

#SBATCH --time=0-03:00

module load singularity/4.1.3

##./program                         # you can use 'nvidia-smi' for a test

singularity exec -e -B /scratch:/scratch -B /export/home/snumber:/export/home/snumber /export/home/snumber/sw/Containers/pytorch "data/myrun1.sh"

>>>>>>>>>>>

To submit the slurm job:
sbatch  slurm.02


Alternatively, to run interactively:
srun --export=PATH,TERM,HOME,LANG  --job-name=hello_word --cpus-per-task=1 --mem-per-cpu=1500MB --time=1:00:00 --qos=work --gres=gpu:a100:1 --pty /bin/bash -l


singularity shell -e -B /scratch/snumber:/scratch/snumber -B /export/home/snumber:/export/home/snumber /export/home/snumber/sw/Containers/pytorch
cd data
sh myrun1.sh 






>>>Some errors you can ignore while building the sandbox>>>
INFO:    Starting build...
INFO:    Fetching OCI image...
INFO:    Extracting OCI image...
2024/05/20 12:37:31  warn rootless{usr/local/nvm/versions/node/v16.20.2/bin/corepack} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:37:32  warn rootless{usr/local/nvm/versions/node/v16.20.2/bin/npm} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:37:32  warn rootless{usr/local/nvm/versions/node/v16.20.2/bin/npx} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"

2024/05/20 12:38:17  warn rootless{usr/lib/libarrow.so} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:17  warn rootless{usr/lib/libarrow.so.1400} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:17  warn rootless{usr/lib/libarrow_acero.so} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:17  warn rootless{usr/lib/libarrow_acero.so.1400} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:17  warn rootless{usr/lib/libarrow_dataset.so} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:17  warn rootless{usr/lib/libarrow_dataset.so.1400} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:34  warn rootless{usr/lib/libparquet.so} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
2024/05/20 12:38:34  warn rootless{usr/lib/libparquet.so.1400} ignoring (usually) harmless EPERM on setxattr "user.rootlesscontainers"
INFO:    Inserting Singularity configuration...
INFO:    Creating sandbox directory...
INFO:    Build complete: pytorch
>>>>

#Go into an interactive run first
srun --export=PATH,TERM,HOME,LANG  --job-name=hello_word --cpus-per-task=1 --mem-per-cpu=60GB --time=19:00:00 --qos=work --gres=gpu:a100_1g.10gb:1  --pty /bin/bash -l
source /usr/local/bin/s3proxy.sh
module load  cuda/12.1
module  load anaconda3/2024.02
conda create --name pytorchCuda121 pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
source activate pytorchCuda121

We do not recommend installing miniconda as we provide conda as a module.
module load anaconda3/2024.02 

If you have installed it, comment out the miniconda section from ~./bashrc which can mess with other non-miniconda env. 
After commenting it out re-log back in.

Here is how you can test pytorch on your home dir for cuda availability:

To test this, please do this;
cd ~/slurm 
There are a couple of sample scripts to run slurm jobs

We will run an interactive job to troubleshoot this problem:
For example:
srun --export=PATH,TERM,HOME,LANG  --job-name=hello_word --cpus-per-task=1 --mem-per-cpu=50GB --time=1:00:00 --qos=work --gres=gpu:a100:1 --pty /bin/bash -l

it would put you inside a job.

squeue
             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
            <snip>
              1718    LocalQ    myrun s5284664  R    4:17:59      1 dgxlogin


Once inside a job, check if it detects gpu in pytorch app. (if already installed)

module load anaconda3/2024.02 
conda info --envs
# conda environments:
#
base                  *  /export/home/s5305964/miniconda3
PHDenv                   /export/home/s5305964/miniconda3/envs/PHDenv

Unfortunately, you have installed it in miniconda. So leaving aside that, I would build a new env

To install new env and packages, do this:

source /usr/local/bin/s3proxy.sh  (need to get access the internet within a slurm job)
conda create --name pytorchCuda121 pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia

(See Qs 90 : https://griffith.atlassian.net/wiki/spaces/GHCD/pages/4030751/FAQ+-+Griffith+HPC+Cluster#FAQ-GriffithHPCCluster-Qs90%3AHowtorunthepytorch(containermethodandcondaenvmethod)ontheICTcluster)

source activate pytorchCuda121
 python ~/isCuda.py 
CUDA AVILABLE
>>>>>>>>>isCuda.py>>>>>>>>>>>>>
import torch
if torch.cuda.is_available():
    print('CUDA AVILABLE')
else:
    print('NO CUDA')

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

PS: Comment out the miniconda install from ~./bashrc which can mess with this env.

If you ever need to use the miniconda env, you can do this:

source deactivate
module purge

__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
eval "$__conda_setup"
source activate PHDenv
python isCuda.py 
CUDA AVILABLE

A group named "alan" has 2 A100 gpus reserved.

scontrol show res
ReservationName=alan100 StartTime=2024-07-01T10:33:01 EndTime=2024-08-02T10:33:01 Duration=32-00:00:00
   Nodes=dgxlogin NodeCnt=1 CoreCnt=32 Features=(null) PartitionName=LocalQ Flags=FLEX,MAGNETIC
     NodeName=dgxlogin CoreIDs=0-10,13,40-43,64-79
   TRES=cpu=64,gres/gpu:a100=2,gres/gpu=2
   Users=(null) Groups=alan Accounts=(null) Licenses=(null) State=ACTIVE BurstBuffer=(null) Watts=n/a
   MaxStartDelay=(null)


This is reservation is automated (using cron) so that it is deleted every 30 days and recreated. Which means , these 2 gpus are permanently reserved for alan group. 

Usage is as follows:
For batch runs
==============
Add this inside the slurm script:
#SBATCH --reservation=alan100 
OR
sbatch  --reservation=alan100 <job script>
e.g:   sbatch  --reservation=alan1g10gb slurm.res
Make sure the job script has resource request covered by the reservation (This command will list that: scontrol show res)

For interactive runs
=================
srun --reservation=alan100 --export=PATH,TERM,HOME,LANG  --job-name=myTestRun --cpus-per-task=1 --mem-per-cpu=15GB --time=00:02:00 --qos=work --gres=gpu:a100:1 --pty /bin/bash -l

This will be illustrated using an actual question that came to our support team:

Support Request: 
================
I’ve been trying for some time to get gene regulatory analysis to run on a large dataset on Gowonda. 
Originally I tried doing this using Genie3, which begins running but errors out after about 5 hours 
due to exceeding memory, even if I run it on bigmem requesting 900GB.

There’s a new package – arboreto – that has a better version of Genie3 but also a much more 
computationally efficient methodology called grnboost2. I’ve made a new conda environment 
and have installed arboreto, however can’t get grnboost2 to run. The files load fine, but it 
stalls at the grnboost2 step with the error 
‘TypeError: descriptor '__call__' for 'type' objects doesn't apply to a 'property' object’. 
Help forums indicate that that this may be due to an issue with the Python and Dask versions.
 I’ve tried making different Conda environments using different version combinations, but still 
can’t get it to work.

Our Answer:
===========
We looked at this page: https://github.com/aertslab/pySCENIC/issues/163
and we think the 1st solution using singularity containers is good. 
Containers are better suited when issues with version mismatch occurs 
(e.g specific versions of python, dask etc).
https://pyscenic.readthedocs.io/en/latest/installation.html#docker-and-singularity-images
We went to this link as directed from above link.

This is the copy and paste of the section using Apptainer (used to be called Singularity)
>>>>>> Clip from the link >>>>>>>>>>>>
Singularity/Apptainer
Singularity/Apptainer images can be build from the Docker Hub image as source:
# pySCENIC CLI version.
singularity build aertslab-pyscenic-0.12.1.sif docker://aertslab/pyscenic:0.12.1
apptainer build aertslab-pyscenic-0.12.1.sif docker://aertslab/pyscenic:0.12.1

# pySCENIC CLI version + ipython kernel + scanpy.
singularity build aertslab-pyscenic-scanpy-0.12.1-1.9.1.sif docker://aertslab/pyscenic_scanpy:0.12.1_1.9.1
apptainer build aertslab-pyscenic-0.12.1-1.9.1.sif docker://aertslab/pyscenic_scanpy:0.12.1_1.9.1
>>>>>>>>>End of the clip from the link >>>>>>>>>>

To download the container from any docker link at Griffith, we have to alter the commands slightly 
(also documented on the singularity page of our wiki) 
https://griffith.atlassian.net/wiki/spaces/GHCD/pages/4030761/Singularity

(The change is to replace docker:// with docker://public.docker.itc.griffith.edu.au/)
So the altered command becomes:
singularity build aertslab-pyscenic-0.12.1.sif docker://public.docker.itc.griffith.edu.au/aertslab/pyscenic:0.12.1
singularity build aertslab-pyscenic-scanpy-0.12.1-1.9.1.sif docker://public.docker.itc.griffith.edu.au/aertslab/pyscenic_scanpy:0.12.1_1.9.1

After sourcing the proxy file (source /usr/local/bin/s3proxy.sh), you can run the above two commands to download those containers.
We have done this for you and placed the containers in this directory: ~/Containers 
If this directory does not exist, create it with mkdir ~/Containers

To check the functionality of the container, please do this:
singularity shell -B /scratch/snumber:/scratch --pwd /scratch/snumber:/scratch --pwd /scratch  /export/home/snumber/Containers/aertslab-pyscenic-0.12.1.sif 
>>>>>>>>>
From inside the container shell, you can run a few commands to check for functionality
e.g:
Apptainer>cd scripts
Apptainer> python grnboost3.py 
INFO:root:Initializing Dask client...
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
DEBUG:asyncio:Using selector: EpollSelector
INFO:root:Dask client initialized: <Client: 'tcp://127.0.0.1:41411' processes=9 threads=72, memory=251.23 GiB>
INFO:root:Loading and transposing expression data...
ERROR:root:An error occurred: [Errno 2] No such file or directory: 'FinalCounts.csv'
<snip>
exit

>>>>>>>>>>>>>
This looks ok (you need to supply the missing files FinalCounts.csv)

If you are happy with this, go to the next step:
(Please note: We have put all the sample pbs and run scripts in your ~scratch/scripts folder)


Next part is to create a pbs script and a run script 

mkdir /scratch/snumber/scripts
mkdir /scratch/snumber/scripts/data
cd  /scratch/snumber/scripts
You can copy all the scripts into this folder as well as create a run script
e.g: cp grnboost3.py ~/scratch/scripts/grnboost3.py

Here is a sample pbs script to use 6 cpus
>>>>>>>>>>>>>>>
#!/bin/bash
#PBS -m e
#PBS -M YourEmail@griffith.edu.au
#PBS -N GRN_pearl
#PBS -q workq
#PBS -l select=1:ncpus=6:mem=66gb,walltime=30:00:00
cd  $PBS_O_WORKDIR
singularity exec -B /scratch/:/scratch --pwd /scratch/snumber:/scratch --pwd /scratch  /export/home/snumber/Containers/aertslab-pyscenic-0.12.1.sif "/scratch/scripts/run1.sh"
#singularity exec -B /scratch/snumber:/scratch --pwd /scratch/snumber:/scratch --pwd /scratch  /export/home/snumber/Containers/aertslab-pyscenic-scanpy-0.12.1-1.9.1.sif "/scratch/scripts/run2.sh"
exit
sleep 2

>>>>>>>>>>>>>>>>


Next create a run script inside : /scratch/snumber/scripts

cat run1.sh
>>>>>>>>>>>>
cd /scratch/scripts
python /scratch/scripts/grnboost3.py

>>>>>>>>>>>>
make it executable:
chmod +x d_run1.sh

Now simply submit the pbs job:
cd  /scratch/snumber/scripts
qsub pbs.01

After the run has completed, check through the output and error files and fix any errors. Voilà!