Versions Compared

Version Old Version 3 New Version Current
Changes made by

Indy Siva

Indy Siva

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

No Format
#!/bin/bash

#PBS -N cuda
#PBS -l walltime=100:00:00
#PBS -l select=1:ncpus=1:mem=15gb2gb:ngpus=1,walltime=01:00:00
#PBS -W group_list=gpu
#PBS -q gpu
 gpuq2
cd $PBS_O_WORKDIR
source $HOME/.bashrc
module load cudaanaconda3/4.0.172022.10
source activate TorchA100
echo "Hello from $HOSTNAME: date = `date`"
nvcc --version
echo "Finished at `date`"

sample 2

No Format
#!/bin/bash -l
#PBS -m abe
#PBS -M emailaddress@griffith.edu.au
#PBS -N CudaJob
#PBS -q gpu
#PBS -l select=21:ncpus=2:mem=2gb:ngpus=1
#PBS -W group_list=gpu
cd $PBS_O_WORKDIR
source $HOME/.bashrc
module load NAMD/NAMD28b1
module load  mpi/intel-4.0

echo "Starting job"
mpirun -r ssh  -n 2 namd2 +idlepoll /export/home/s2594054/pbs/namd/apoa1/namd/apoa1.namd > apoa1.namd.log
echo "Done with job"

...

No Format
qsub run.pbs
824.pbsserver
[s2594054@n027 namd]$ qstat
Job id            Name             User              Time Use S Queue
----------------  ---------------- ----------------  -------- - -----
812.pbsserver     3nss             s2795116          00:00:02 R workq
813.pbsserver     1ivf_naen        s2795116          00:00:01 R workq
818.pbsserver     1ivf_apo         s2795116          00:00:01 R workq
819.pbsserver     1nn2             s2795116          00:00:00 R workq
821.pbsserver     1ivg             s2795116          00:00:00 R workq
824.pbsserver     CudaJob          s2594054          00:00:00 R gpu

Installation

We use Tesla nvidia C2050 GPUs.

CUDA-enabled Device Driver

A specific device driver has been installed to support CUDA

...


Another sample PBS script (n060 node)

No Format
#!/bin/bash 
#PBS -m abe
#PBS -M yourEmail@griffith.edu.au 
#PBS -N testImage
#PBS -q dljun@n060
#PBS -W group_list=deeplearning -A deeplearning
#PBS -l select=1:ncpus=1:ngpus=1:mem=12gb,walltime=300:00:00
#PBS -j oe
cd  $PBS_O_WORKDIR
module load anaconda/5.3.1py3
source activate tensorflow-gpu
echo $CUDA_VISIBLE_DEVICES
GPUNUM=`echo $CUDA_VISIBLE_DEVICES`
sleep 2000
##echo "python main.py --cfg cfg/config3.yml --gpu $GPUNUM &"


Installation

We use Tesla nvidia C2050 GPUs.

CUDA-enabled Device Driver

A specific device driver has been installed to support CUDA

http://www.nvidia.com/Download/index.aspx?lang=en-us

...

No Format
Install the following packages before compiling:

freeglut-2.6.0-1.el6.x86_64.rpm        libdrm-devel-2.4.23-1.el6.x86_64.rpm     mesa-libGL-devel-7.10-1.el6.x86_64.rpm
freeglut-devel-2.6.0-1.el6.x86_64.rpm  libXxf86vm-devel-1.1.0-1.el6.x86_64.rpm  mesa-libGLlibGLU-devel-7.10-1.el6.x86_64.rpm
freeglut-devel-2.6.0-1.el6.x86_64.rpm  libXxf86vm-devel-1.1.0-1.el6.x86_64.rpm  mesa-libGLU-devel-7.10-1.el6.x86_64.rpm

notes2

No Format
ln -s /usr/lib64/libGLU.so.1.3.071000  /usr/lib64/libGLU.so
ln -s /usr/lib64/libglut.so.3 /usr/lib64/libglut.so
cp -r

notes2

No Format
ln -s /usr/lib64/libGLU.so.1.3.071000  /usr/lib64/libGLU.so
ln -s /usr/lib64/libglut.so.3 /usr/lib64/libglut.so
cp -r /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL  /sw/cuda/NVIDIA_GPU_Computing_SDK/4.0.17/OpenCL/src/oclVolumeRender


cp -r /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL  /sw/cuda/NVIDIA_GPU_Computing_SDK/4.0.17/OpenCL/src/


cp -r /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL  /sw/cuda/NVIDIA_GPU_Computing_SDK/4.0.17/OpenCL/common/inc/CL/

ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL  /sw/cuda/NVIDIA_GPU_Computing_SDK/4.0.17/OpenCL/src/oclMarchingCubes/oclVolumeRenderGL




cp -rln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclBandwidthTest/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/includeGL oclBlackScholes/GL
ln -s /sw/cuda/NVIDIA_GPU_Computing_SDKCUDAToolsSDK/4.0.17/OpenCLCUPTI/srcinclude/GL oclBoxFilter/GL

cpln -rs /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclConvolutionSeparable/GL
ln -s /sw/cuda/NVIDIA_GPU_Computing_SDKCUDAToolsSDK/4.0.17/OpenCLCUPTI/commoninclude/inc/CL/
GL oclCopyComputeOverlap/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclDCT8x8/GL
ln -s /sw/cuda/NVIDIA_GPU_Computing_SDKCUDAToolsSDK/4.0.17/OpenCLCUPTI/srcinclude/oclMarchingCubesGL oclDeviceQuery/GL



ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclBandwidthTestoclDotProduct/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclBlackScholesoclDXTCompression/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclBoxFilteroclFDTD3d/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclConvolutionSeparableoclHiddenMarkovModel/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclCopyComputeOverlapoclHistogram/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclDCT8x8oclInlinePTX/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclDeviceQueryoclMarchingCubes/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclDotProductoclMatrixMul/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclDXTCompressionoclMatVecMul/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclFDTD3doclMedianFilter/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclHiddenMarkovModeloclMersenneTwister/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclHistogramoclNbody/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclInlinePTXoclParticles/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclMarchingCubesoclPostprocessGL/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclMatrixMuloclQuasirandomGenerator/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclMatVecMuloclRadixSort/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclMedianFilteroclRecursiveGaussian/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclMersenneTwisteroclReduction/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclNbodyoclScan/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclParticlesoclSimpleGL/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclPostprocessGLoclSimpleMultiGPU/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclQuasirandomGeneratoroclSimpleTexture3D/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclRadixSortoclSobelFilter/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclRecursiveGaussianoclSortingNetworks/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclReductionoclTranspose/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclScanoclTridiagonal/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclSimpleGLoclVectorAdd/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclSimpleMultiGPUoclVolumeRender/GL

ln
-scd /sw/cuda/CUDAToolsSDKNVIDIA_GPU_Computing_SDK/4.0.17/CUPTI/include/GL oclSimpleTexture3D/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclSobelFilter/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclSortingNetworks/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclTranspose/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclTridiagonal/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclVectorAdd/GL
ln -s /sw/cuda/CUDAToolsSDK/4.0.17/CUPTI/include/GL oclVolumeRender/GL


cd /sw/cuda/NVIDIA_GPU_Computing_SDK/4.0.17/OpenCL
make

CuDDN

No Format
cudnn-10.0.0.17/OpenCL
make


CuDDN

No Format
cudnn-10.0
============
tar -zxvf cudnn-10.0-linux-x64-v7.6.5.32.tgz 
cuda/include/cudnn.h
cuda/NVIDIA_SLA_cuDNN_Support.txt
cuda/lib64/libcudnn.so
cuda/lib64/libcudnn.so.7
cuda/lib64/libcudnn.so.7.6.5
cuda/lib64/libcudnn_static.a

cp cuda/include/cudnn.h /usr/local/cuda-10.0/include/
cp cuda/lib64/libcudnn* /usr/local/cuda-10.0/lib64
chmod a+r /usr/local/cuda-10.0/include/cudnn.h /usr/local/cuda-10.0/lib64/libcudnn*


cudnn-10.1
============

cd /tmp; tar -zxvf cudnn-10.01-linux-x64-v7.6.5.32.tgz 
cuda/include/cudnn.h
cuda/NVIDIA_SLA_cuDNN_Support.txt
cuda/lib64/libcudnn.so
cuda/lib64/libcudnn.so.7
cuda/lib64/libcudnn.so.7.6.5
cuda/lib64/libcudnn_static.a

cp cuda/include/cudnn.h /usr/local/cuda-10.01/include/
cp cuda/lib64/libcudnn* /usr/local/cuda-10.01/lib64
 chmod a+r /usr/local/cuda-10.01/include/cudnn.h /usr/local/cuda-10.01/lib64/libcudnn*


cudnn-10.12
==========

cd
/tmp; tar -zxvf cudnn-10.12-linux-x64-v7.6.5.32.tgz 
cp cuda/include/cudnn.h /usr/local/cuda-10.12/include/
cp cuda/lib64/libcudnn* /usr/local/cuda-10.2/lib64
chmod a+r /usr/local/cuda-10.1/lib64
 chmod a+r /usr/local/cuda-10.1/include/cudnn.h /usr/local/cuda-10.1/lib64/libcudnn*

cudnn-10.2
==========

tar -zxvf cudnn-10.2-linux-x64-v7.6.5.32.tgz 
cp cuda/include/cudnn.h /usr/local/cuda-10.2/include/
cp cuda/lib64/libcudnn* /usr/local/cuda-10.2/lib64
chmod a+r /usr/local/cuda-10.2/include/cudnn.h /usr/local/cuda-10.2/lib64/libcudnn*2/include/cudnn.h /usr/local/cuda-10.2/lib64/libcudnn*


Sample pbs script to run on n061 - gpuq2


No Format
#!/bin/bash
#PBS -m abe
#PBS -M emailaddress@griffith.edu.au
#PBS -N CudaJob
#PBS -q gpuq2
#PBS -l select=1:ncpus=1:mem=2gb:ngpus=1,walltime=01:00:00
cd $PBS_O_WORKDIR
source $HOME/.bashrc
module load anaconda3/2022.10
source activate TorchA100

echo "Starting job"
python isCuda
echo "Done with job"