Examples Thin Nodes

Parallel MPI Job (IBM MPI)

#!/bin/bash
# DO NOT USE environment = COPY_ALL
##
## optional: energy policy tags
##
#@ job_type = parallel
#@ class = general
#@ node = 200
###@ island_count= not needed for
#### class general
#@ total_tasks= 3200
## other example
##@ tasks_per_node = 16
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ notification=always
#@ notify_user=youremail_at_yoursite.xx
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
poe ./myprog.exe

Parallel MPI Job (Intel MPI)

#!/bin/bash
##
## optional: energy policy tags
##
# DO NOT USE environment = COPY_ALL
#@ job_type = MPICH
#@ class = large
#@ node = 1000
### schedule the job to 2 to 4 islands 
#@ island_count=2,4
#@ total_tasks= 16000
## other example
##@ tasks_per_node = 16
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ notification=always
#@ notify_user=youremail_at_yoursite.xx
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
#setup of environment
module unload mpi.ibm
module load mpi.intel
mpiexec -n 16000 ./myprog.exe

Hybrid MPI Job (IBM MPI)

#!/bin/bash
####
## optional: energy policy tags
#
# DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = large
#@ node = 1000
#@ island_count=2,3
#@ total_tasks=8000
## other example
##@ tasks_per_node = 8
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ notification=always
#@ notify_user=youremail_at_yoursite.xx
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
export MP_SINGLE_THREAD=no
export OMP_NUM_THREADS=2
# Pinning
export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS
mpiexec -n 8000./myprog.exe

Hybrid MPI/OpenMP Job (Intel MPI)

#!/bin/bash
##
## optional: energy policy tags
#
# DO NOT USE environment = COPY_ALL
#@ job_type = MPICH
#@ class = large
#@ node = 1000
#@ island_count=2,3
#@ total_tasks=8000
## other example
##@ tasks_per_node = 8
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ notification=always
#@ notify_user=youremail_at_yoursite.xx
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
#setup of environment
module unload mpi.ibm
module load mpi.intel
export OMP_NUM_THREADS=2
#optional: 
#module load mpi_pinning/hybrid_blocked
mpiexec -n 8000./myprog.exe

Hybrid MPI Job with oversubscription/hyperthreading  (IBM MPI)

#!/bin/bash
## optional: energy policy tags

# DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = large
#@ node = 1000
#@ island_count=2,3
#@ total_tasks=16000
## other example
##@ tasks_per_node = 16
#@ wall_clock_limit = 1:20:30
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
export MP_SINGLE_THREAD=no
export OMP_NUM_THREADS=2
# Pinning, this will use the physical and virtual cores
export MP_TASK_AFFINITY=cpu:$OMP_NUM_THREADS mpiexec -n 16000./myprog.exe

Pure OpenMP Job (on Thin Node Island)

#!/bin/bash
##
## optional: energy policy tags
##
#@ wall_clock_limit = 01:20:00,01:19:30
## with softlimit
#@ job_name = mytest
# DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = general
#@ node = 1
#@ total_tasks = 1
## OR
#@ tasks_per_node = 1
#@ node_usage = not_shared
#@ initialdir = $(home)/mydir
#@ output = job$(jobid).out
#@ error = job$(jobid).err
#@ notification=always
#@ notify_user=youremail_at_yoursite.xx
#@ queue
. /etc/profile
. /etc/profile.d/modules.sh
export OMP_NUM_THREADS=16
export KMP_AFFINITY="granularity=core,compact,1"
./myprog.exe