Examples Haswell Nodes

MPI only Job (IBM MPI)

#!/bin/bash
# DO NOT USE environment = COPY_ALL
##
## optional: energy policy tags
##
#@ job_type = parallel
#@ class = general
#@ node = 100
###@ island_count= not needed for
#### class general
####
####
#@ total_tasks= 2800
## other example ##@ tasks_per_node = 28 #@ wall_clock_limit = 1:20:30 ## 1 h 20 min 30 secs #@ job_name = mytest #@ network.MPI = sn_all,not_shared,us #@ initialdir = $(home)/mydir #@ output = job.$(schedd_host).$(jobid).out #@ error = job.$(schedd_host).$(jobid).err #@ notification=always #@ notify_user=youremail_at_yoursite.xx #@ queue . /etc/profile . /etc/profile.d/modules.sh poe ./myprog.exe

MPI only Job (Intel MPI)

#!/bin/bash
##
## optional: energy policy tags
##
# DO NOT USE environment = COPY_ALL
#@ job_type = MPICH
#@ class = general
#@ node = 1000
### schedule the job to exactly 1 island 
#########(more than one will fail) #@ island_count=1 #@ island_count=1
#@ total_tasks= 28000
## other example ##@ tasks_per_node = 28 #@ wall_clock_limit = 1:20:30 ## 1 h 20 min 30 secs #@ job_name = mytest #@ network.MPI = sn_all,not_shared,us #@ initialdir = $(home)/mydir
#@ output = job.$(schedd_host).$(jobid).out #@ error = job.$(schedd_host).$(jobid).err #@ notification=always #@ notify_user=youremail_at_yoursite.xx #@ queue . /etc/profile . /etc/profile.d/modules.sh #setup of environment module unload mpi.ibm module load mpi.intel mpiexec -n 28000 ./myprog.exe

Hybrid MPI/OpenMP Job (IBM MPI)

#!/bin/bash
####
## optional: energy policy tags
#
# DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = general
#@ node = 200
#@ island_count=1,1
#@ total_tasks=800
##@ tasks_per_node = 4
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job.$(schedd_host).$(jobid).out
 #@ error = job.$(schedd_host).$(jobid).err #@ notification=always #@ notify_user=youremail_at_yoursite.xx #@ queue . /etc/profile . /etc/profile.d/modules.sh export MP_SINGLE_THREAD=no export OMP_NUM_THREADS=7 # Pinning export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS mpiexec -n 800 ./myprog.exe

Hybrid MPI/OpenMP Job (Intel MPI)

#!/bin/bash
##
## optional: energy policy tags
#
# DO NOT USE environment = COPY_ALL
#@ job_type = MPICH
#@ class = general
#@ node = 500
#@ island_count=1,2
###@ total_tasks=2000
## other example
#@ tasks_per_node = 4
#@ wall_clock_limit = 1:20:30
##                    1 h 20 min 30 secs
#@ job_name = mytest
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job.$(schedd_host).$(jobid).out 
#@ error = job.$(schedd_host).$(jobid).err #@ notification=always #@ notify_user=youremail_at_yoursite.xx #@ queue . /etc/profile . /etc/profile.d/modules.sh #setup of environment module unload mpi.ibm module load mpi.intel export OMP_NUM_THREADS=7 #optional: #module load mpi_pinning/hybrid_blocked mpiexec -n 2000 ./myprog.exe

Hybrid MPI/OpenMP with oversubscription/hyperthreading Job (IBM MPI)

#!/bin/bash
####
## optional: energy policy tags
## DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = general
#@ node = 200
#@ island_count=1,1
#@ tasks_per_node = 28
#@ network.MPI = sn_all,not_shared,us
#@ initialdir = $(home)/mydir
#@ output = job.$(schedd_host).$(jobid).out
 #@ error = job.$(schedd_host).$(jobid).err #@ queue . /etc/profile . /etc/profile.d/modules.sh export MP_SINGLE_THREAD=no export OMP_NUM_THREADS=2 # Pinning export MP_TASK_AFFINITY=cpu:$OMP_NUM_THREADS mpiexec -n 5600 ./myprog.exe

Pure OpenMP Job (on  medium nodes)

#!/bin/bash
##
## optional: energy policy tags
##
#@ wall_clock_limit = 01:20:00,01:19:30
## with softlimit
#@ job_name = mytest
# DO NOT USE environment = COPY_ALL
#@ job_type = parallel
#@ class = test
#@ node = 1
#@ total_tasks = 1
## OR
#@ tasks_per_node = 1
#@ node_usage = not_shared
#@ initialdir = $(home)/mydir
#@ output = job.$(schedd_host).$(jobid).out 
#@ error = job.$(schedd_host).$(jobid).err #@ notification=always #@ notify_user=youremail_at_yoursite.xx #@ queue . /etc/profile . /etc/profile.d/modules.sh export OMP_NUM_THREADS=28 export KMP_AFFINITY="granularity=core,compact,1"