Fortran program for writing
to temporary directory in
/start/partition1

!****************************************************************
!  This is a simple hello world program. Each processor 
!  writes out its rank and total number of processors 
!  in the current MPI run to a file in /state/partition1
!****************************************************************
      program hello
      use ifport  ! ## needed for GETENV called below ##
      include "mpif.h"
      character (len=MPI_MAX_PROCESSOR_NAME):: nodename
      character (len=20) :: myname
      character (len=40) :: basedir,fname
      call MPI_INIT( ierr )
      call MPI_COMM_RANK( MPI_COMM_WORLD, myid, ierr )
      call MPI_COMM_SIZE( MPI_COMM_WORLD, numprocs, ierr )
! get the name of the node on which i am running
      call MPI_Get_processor_name(nodename,nlen,ierr)
! get my username
      CALL GETENV ("LOGNAME",myname)
!  for every mpi task...
!  create a unique file name based on a base directory 
!  name, a username, "f_", and the mpi process id
      basedir="/state/partition1/"
      write(fname,'(a,a,"/f_",i5.5)'),trim(basedir),trim(myname),myid
! echo the full path to the file for each mpi task
      write(*,*)"opening file ",trim(fname)
! open the file
      open(10,file=fname)
! write the mpi process id to the file along with the node name
      write(10,'("Hello from",i4," on ",a)')myid,trim(nodename)
      close(10)
      call MPI_FINALIZE(ierr)
      stop
      end

pbs script for creating a temporary directory in
/start/partition1 and copying files from it

#!/bin/bash
#PBS -l nodes=2:ppn=8
#PBS -l naccesspolicy=singleuser
#PBS -l walltime=00:05:00
#PBS -N testIO
#PBS -o testIOo.$PBS_JOBID
#PBS -e testIOe.$PBS_JOBID
#PBS -r n
#PBS -V 
#-----------------------------------------------------
cd $PBS_O_WORKDIR

# Save a nicely sorted list of nodes with no repeats
sort -u  $PBS_NODEFILE > shortlist

# Get our list of nodes
export nlist=`cat shortlist`

# For each node...
for i in $nlist
do 
# Create my temporary directory in /state/partion1 on each node
  ssh $i mkdir /state/partition1/$USER
done


# Run the program that writes to the temporary directory
mpiexec -np 16 ./f_ex00b

#for each node...
for i in $nlist
do 
# Copy files from /state/partion1 on each node back to
# my working directory with a subdirectory for each node.
  mkdir -p $PBS_O_WORKDIR/$i
  scp -r $USER@$i:/state/partition1/$USER $USER@ra.mines.edu:$PBS_O_WORKDIR/$i
# Remove the temporary directory
  ssh $i rmdir /state/partition1/$USER
done