This example shell script demonstrates the use of various ECFS commands to
- list the contents of and ECFS directory and
- copy files to and from ECFS.
#!/bin/bash
#
# **************************** LICENSE START ***********************************
#
# Copyright 2021 ECMWF. This software is distributed under the terms
# of the Apache License version 2.0. In applying this license, ECMWF does not
# waive the privileges and immunities granted to it by virtue of its status as
# an Intergovernmental Organization or submit itself to any jurisdiction.
#
# ***************************** LICENSE END ************************************
#
# ecfs1 USER SERVICES NOVEMBER 2021 - ECMWF
#
#
# This shell-script:
#
# - executes a MARS request
# - sets an ECFS directory
# - archives the contents of a directory
# (containing the output of the MARS request)
# - lists the contents of the archived ECFS directory
# - copies the ECFS directory back to a Unix directory
#
# This shell script produces the standard output file
#
# ecfs1.<JOB-ID>.out
#
# in the workdir directory, containing the log of job execution.
#
#
#-------------------------------
# setting options for SLURM
#-------------------------------
# Options that are specified within the script file should precede the
# first executable shell command in the file.
# All options, which are case sensitive, are preceded by #SBATCH.
# These lines become active only when the script is submitted
# using the "sbatch" command.
# All job output is written to the workdir directory, by default.
#SBATCH --qos=ef
# Specifies that your job will run in the queue (Quality Of
# Service) "ef".
#SBATCH --job-name=ecfs1
# Assigns the specified name to the request
#SBATCH --output=ecfs1.%j.out
# Specifies the name and location of STDOUT where %j is the job-id
# The file will be # written in the workdir directory if it is a
# relative path. If not given, the default is slurm-%j.out in the
# workdir.
#SBATCH --error=ecfs1.%j.out
# Specifies the name and location of STDERR where %j is the job-id
# The file will be # written in the workdir directory if it is a
# relative path. If not given, the default is slurm-%j.out in the
# workdir.
#SBATCH --chdir=/scratch/...
# Sets the working directory of the batch script before it is
# executed.
#SBATCH --mail-type=FAIL
# Specifies that an email should be sent in case the job fails.
# Other options include BEGIN, END, REQUEUE and ALL (any state
# change).
#SBATCH --time=00:05:00
# Specifies that your job my run up to HH:MM:SS of wall clock
# time. The job will be killed if it exceeds this limit. If
# time is not defined, the default limit for the queue (qos)
# will be used.
#-------------------------------
# setting environment variables
#-------------------------------
export PATH=$PATH:. # Allows you to run any of your programs or
# scripts held in the current directory (not
# required if already done in your .user_profile
# or .user_kshrc)
set -ev
#-------------------------------
# commands to be executed
#-------------------------------
cd $SCRATCHDIR # See keyword initialdir above.
# All the files created in this directory will be
# deleted when the job terminates.
echo $SCRATCHDIR
#-------------------------------
# MARS request
#-------------------------------
file1=MARSDAT1 # These are the files that will hold the retrieved
file2=MARSDAT2 # data from MARS.
# Read request parameters into the file 'myrequest'
cat >myrequest <<EOF # ($file1 and $file2 are substituted).
retrieve,
class = od,
stream = oper,
date = -2,
time = 12,
type = fc,
grid = 2.5/2.5,
area = 75/-27.5/32.5/45,
target = "$file1"
retrieve,
date = -3,
target = "$file2"
EOF
echo mars myrequest # Send request to MARS.
if [ $? != 0 ]
then
echo " The MARS request failed!"
exit 1
fi
#-------------------------------
# ECFS commands # (for more information type: man ecfs)
#-------------------------------
epwd ec: # Show the current ECFS working directory in
# the specified domain ('ec:/uid' by default).
ecd ec: # Set the environment variable ECDIR used to
# maintain the current ec: working directory
# for ECFS to the path specified
# (default value '/uid').
els -l ec: # Show the contents of the ECFS directory
# ec:/uid. The leading characters * b - indicate
# the following:
# *: file in TSM
# b: file in HPSS with backup
# -: file in HPSS without backup
emkdir ec:batch_ex1 # Create the ECFS directory
# ec:/uid/batch_ex1.
ecp * ec:batch_ex1 # Copy the contents of the current working
# directory ($SCRATCHDIR) to the ECFS
# directory ec:/uid/batch_ex1.
# If the target directory does not exist
# it is created automatically.
# It is possible to use wild characters
# (* , ? ) in the specification of the
# filenames to be copied.
els -l ec: # Show the contents of the ECFS directory
# ec:/uid. After the execution of the previous
# command the directory ec:/uid/batch_ex1
# is present.
els -l ec:batch_ex1 # Show the contents of the ECFS directory
# ec:/uid/batch_ex1.
ecp ec:batch_ex1/* $SCRATCHDIR/batch_ex1
# Retrieve the contents of the ECFS directory
# ec:/uid/batch_ex1 and put it in a
# directory $SCRATCHDIR/batch_ex1.
# If the target directory does not exist
# it is created automatically.
ls -al $SCRATCHDIR/batch_ex1
# Show the contents of the directory
# $SCRATCHDIR/batch_ex1.
ecd ec:batch_ex1 # Change ECFS working directory to
# ec:/uid/batch_ex1.
erm ec:* # Remove the contents of the ECFS directory
# ec:/uid/batch_ex1.
els -l ec: # Show the contents of the ECFS directory
# ec:/uid/batch_ex1.
ecd .. # Change ECFS working directory back
# to parent directory.
ermdir ec:batch_ex1 # Remove the ECFS directory ec:/uid/batch_ex1.
#-------------------------------
# tidy up by deleting unwanted files
#-------------------------------
# This is done automatically when using $SCRATCHDIR.
exit 0
# End of example job 'ecfs1'