This example shell scripts shows how to
- use the ecfsdir command to save and retrieve a complete Unix directory as one ECFS file and
- print the ecfs_audit file using ecat:
#!/bin/bash
#
# **************************** LICENSE START ***********************************
#
# Copyright 2021 ECMWF. This software is distributed under the terms
# of the Apache License version 2.0. In applying this license, ECMWF does not
# waive the privileges and immunities granted to it by virtue of its status as
# an Intergovernmental Organization or submit itself to any jurisdiction.
#
# ***************************** LICENSE END ************************************
#
# ecfs2 USER SERVICES NOVEMBER 2021 - ECMWF
#
#
# This shell-script:
#
# - retrieves MARS data into multiple target files
# - save and retrieve a complete Unix directory (containing the
# output of the MARS request) as one ECFS file using ecfsdir
# - uses the ECFS temporary domain ectmp:
# - prints the ecfs_audit file using ecat
#
# This shell script produces the standard output file
#
# ecfs2.<JOB-ID>.out
#
# in the workdir directory, containing the log of job execution.
#
#
#-------------------------------
# setting options for SLURM
#-------------------------------
# Options that are specified within the script file should precede the
# first executable shell command in the file.
# All options, which are case sensitive, are preceded by #SBATCH.
# These lines become active only when the script is submitted
# using the "sbatch" command.
# All job output is written to the workdir directory, by default.
#SBATCH --qos=ef
# Specifies that your job will run in the queue (Quality Of
# Service) "ef".
#SBATCH --job-name=ecfs2
# Assigns the specified name to the request
#SBATCH --output=ecfs2.%j.out
# Specifies the name and location of STDOUT where %j is the job-id
# The file will be # written in the workdir directory if it is a
# relative path. If not given, the default is slurm-%j.out in the
# workdir.
#SBATCH --error=ecfs2.%j.out
# Specifies the name and location of STDERR where %j is the job-id
# The file will be # written in the workdir directory if it is a
# relative path. If not given, the default is slurm-%j.out in the
# workdir.
#SBATCH --chdir=/scratch/...
# Sets the working directory of the batch script before it is
# executed.
#SBATCH --mail-type=FAIL
# Specifies that an email should be sent in case the job fails.
# Other options include BEGIN, END, REQUEUE and ALL (any state
# change).
#SBATCH --time=00:05:00
# Specifies that your job my run up to HH:MM:SS of wall clock
# time. The job will be killed if it exceeds this limit. If
# time is not defined, the default limit for the queue (qos)
# will be used.
#-------------------------------
# setting environment variables
#-------------------------------
export PATH=$PATH:. # Allows you to run any of your programs or
# scripts held in the current directory (not
# required if already done in your .user_profile
# or .user_kshrc)
set -ev
#-------------------------------
# commands to be executed
#-------------------------------
cd $SCRATCHDIR # See keyword initialdir above.
# All the files created in this directory will be
# deleted when the job terminates.
#-------------------------------
# MARS request
#-------------------------------
# The following MARS invocation retrieves pressure level analysis for
# the four main synoptic hours, storing them in four different target
# files, one per synoptic time:
mars <<EOF
retrieve,
class = od,
stream = oper,
type = an,
expver = 1,
levtype = pl,
levelist = 1000/850/700/500/400/300,
grid = 2.5/2.5,
area = 75/-27.5/32.5/45,
param = 129,
date = -2,
time = 0000/0600/1200/1800,
target = "analysis.[time]"
EOF
if [ $? != 0 ] # Check for the exit code returned from MARS.
then
echo " The MARS request failed!"
exit 1
fi
#-------------------------------
# ECFS commands # (For more information type: man ecfs)
#-------------------------------
echo "
#======================================================================#
# Please note that in general plain MARS data should not be #
# re-archived in ECFS! Unless you have done a considerable amount of #
# post-processing it is more efficient to retrieve the data from MARS #
# again. #
#======================================================================#
"
epwd ec: # Show the current ECFS working directory
# when the job starts (default ec:/uid)
ecd ectmp: # Set the environment variable EC_TMPDIR used
# to maintain the current working directory
# in the temporary ectmp: domain to the
# default value uid.
# Files in the ectmp domain will be
# AUTOMATICALLY DELETED after 90 days!
epwd ectmp: # Return the value of the current ECFS working
# directory in the specified domain
els -l ectmp: # Show the contents of the ECFS directory
# ec:/uid. The leading characters * b - indicate
# the following:
# *: file in TSM
# b: file in HPSS with backup
# -: file in HPSS without backup
ecfsdir $SCRATCHDIR ectmp:ex1.ecfsdir
# Save the complete directory
# $SCRATCHDIR into the ECFS file
# ectmp:/uid/ex1.ecfsdir
els -l ectmp: # Show the contents of the ECFS
# directory ectmp:/uid. After the
# execution of the previous command
# the file ectmp:/uid/ex1 is present
ecfsdir ectmp:ex1.ecfsdir $SCRATCH/ex1_ret
# The ECFS file ectmp:/uid/ex1.ecfsdir
# is retrieved and the content written
# to the UNIX directory
# $SCRATCH/ex1_ret. If
# $SCRATCH/ex1_ret already exists the
# 'ecfsdir' command will fail (unless
# you use the -o option).
if [ $? != 0 ] # check ecfsdir exit code
then
echo " The ECFSDIR command has failed."
exit 1
fi
ls -al $SCRATCH/ex1_ret # Show the contents of the directory
# $SCRATCH/ex1_ret
erm ectmp:ex1 # Remove the ECFS file ectmp:/uid/ex1
ecat ec:ecfs_audit - # Print the content of the ecfs_audit file
# using the ecat command
#-------------------------------
# tidy up by deleting unwanted files
#-------------------------------
# This is done automatically when using $SCRATCHDIR but
/bin/rm -r $SCRATCH/ex1_ret
exit 0
# End of example job 'ecfs2'