This example shell script is designed to run as a time-critical job under ECaccess control using the base date and time of the latest forecast from ECaccess. The script:
- creates 10-day ENS meteogram by running a Metview macro (users can also choose to create 10-day Wave ENSgrams or 15-day ENSgrams)
- sends plot to a remote site using ectrans
#!/bin/bash
# **************************** LICENSE START ***********************************
#
# Copyright 2021 ECMWF. This software is distributed under the terms
# of the Apache License version 2.0. In applying this license, ECMWF does not
# waive the privileges and immunities granted to it by virtue of its status as
# an Intergovernmental Organization or submit itself to any jurisdiction.
#
# ***************************** LICENSE END ************************************
#
# realtime_metgram USER SERVICES NOVEMBER 2021 - ECMWF
#
#
# This shell-script:
#
# - is designed to run as a time-critical job under ECaccess control
# - gets base date and time of the latest forecast from ECaccess
# - creates 10-day ENS meteogram by running a Metview macro,
# but users can also choose to create 10-day (Wave) ENSgrams
# or 15-day ENSgrams (including with climate) or Ensemble plumes
# - sends Metgrams to a Member State using ectrans
#
# This shell script, if submitted to the time-critical
# environment, produces the standard output file in the ECaccess
# spool, as the output file name and error file name are not
# mentioned. The output and error files are visible through the
# ECaccess Web interface or through the ECaccess WebToolkit
# ecaccess-job-list and ecaccess-job-get.
#
# To run this script in realtime under ECaccess control, submit
# it with the ECtools (ecaccess-job-submit) to a ???metgram
# event, according to the chosen Metgram type. See below for
# more details.
#
# For more information on time-critical job submission, please refer to
#
# https://confluence.ecmwf.int/display/UDOC/Simple+time-critical+jobs
#
#-------------------------------
# setting options for SLURM
#-------------------------------
# Options that are specified within the script file should precede the
# first executable shell command in the file.
# All options, which are case sensitive, are preceded by #SBATCH.
# These lines become active only when the script is submitted
# using the "sbatch" command.
# All job output is written to the workdir directory, by default.
#SBATCH --qos=ef
# Specifies that your job will run in the queue (Quality Of
# Service) "ef".
#SBATCH --job-name=realtime-metgram
# Assigns the specified name to the request
# Note that no job output file name and error file names are
# specified. This will allow the ECMWF operators to see the files.
# To get access to these files, you will have to use the ecaccess web
# inerface or ECtools command ecaccess-job-get.
#SBATCH --chdir=/scratch/...
# Sets the working directory of the batch script before it is
# executed.
#SBATCH --mail-type=FAIL
# Specifies that an email should be sent in case the job fails.
# Other options include BEGIN, END, REQUEUE and ALL (any state
# change).
#SBATCH --time=00:10:00
# Specifies that your job my run up to HH:MM:SS of wall clock
# time. The job will be killed if it exceeds this limit. If
# time is not defined, the default limit for the queue (qos)
# will be used.
#--------------------------------------------------------------------
# setting environment variables
#--------------------------------------------------------------------
set -xe
#--------------------------------------------------------------------
# commands to be executed
#--------------------------------------------------------------------
cd $SCRATCHDIR # All the files created in this directory
# will be deleted when the job terminates.
# See also keyword initialdir above.
#--------------------------------------------------------------------
# Set up the DATE and TIME for the metgrams from the MSJ_* variables
# provided by ECaccess
#--------------------------------------------------------------------
echo ${MSJ_DAY:-not_set}
echo ${MSJ_MONTH:-not_set}
echo ${MSJ_YEAR:-not_set}
echo ${MSJ_BASETIME:-not_set}
echo ${MSJ_EXPVER:-not_set}
# Define the date used by the MARS request
if [[ "${MSJ_YEAR:-}" != "" && "${MSJ_MONTH:-}" != "" && "${MSJ_DAY:-}" != "" ]]
then
DATE=$MSJ_YEAR$MSJ_MONTH$MSJ_DAY # date set for job, if defined
PLOT_DATABASE='date'
else
echo "SMS date not defined!"
DATE='latest' # Use latest available
PLOT_DATABASE='latest'
fi
if [ "${MSJ_BASETIME:-}" != "" ] # Define the date used by the MARS request
then
TIME=$MSJ_BASETIME # basetime set for job
else
echo "SMS base time not defined!"
TIME='12' # Set to 12 UTC
fi
if [ "${MSJ_EXPVER:-}" != "" ] # Define the expver to be used by the MARS request
then
EXPVER=$MSJ_EXPVER # EXPVER set for job
else
echo "SMS experiment version not defined!"
EXPVER='0001' # Set to operational "0001"
fi
echo "Date for Metview batch retrieval: $DATE"
echo "TIME for Metview batch retrieval: $TIME"
echo "Using experiment version: $EXPVER"
#--------------------------------------------------------------------
# Set the following variables to choose the type of metgram and
# EXPVER used (0001 is current operational):
#
# Valid options for PLOT_TYPE are:
#
# - 10_days_epsgram - for the 10-day ENSgram
# - 15_days_epsgram - for the 15-day ENSgram
# - 15_days_epsgram_with_climate - for the 15 days with climate
# - 10_days_wave - for the 10-day Wavegram
# - 10_days_plumes - for the 10-day ENS plumes
#
# Scripts should be submitted to the following event notifications:
#
# 10-day ENSgram:
#
# ecaccess-job-submit -queueName ecs -eventIds ef00h240metgram metgram.cmd # 00 UTC
# ecaccess-job-submit -queueName ecs -eventIds ef12h240metgram metgram.cmd # 12 UTC
#
# 15-day ENSgram, 15_days_epsgram_with_climate and 10_days_plumes:
#
# ecaccess-job-submit -queueName ecs -eventIds ef00h360metgram metgram.cmd # 00 UTC
# ecaccess-job-submit -queueName ecs -eventIds ef12h360metgram metgram.cmd # 12 UTC
#
# 10-day Wave ENSgram:
#
# ecaccess-job-submit -queueName ecs -eventIds ef00h240metgram metgram.cmd # 00 UTC
# ecaccess-job-submit -queueName ecs -eventIds ef12h240metgram metgram.cmd # 12 UTC
#
#--------------------------------------------------------------------
PLOT_TYPE=10_days_epsgram
PLOT_EXPVER="0001" # Or can set to $MSJ_EXPVER
PLOT_FORMAT="png" # Can be set to png, ps, pdf
# Only one format can be used at a time !
PLOT_OUTPUT_DIR=$SCRATCH # Default directory to write output
JOIN_PDF=0 # JOIN_PDF=1 - join PDF files into one`
JOIN_PS=0 # JOIN_PS=1 - join PostScript files into one
USE_TAR=0 # USE_TAR=1 - create a tar file with all output
#--------------------------------------------------------------------
# Information for ectrans if required (only used if USE_ECTRANS=1)
# If USE_ECTRANS=1, these will need adapting for your local site.
#--------------------------------------------------------------------
USE_ECTRANS=0 # USE_ECTRANS=1 - use ectrans to transfer files
GATEWAY="msgateway.met.co"
ASSOCIATION="association"
PUT="-put"
OVERWRITE="-overwrite"
RETRY_COUNT="-retryCnt 3"
RETRY_FREQ="-retryFrq 10m"
VERBOSE="-verbose"
#--------------------------------------------------------------------
# Create the stations.list file. This is in the format:
#
# station : name : lat : long : height : output
#
# where
# station is a label for the location used by Metview (required)
# name is the name displayed on the plot (required)
# lat is the latitude value (required)
# lon is the longitude value (required)
# height is the station height (optional)
# output is the output file name (full path) (optional)
#
# Note that 'name' cannot include any apostrophes so "L'Aquila" will NOT work !
# But HTML codes can be used, e.g., "L'Aquila".
#
# If no output file is specified then output will be copied to a filename
# of the form $SCRATCH/<station>.<format> e.g.: $SCRATCH/reading.gif
#
# If you prefer, you can maintain a copy of the stations.list file in
# your $HOME directory and replace the following lines with a copy as
# follows:
#
# cp $HOME/stations.list ./stations.list
#
#--------------------------------------------------------------------
#
cat<<EOF_stations> stations.list
reading : Reading : 51.46 : -1.33 : 48.0 : ${PLOT_OUTPUT_DIR}/reading_${PLOT_TYPE}_${DATE}_${TIME}.${PLOT_FORMAT}
london : London : 51.46 : 0.00 : 14.0 : ${PLOT_OUTPUT_DIR}/london_${PLOT_TYPE}_${DATE}_${TIME}.${PLOT_FORMAT}
laquila : L'Aquila : 42.35 : 13.49 : - : ${PLOT_OUTPUT_DIR}/laquila_${PLOT_TYPE}_${DATE}${TIME}.${PLOT_FORMAT}
EOF_stations
#--------------------------------------------------------------------
# End of user-configurable section
#--------------------------------------------------------------------
export PLOT_TYPE PLOT_EXPVER DATE TIME PLOT_FORMAT PLOT_DATABASE
#--------------------------------------------------------------------
# Create Metview macro and execute
#--------------------------------------------------------------------
#
# cat is used to read the Metview macro from the input stream
# and to write it to the file 'metgram_req'.
# cat reads line by line until it reaches a line which starts with EOF.
cat >metgram_req <<EOF_mv1
#
# Get date and experiment version, etc. If not set, use specified values
# set below as defaults
#
mg_type = getenv("PLOT_TYPE")
mg_date = getenv("DATE")
mg_time = getenv("TIME")
mg_database = getenv("PLOT_DATABASE")
mg_format = getenv("PLOT_FORMAT")
scratch = getenv("SCRATCH")
mg_template = mg_type
mg_expver = getenv("PLOT_EXPVER")
if mg_expver = '' then
mg_expver = "0001"
end if
mode = runmode()
print (" ")
print ("Running EPS Metgram in a Macro")
print ("Mode is ", mode, "; Expver is ", mg_expver)
print (" Producing a Metgram of type ", mg_type)
print (" Date is ", mg_date)
print (" Time is ", mg_time)
print (" Plot format is ",mg_format)
print (" ")
print ("TMPDIR is ", getenv("TMPDIR"))
EOF_mv1
#--------------------------------------------------------------------
# Read the station list file and build up the list of stations for
# the Metview macro
#--------------------------------------------------------------------
# Remove extra spaces from around the ":" delimeter
sed -e "s/[[:space:]]*:[[:space:]]*/:/g" stations.list > stations.temp
count=1
while IFS=':' read station name lat lon height output ; do
if [[ "${station_list}" == "" ]] ; then
station_list=${station}
else
station_list="${station_list},${station}"
fi
if [[ "${output}" == "" ]] ; then
output="${SCRATCH}/${station}_${DATE}${TIME}.${PLOT_FORMAT}"
fi
if [[ "${station_output}" == "" ]] ; then
station_output=\"${output}\"
else
station_output=${station_output},\"${output}\"
fi
if [[ "${height}" == "-" || "${height}" == "" ]] ; then
station_height=""
else
station_height="height: ${height},"
fi
cat << EOF_station >> metgram_req
${station} = stations(
search_stations_database:"no",
name: "$name",
${station_height}
position:[ $lat, $lon ]
)
EOF_station
count=$((count + 1))
done < stations.temp
cat<<EOF_mv2>>metgram_req
station_list = [${station_list}]
station_output = [${station_output}]
numberOfStations = count(station_list)
print ("Number of Stations = ", numberOfStations)
print (" ")
for stationCount = 1 to numberOfStations do
thisStation = station_list[stationCount]
print("Calling meteogram for " & thisStation["NAME"] & ": Latitude=" & thisStation["LATITUDE"] & " Longitude=" & thisStation["LONGITUDE"] )
stationOutput = station_output[stationCount]
if mg_database = 'date' then
meteogram_station = meteogram (
type : mg_template,
forecast_run_time : mg_time,
date : mg_date,
data_selection_type : mg_database,
format : mg_format,
station : thisStation
)
else
meteogram_station = meteogram (
type : mg_template,
format : mg_format,
station : thisStation
)
end if
if stationOutput = '-' then
#
# If no output file is specified then produce a default name based on parameters entered
# (Should not be needed as default output file name is created before the macro is called.)
#
stationOutput = scratch & '/' & thisStation[1] & '.' & mg_format
end if
print (' Output is in ' & stationOutput)
print (" ")
write (stationOutput,meteogram_station)
end for
EOF_mv2
#--------------------------------------------------------------------
# Run the Metvew macro
#--------------------------------------------------------------------
module load ecmwf-toolbox/new
metview -b metgram_req # execute metview
exitstatus=$?
#--------------------------------------------------------------------
# Check the Metview exit status
#--------------------------------------------------------------------
if [ $exitstatus != 0 ]
then
echo "The Metview batch request failed"
fi
#--------------------------------------------------------------------
# Check that the output files contain something
#--------------------------------------------------------------------
errorstatus=0
FILE_EXTN=$(echo ${PLOT_FORMAT} | sed -e 's/"//g;s/,/ /g')
OUTFILE_LIST=""
for outfile in $(awk -F : '{print $NF}' stations.list) ; do
if [[ ! -s $outfile ]] ; then
echo "$outfile1 does not exist or is empty"
errorstatus=$((errorstatus+1))
else
OUTFILE_LIST="${OUTFILE_LIST:-} ${outfile}"
fi
done
if [[ $errorstatus != 0 ]] ; then
echo "The Meteogram plot failed - some output files do not exist or are empty"
exit $errorstatus
fi
#--------------------------------------------------------------------
# For PostScript output it is possible to join all the output
# files into a single file. To do this set JOIN_PS=1.
#--------------------------------------------------------------------
if [[ $JOIN_PS == 1 && "${PLOT_FORMAT}" == "ps" ]] ; then
outfilelist=""
n_ps_files=0
ps_joined_file=${PLOT_OUTPUT_DIR}/all_metgrams_${DATE}${TIME}.ps
for outfile in $(awk -F : '{print $NF}' stations.list) ; do
outfile1=${outfile}
if [[ -s $outfile1 ]] ; then
outfilelist=$outfilelist" "${outfile1}
n_ps_files=$((n_ps_files + 1))
fi
done
if [[ ${n_ps_files} != 0 ]] ; then
gs -dBATCH -dNOPAUSE -q -sDEVICE=ps2write -sOutputFile=${ps_joined_file} ${outfilelist}
echo "Joined files are in " ${ps_joined_file}
OUTFILE_LIST="${OUTFILE_LIST:-} ${ps_joined_file}"
else
echo "No PostScript output found"
fi
fi
#--------------------------------------------------------------------
# For PDF output it is possible to join all the output
# files into a single file. To do this set JOIN_PDF=1.
#--------------------------------------------------------------------
if [[ $JOIN_PDF == 1 && "${PLOT_FORMAT}" == "pdf" ]] ; then
outfilelist=""
n_pdf_files=0
pdf_joined_file=${PLOT_OUTPUT_DIR}/allmetgrams_${DATE}_${TIME}.pdf
for outfile in $(awk -F : '{print $NF}' stations.list) ; do
outfile1=${outfile}
if [[ -s $outfile1 ]] ; then
outfilelist=$outfilelist" "${outfile1}
n_pdf_files=$((n_pdf_files + 1))
fi
done
if [[ ${n_pdf_files} != 0 ]] ; then
gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=${pdf_joined_file} ${outfilelist}
echo "Joined files are in " ${pdf_joined_file}
OUTFILE_LIST="${OUTFILE_LIST:-} ${pdf_joined_file}"
else
echo "No PDF output found"
fi
fi
#--------------------------------------------------------------------
# If transferring files back to your local system, it may be better
# to tar the files together and transfer in one go.
#--------------------------------------------------------------------
if [[ $USE_TAR == 1 ]] ; then
outfilelist=""
for outfile in ${OUTFILE_LIST} ; do
if [[ -s $outfile ]] ; then
outfilelist=$outfilelist" -C "${outfile%/*}" "${outfile##*/}
fi
done
tarfile=${PLOT_OUTPUT_DIR}/allmetgrams_${DATE}_${TIME}.tar
[ -s ${tarfile} ] && rm -f ${tarfile}
tar cvf ${tarfile} ${outfilelist}
OUTFILE_LIST=${tarfile}
fi
#--------------------------------------------------------------------
# Transfer the output using ectrans if required (USE_ECTRANS=1)
#
# W A R N I N G
#
# Before executing this job:
#
# a) make sure your remote host is running a local ECaccess
# gateway (or use the ECMWF gateway ecaccess.ecmwf.int)
# b) make sure your ECMWF user-id is associated with a local
# (Member State) user-id, also known as a MS association;
# this can be done using your gateway's web-interface
# c) change the following in the 'ectrans' line below:
#
# ms_uid to be your user id on your local (Member
# State) host
#
# your_gateway to be your local ECaccess gateway
#
#
# To get a command summary, type: ectrans -help
#
# ectrans -gateway your_gateway -remote ms_uid@genericFtp \
# -source $tarfile -verbose -onfailure
#-----------------------------------------------------------------
if [[ $USE_ECTRANS == 1 ]] ; then
module load ecaccess # Add ectrans to the $PATH
for outfile in ${OUTFILE_LIST} ; do
OUTPUT_FILE=$outfile
TARGET_FILE=$(basename ${OUTPUT_FILE})
ectrans -gateway $GATEWAY $PUT ${OVERWRITE} \
-remote $ASSOCIATION ${RETRY_COUNT} \
-source ${OUTPUT_FILE} \
-target ${TARGET_FILE} \
${RETRY_FREQ} ${VERBOSE}
done
fi
#
#-------------------------------
# tidy up by deleting unwanted files
#-------------------------------
# This is done automatically when using $SCRATCHDIR.
exit 0
# End of example job 'realtime_metgram'