This example shell script is designed to run as a time-critical job under ECaccess control using the base date and time of the latest forecast from ECaccess. The script:
- creates 10-day ENS meteogram by running a Metview macro (users can also choose to create 10-day Wave ENSgrams or 15-day ENSgrams)
- sends plot to a remote site using ectrans
#!/bin/bash # **************************** LICENSE START *********************************** # # Copyright 2021 ECMWF. This software is distributed under the terms # of the Apache License version 2.0. In applying this license, ECMWF does not # waive the privileges and immunities granted to it by virtue of its status as # an Intergovernmental Organization or submit itself to any jurisdiction. # # ***************************** LICENSE END ************************************ # # realtime_metgram USER SERVICES NOVEMBER 2021 - ECMWF # # # This shell-script: # # - is designed to run as a time-critical job under ECaccess control # - gets base date and time of the latest forecast from ECaccess # - creates 10-day ENS meteogram by running a Metview macro, # but users can also choose to create 10-day (Wave) ENSgrams # or 15-day ENSgrams (including with climate) or Ensemble plumes # - sends Metgrams to a Member State using ectrans # # This shell script, if submitted to the time-critical # environment, produces the standard output file in the ECaccess # spool, as the output file name and error file name are not # mentioned. The output and error files are visible through the # ECaccess Web interface or through the ECaccess WebToolkit # ecaccess-job-list and ecaccess-job-get. # # To run this script in realtime under ECaccess control, submit # it with the ECtools (ecaccess-job-submit) to a ???metgram # event, according to the chosen Metgram type. See below for # more details. # # For more information on time-critical job submission, please refer to # # https://confluence.ecmwf.int/display/UDOC/Simple+time-critical+jobs # #------------------------------- # setting options for SLURM #------------------------------- # Options that are specified within the script file should precede the # first executable shell command in the file. # All options, which are case sensitive, are preceded by #SBATCH. # These lines become active only when the script is submitted # using the "sbatch" command. # All job output is written to the workdir directory, by default. #SBATCH --qos=ef # Specifies that your job will run in the queue (Quality Of # Service) "ef". #SBATCH --job-name=realtime-metgram # Assigns the specified name to the request # Note that no job output file name and error file names are # specified. This will allow the ECMWF operators to see the files. # To get access to these files, you will have to use the ecaccess web # inerface or ECtools command ecaccess-job-get. #SBATCH --chdir=/scratch/... # Sets the working directory of the batch script before it is # executed. #SBATCH --mail-type=FAIL # Specifies that an email should be sent in case the job fails. # Other options include BEGIN, END, REQUEUE and ALL (any state # change). #SBATCH --time=00:10:00 # Specifies that your job my run up to HH:MM:SS of wall clock # time. The job will be killed if it exceeds this limit. If # time is not defined, the default limit for the queue (qos) # will be used. #-------------------------------------------------------------------- # setting environment variables #-------------------------------------------------------------------- set -xe #-------------------------------------------------------------------- # commands to be executed #-------------------------------------------------------------------- cd $SCRATCHDIR # All the files created in this directory # will be deleted when the job terminates. # See also keyword initialdir above. #-------------------------------------------------------------------- # Set up the DATE and TIME for the metgrams from the MSJ_* variables # provided by ECaccess #-------------------------------------------------------------------- echo ${MSJ_DAY:-not_set} echo ${MSJ_MONTH:-not_set} echo ${MSJ_YEAR:-not_set} echo ${MSJ_BASETIME:-not_set} echo ${MSJ_EXPVER:-not_set} # Define the date used by the MARS request if [[ "${MSJ_YEAR:-}" != "" && "${MSJ_MONTH:-}" != "" && "${MSJ_DAY:-}" != "" ]] then DATE=$MSJ_YEAR$MSJ_MONTH$MSJ_DAY # date set for job, if defined PLOT_DATABASE='date' else echo "SMS date not defined!" DATE='latest' # Use latest available PLOT_DATABASE='latest' fi if [ "${MSJ_BASETIME:-}" != "" ] # Define the date used by the MARS request then TIME=$MSJ_BASETIME # basetime set for job else echo "SMS base time not defined!" TIME='12' # Set to 12 UTC fi if [ "${MSJ_EXPVER:-}" != "" ] # Define the expver to be used by the MARS request then EXPVER=$MSJ_EXPVER # EXPVER set for job else echo "SMS experiment version not defined!" EXPVER='0001' # Set to operational "0001" fi echo "Date for Metview batch retrieval: $DATE" echo "TIME for Metview batch retrieval: $TIME" echo "Using experiment version: $EXPVER" #-------------------------------------------------------------------- # Set the following variables to choose the type of metgram and # EXPVER used (0001 is current operational): # # Valid options for PLOT_TYPE are: # # - 10_days_epsgram - for the 10-day ENSgram # - 15_days_epsgram - for the 15-day ENSgram # - 15_days_epsgram_with_climate - for the 15 days with climate # - 10_days_wave - for the 10-day Wavegram # - 10_days_plumes - for the 10-day ENS plumes # # Scripts should be submitted to the following event notifications: # # 10-day ENSgram: # # ecaccess-job-submit -queueName ecs -eventIds ef00h240metgram metgram.cmd # 00 UTC # ecaccess-job-submit -queueName ecs -eventIds ef12h240metgram metgram.cmd # 12 UTC # # 15-day ENSgram, 15_days_epsgram_with_climate and 10_days_plumes: # # ecaccess-job-submit -queueName ecs -eventIds ef00h360metgram metgram.cmd # 00 UTC # ecaccess-job-submit -queueName ecs -eventIds ef12h360metgram metgram.cmd # 12 UTC # # 10-day Wave ENSgram: # # ecaccess-job-submit -queueName ecs -eventIds ef00h240metgram metgram.cmd # 00 UTC # ecaccess-job-submit -queueName ecs -eventIds ef12h240metgram metgram.cmd # 12 UTC # #-------------------------------------------------------------------- PLOT_TYPE=10_days_epsgram PLOT_EXPVER="0001" # Or can set to $MSJ_EXPVER PLOT_FORMAT="png" # Can be set to png, ps, pdf # Only one format can be used at a time ! PLOT_OUTPUT_DIR=$SCRATCH # Default directory to write output JOIN_PDF=0 # JOIN_PDF=1 - join PDF files into one` JOIN_PS=0 # JOIN_PS=1 - join PostScript files into one USE_TAR=0 # USE_TAR=1 - create a tar file with all output #-------------------------------------------------------------------- # Information for ectrans if required (only used if USE_ECTRANS=1) # If USE_ECTRANS=1, these will need adapting for your local site. #-------------------------------------------------------------------- USE_ECTRANS=0 # USE_ECTRANS=1 - use ectrans to transfer files GATEWAY="msgateway.met.co" ASSOCIATION="association" PUT="-put" OVERWRITE="-overwrite" RETRY_COUNT="-retryCnt 3" RETRY_FREQ="-retryFrq 10m" VERBOSE="-verbose" #-------------------------------------------------------------------- # Create the stations.list file. This is in the format: # # station : name : lat : long : height : output # # where # station is a label for the location used by Metview (required) # name is the name displayed on the plot (required) # lat is the latitude value (required) # lon is the longitude value (required) # height is the station height (optional) # output is the output file name (full path) (optional) # # Note that 'name' cannot include any apostrophes so "L'Aquila" will NOT work ! # But HTML codes can be used, e.g., "L'Aquila". # # If no output file is specified then output will be copied to a filename # of the form $SCRATCH/<station>.<format> e.g.: $SCRATCH/reading.gif # # If you prefer, you can maintain a copy of the stations.list file in # your $HOME directory and replace the following lines with a copy as # follows: # # cp $HOME/stations.list ./stations.list # #-------------------------------------------------------------------- # cat<<EOF_stations> stations.list reading : Reading : 51.46 : -1.33 : 48.0 : ${PLOT_OUTPUT_DIR}/reading_${PLOT_TYPE}_${DATE}_${TIME}.${PLOT_FORMAT} london : London : 51.46 : 0.00 : 14.0 : ${PLOT_OUTPUT_DIR}/london_${PLOT_TYPE}_${DATE}_${TIME}.${PLOT_FORMAT} laquila : L'Aquila : 42.35 : 13.49 : - : ${PLOT_OUTPUT_DIR}/laquila_${PLOT_TYPE}_${DATE}${TIME}.${PLOT_FORMAT} EOF_stations #-------------------------------------------------------------------- # End of user-configurable section #-------------------------------------------------------------------- export PLOT_TYPE PLOT_EXPVER DATE TIME PLOT_FORMAT PLOT_DATABASE #-------------------------------------------------------------------- # Create Metview macro and execute #-------------------------------------------------------------------- # # cat is used to read the Metview macro from the input stream # and to write it to the file 'metgram_req'. # cat reads line by line until it reaches a line which starts with EOF. cat >metgram_req <<EOF_mv1 # # Get date and experiment version, etc. If not set, use specified values # set below as defaults # mg_type = getenv("PLOT_TYPE") mg_date = getenv("DATE") mg_time = getenv("TIME") mg_database = getenv("PLOT_DATABASE") mg_format = getenv("PLOT_FORMAT") scratch = getenv("SCRATCH") mg_template = mg_type mg_expver = getenv("PLOT_EXPVER") if mg_expver = '' then mg_expver = "0001" end if mode = runmode() print (" ") print ("Running EPS Metgram in a Macro") print ("Mode is ", mode, "; Expver is ", mg_expver) print (" Producing a Metgram of type ", mg_type) print (" Date is ", mg_date) print (" Time is ", mg_time) print (" Plot format is ",mg_format) print (" ") print ("TMPDIR is ", getenv("TMPDIR")) EOF_mv1 #-------------------------------------------------------------------- # Read the station list file and build up the list of stations for # the Metview macro #-------------------------------------------------------------------- # Remove extra spaces from around the ":" delimeter sed -e "s/[[:space:]]*:[[:space:]]*/:/g" stations.list > stations.temp count=1 while IFS=':' read station name lat lon height output ; do if [[ "${station_list}" == "" ]] ; then station_list=${station} else station_list="${station_list},${station}" fi if [[ "${output}" == "" ]] ; then output="${SCRATCH}/${station}_${DATE}${TIME}.${PLOT_FORMAT}" fi if [[ "${station_output}" == "" ]] ; then station_output=\"${output}\" else station_output=${station_output},\"${output}\" fi if [[ "${height}" == "-" || "${height}" == "" ]] ; then station_height="" else station_height="height: ${height}," fi cat << EOF_station >> metgram_req ${station} = stations( search_stations_database:"no", name: "$name", ${station_height} position:[ $lat, $lon ] ) EOF_station count=$((count + 1)) done < stations.temp cat<<EOF_mv2>>metgram_req station_list = [${station_list}] station_output = [${station_output}] numberOfStations = count(station_list) print ("Number of Stations = ", numberOfStations) print (" ") for stationCount = 1 to numberOfStations do thisStation = station_list[stationCount] print("Calling meteogram for " & thisStation["NAME"] & ": Latitude=" & thisStation["LATITUDE"] & " Longitude=" & thisStation["LONGITUDE"] ) stationOutput = station_output[stationCount] if mg_database = 'date' then meteogram_station = meteogram ( type : mg_template, forecast_run_time : mg_time, date : mg_date, data_selection_type : mg_database, format : mg_format, station : thisStation ) else meteogram_station = meteogram ( type : mg_template, format : mg_format, station : thisStation ) end if if stationOutput = '-' then # # If no output file is specified then produce a default name based on parameters entered # (Should not be needed as default output file name is created before the macro is called.) # stationOutput = scratch & '/' & thisStation[1] & '.' & mg_format end if print (' Output is in ' & stationOutput) print (" ") write (stationOutput,meteogram_station) end for EOF_mv2 #-------------------------------------------------------------------- # Run the Metvew macro #-------------------------------------------------------------------- module load ecmwf-toolbox/new metview -b metgram_req # execute metview exitstatus=$? #-------------------------------------------------------------------- # Check the Metview exit status #-------------------------------------------------------------------- if [ $exitstatus != 0 ] then echo "The Metview batch request failed" fi #-------------------------------------------------------------------- # Check that the output files contain something #-------------------------------------------------------------------- errorstatus=0 FILE_EXTN=$(echo ${PLOT_FORMAT} | sed -e 's/"//g;s/,/ /g') OUTFILE_LIST="" for outfile in $(awk -F : '{print $NF}' stations.list) ; do if [[ ! -s $outfile ]] ; then echo "$outfile1 does not exist or is empty" errorstatus=$((errorstatus+1)) else OUTFILE_LIST="${OUTFILE_LIST:-} ${outfile}" fi done if [[ $errorstatus != 0 ]] ; then echo "The Meteogram plot failed - some output files do not exist or are empty" exit $errorstatus fi #-------------------------------------------------------------------- # For PostScript output it is possible to join all the output # files into a single file. To do this set JOIN_PS=1. #-------------------------------------------------------------------- if [[ $JOIN_PS == 1 && "${PLOT_FORMAT}" == "ps" ]] ; then outfilelist="" n_ps_files=0 ps_joined_file=${PLOT_OUTPUT_DIR}/all_metgrams_${DATE}${TIME}.ps for outfile in $(awk -F : '{print $NF}' stations.list) ; do outfile1=${outfile} if [[ -s $outfile1 ]] ; then outfilelist=$outfilelist" "${outfile1} n_ps_files=$((n_ps_files + 1)) fi done if [[ ${n_ps_files} != 0 ]] ; then gs -dBATCH -dNOPAUSE -q -sDEVICE=ps2write -sOutputFile=${ps_joined_file} ${outfilelist} echo "Joined files are in " ${ps_joined_file} OUTFILE_LIST="${OUTFILE_LIST:-} ${ps_joined_file}" else echo "No PostScript output found" fi fi #-------------------------------------------------------------------- # For PDF output it is possible to join all the output # files into a single file. To do this set JOIN_PDF=1. #-------------------------------------------------------------------- if [[ $JOIN_PDF == 1 && "${PLOT_FORMAT}" == "pdf" ]] ; then outfilelist="" n_pdf_files=0 pdf_joined_file=${PLOT_OUTPUT_DIR}/allmetgrams_${DATE}_${TIME}.pdf for outfile in $(awk -F : '{print $NF}' stations.list) ; do outfile1=${outfile} if [[ -s $outfile1 ]] ; then outfilelist=$outfilelist" "${outfile1} n_pdf_files=$((n_pdf_files + 1)) fi done if [[ ${n_pdf_files} != 0 ]] ; then gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=${pdf_joined_file} ${outfilelist} echo "Joined files are in " ${pdf_joined_file} OUTFILE_LIST="${OUTFILE_LIST:-} ${pdf_joined_file}" else echo "No PDF output found" fi fi #-------------------------------------------------------------------- # If transferring files back to your local system, it may be better # to tar the files together and transfer in one go. #-------------------------------------------------------------------- if [[ $USE_TAR == 1 ]] ; then outfilelist="" for outfile in ${OUTFILE_LIST} ; do if [[ -s $outfile ]] ; then outfilelist=$outfilelist" -C "${outfile%/*}" "${outfile##*/} fi done tarfile=${PLOT_OUTPUT_DIR}/allmetgrams_${DATE}_${TIME}.tar [ -s ${tarfile} ] && rm -f ${tarfile} tar cvf ${tarfile} ${outfilelist} OUTFILE_LIST=${tarfile} fi #-------------------------------------------------------------------- # Transfer the output using ectrans if required (USE_ECTRANS=1) # # W A R N I N G # # Before executing this job: # # a) make sure your remote host is running a local ECaccess # gateway (or use the ECMWF gateway ecaccess.ecmwf.int) # b) make sure your ECMWF user-id is associated with a local # (Member State) user-id, also known as a MS association; # this can be done using your gateway's web-interface # c) change the following in the 'ectrans' line below: # # ms_uid to be your user id on your local (Member # State) host # # your_gateway to be your local ECaccess gateway # # # To get a command summary, type: ectrans -help # # ectrans -gateway your_gateway -remote ms_uid@genericFtp \ # -source $tarfile -verbose -onfailure #----------------------------------------------------------------- if [[ $USE_ECTRANS == 1 ]] ; then module load ecaccess # Add ectrans to the $PATH for outfile in ${OUTFILE_LIST} ; do OUTPUT_FILE=$outfile TARGET_FILE=$(basename ${OUTPUT_FILE}) ectrans -gateway $GATEWAY $PUT ${OVERWRITE} \ -remote $ASSOCIATION ${RETRY_COUNT} \ -source ${OUTPUT_FILE} \ -target ${TARGET_FILE} \ ${RETRY_FREQ} ${VERBOSE} done fi # #------------------------------- # tidy up by deleting unwanted files #------------------------------- # This is done automatically when using $SCRATCHDIR. exit 0 # End of example job 'realtime_metgram'