This example shell script demonstrates the use of various ECFS commands to
- list the contents of and ECFS directory and
- copy files to and from ECFS.
#!/bin/bash # # **************************** LICENSE START *********************************** # # Copyright 2021 ECMWF. This software is distributed under the terms # of the Apache License version 2.0. In applying this license, ECMWF does not # waive the privileges and immunities granted to it by virtue of its status as # an Intergovernmental Organization or submit itself to any jurisdiction. # # ***************************** LICENSE END ************************************ # # ecfs1 USER SERVICES NOVEMBER 2021 - ECMWF # # # This shell-script: # # - executes a MARS request # - sets an ECFS directory # - archives the contents of a directory # (containing the output of the MARS request) # - lists the contents of the archived ECFS directory # - copies the ECFS directory back to a Unix directory # # This shell script produces the standard output file # # ecfs1.<JOB-ID>.out # # in the workdir directory, containing the log of job execution. # # #------------------------------- # setting options for SLURM #------------------------------- # Options that are specified within the script file should precede the # first executable shell command in the file. # All options, which are case sensitive, are preceded by #SBATCH. # These lines become active only when the script is submitted # using the "sbatch" command. # All job output is written to the workdir directory, by default. #SBATCH --qos=ef # Specifies that your job will run in the queue (Quality Of # Service) "ef". #SBATCH --job-name=ecfs1 # Assigns the specified name to the request #SBATCH --output=ecfs1.%j.out # Specifies the name and location of STDOUT where %j is the job-id # The file will be # written in the workdir directory if it is a # relative path. If not given, the default is slurm-%j.out in the # workdir. #SBATCH --error=ecfs1.%j.out # Specifies the name and location of STDERR where %j is the job-id # The file will be # written in the workdir directory if it is a # relative path. If not given, the default is slurm-%j.out in the # workdir. #SBATCH --chdir=/scratch/... # Sets the working directory of the batch script before it is # executed. #SBATCH --mail-type=FAIL # Specifies that an email should be sent in case the job fails. # Other options include BEGIN, END, REQUEUE and ALL (any state # change). #SBATCH --time=00:05:00 # Specifies that your job my run up to HH:MM:SS of wall clock # time. The job will be killed if it exceeds this limit. If # time is not defined, the default limit for the queue (qos) # will be used. #------------------------------- # setting environment variables #------------------------------- export PATH=$PATH:. # Allows you to run any of your programs or # scripts held in the current directory (not # required if already done in your .user_profile # or .user_kshrc) set -ev #------------------------------- # commands to be executed #------------------------------- cd $SCRATCHDIR # See keyword initialdir above. # All the files created in this directory will be # deleted when the job terminates. echo $SCRATCHDIR #------------------------------- # MARS request #------------------------------- file1=MARSDAT1 # These are the files that will hold the retrieved file2=MARSDAT2 # data from MARS. # Read request parameters into the file 'myrequest' cat >myrequest <<EOF # ($file1 and $file2 are substituted). retrieve, class = od, stream = oper, date = -2, time = 12, type = fc, grid = 2.5/2.5, area = 75/-27.5/32.5/45, target = "$file1" retrieve, date = -3, target = "$file2" EOF echo mars myrequest # Send request to MARS. if [ $? != 0 ] then echo " The MARS request failed!" exit 1 fi #------------------------------- # ECFS commands # (for more information type: man ecfs) #------------------------------- epwd ec: # Show the current ECFS working directory in # the specified domain ('ec:/uid' by default). ecd ec: # Set the environment variable ECDIR used to # maintain the current ec: working directory # for ECFS to the path specified # (default value '/uid'). els -l ec: # Show the contents of the ECFS directory # ec:/uid. The leading characters * b - indicate # the following: # *: file in TSM # b: file in HPSS with backup # -: file in HPSS without backup emkdir ec:batch_ex1 # Create the ECFS directory # ec:/uid/batch_ex1. ecp * ec:batch_ex1 # Copy the contents of the current working # directory ($SCRATCHDIR) to the ECFS # directory ec:/uid/batch_ex1. # If the target directory does not exist # it is created automatically. # It is possible to use wild characters # (* , ? ) in the specification of the # filenames to be copied. els -l ec: # Show the contents of the ECFS directory # ec:/uid. After the execution of the previous # command the directory ec:/uid/batch_ex1 # is present. els -l ec:batch_ex1 # Show the contents of the ECFS directory # ec:/uid/batch_ex1. ecp ec:batch_ex1/* $SCRATCHDIR/batch_ex1 # Retrieve the contents of the ECFS directory # ec:/uid/batch_ex1 and put it in a # directory $SCRATCHDIR/batch_ex1. # If the target directory does not exist # it is created automatically. ls -al $SCRATCHDIR/batch_ex1 # Show the contents of the directory # $SCRATCHDIR/batch_ex1. ecd ec:batch_ex1 # Change ECFS working directory to # ec:/uid/batch_ex1. erm ec:* # Remove the contents of the ECFS directory # ec:/uid/batch_ex1. els -l ec: # Show the contents of the ECFS directory # ec:/uid/batch_ex1. ecd .. # Change ECFS working directory back # to parent directory. ermdir ec:batch_ex1 # Remove the ECFS directory ec:/uid/batch_ex1. #------------------------------- # tidy up by deleting unwanted files #------------------------------- # This is done automatically when using $SCRATCHDIR. exit 0 # End of example job 'ecfs1'