Page History

To help users to improve S2S CMA MARS requests performance via the WebAPI.

A good understanding of the MARS efficiency issues is essential especially for users that are interested in downloading large amounts of data.

In general it is organised, as a huge tree, with the indentation below, showing different levels down that tree:

• centre (CMA, ECMWF, NCEP, JMA, ...)
  • realtime or reforecast
    •  type of data (control forecast or perturbed forecast)
      • type of level (single level or pressure level or potential  temperature)
        • model version date (2014-05-01 or ...)
          
          • hindcast dates (2014-01-01 or 2014-01-02 or 2014-01-03, ...)
            •  time-steps
              • members (for perturbed forecast)
                • levels (for pl or pt)
                  • parameters

The idea is to request as much data as possible from the same tape file. The natural way to group requests would be:
all parameters, all levels, all members, all time-steps for 1 hindcast date for a type of level for a type

Note the following:

1. 'all' means 'all' that the user wants. It doesn't have to be all parameters.
2. If a user is interested only on z500,  he may request more hindcast dates in one go, since the overall request will not be so big.

#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
modelVersionDate = "2014-05-01"  # This is the first model version for CMA (babj)
hindcastDate = "2014-04-01"      # The selected hindcast date
server = ECMWFDataServer()
server.retrieve({
    "class": "s2",
    "dataset": "s2s",
    "date": modelVersionDate,
    "expver": "prod",
    "hdate": hindcastDate, 
    "levtype": "sfc",
    "origin": "babj",
    "param": "165/166",
    "step": "24/to/1440/by/24",
    "stream": "enfh",
    "target": "data.pf.sfc",
    "time": "00",
    "number": "1/2/3",
    "type": "pf",
})

#!/usr/bin/env python
import calendar
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()

origin = "babj"
modelVersionDate = "2014-05-01"

def retrieve_cma_datareforecast(hindcastDate,modelVersionDate,centre):
    target = "target_s2s_%s_%s.grb" % centre % hindcastDate
"""       server.retrieve({
       "class": "s2",
       "dataset": "s2s",
  A function to demonstrate how to iterate efficiently over all hindcastYears, hindcastMonths etc     "date": modelVersionDate,
       "expver": "prod",
  for a particular cma_reforecast_request.     "hdate": hindcastDate,
       "levtype": "sfc",
       "origin": centre,
  Change the variables below to adapt the iteration to your needs 
     "param": "165",
    hindcastYearStart   "step": "0",= 1994
    hindcastYearEnd   "stream": "enfh",= 2014
    hindcastMonthStart   "target": target,= 1
    hindcastMonthEnd   "time": "00",
= 12
    
    "type": "cf",
})

for hindcastYear in ["2012", "2013"]list(range(hindcastYearStart, hindcastYearEnd + 1)):
    for hindcastMonth in ["08", "09"]:
		 for hindcastDayhindcastMonth in ["01" "02" ]list(range(hindcastMonthStart, hindcastMonthEnd + 1)):
            hindcastDatenumberOfDays = hindcastYear+hindcastMonth+hindcastDaycalendar.monthrange(hindcastYear, hindcastMonth)[1]
            retrieve_data(hindcastDate,"2014-05-01","babj")

#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()

origin = "babj"
modelVersionDate = "2014-05-01"
hindcastYearList = ["2012", "2013"]
hindcastMonthList = ["08", "09"]
hindcastDayList = ["01", "02"]

def retrieve_data(hindcastDate):
    target = "target_s2s_%s_%s.grb" % (origin, hindcastDate)
    server.retrieve({
        "class": "s2",
for hindcastDay in list(range(numberOfDays)):
                hindcastDate = '%04d%02d%02d' % (
                    hindcastYear, hindcastMonth, hindcastDay + 1)
         "dataset": "s2s",
      target = "date": modelVersionDate,%s%s.grb" % (origin, hindcastDate)
        "expver": "prod",
        "hdate": hindcastDate,
  cma_reforecast_request(hindcastDate, target)

def cma_reforecast_request(hindcastDate, target):
      "levtype": "sfc",
       
 "origin": "babj",
      A CMA "param": "165",
        "step": "0",
    reforecast request. Change the keywords below to adapt it to your needs.   
    "stream": "enfh",
    
     "target": target,server.retrieve({
        "timeclass": "00s2",
        "typedataset": "cfs2s",
    })

def iterate():
    for hindcastYear in hindcastYearList:
        for hindcastMonth in hindcastMonthList:"date": modelVersionDate,
            for hindcastDay in hindcastDayList:
                hindcastDate = hindcastYear + hindcastMonth + hindcastDay"expver": "prod",
                retrieve_data(hindcastDate)

if __name__ == '__main__':
    iterate()

for hindcastYear in ["2012", "2013"]:"hdate": hindcastDate,
        "levtype": "sfc",
    for hindcastMonth in ["08", "09origin"]: origin,
         if hindcastMonth in ["01param",: "03165/166", "05", "07", "08", "10", "12"]: hindcastDateEnd = "31"
    
        "step": "24/to/1440/by/24",
     if hindcastMonth in [ "04stream",: "06enfh", "09", "11"]: hindcastDateEnd = "30"
         for hindcastDay in ["01" ... hindcastDateEnd ]:
            hindcastDate = hindcastYear+hindcastMonth+hindcastDay
  "target": target,
          retrieve_data(hindcastDate,"2014-05-01","babj")

for hindcastYear in ["2012", "2013"]:
"time": "00",
     for hindcastMonth in ["08number",: "09"]:1/2/3",
         for hindcastDay in ["01"  "06" "11" "16" "21" "26"]:
            hindcastDate = hindcastYear+hindcastMonth+hindcastDay
            retrieve_data(hindcastDate,"2014-01-01","ammc")