Page History

'''
# Copyright 2005-2018 ECMWF.
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction
   
This is a test program to encode Wigos Synop
requires
   
1) ecCodes version 2.14.1 or above (available at https://confluence.ecmwf.int/display/ECC/Releases)
2) python3.6.8-01
   
To run the program
   
   ./addWigosProg.py  -i synop_multi_subset.bufr -o out_synop_multisubset.bufr  -w WIGOS_TEMP_IDENT.csv
      
Uses BUFR version 4 template  and adds the WIGOS Identifier 301150
REQUIRES TablesVersionNumber above 28
   
Author : Roberto Ribas Garcia ECMWF 28/10/2019

Modifications
    performance improvement ( uses skipExtraKeyAttributes)  and codes_clone   04/11/2019
    changes for SYNOP and TEMP messages                                       05/11/2019

'''
from eccodes import *
import argparse 
import json 
import re 
import pandas as pd 
import numpy as np 
import logging 
import requests 
import os 

def read_cmd_line():
    p=argparse.ArgumentParser()
    p.add_argument("-i","--input",help="input bufr file")
    p.add_argument("-o","--output",help="output bufr file with wigos")
    p.add_argument("-m","--mode",choices=["web","json"],help=" wigos source [ json file or web ]")
    p.add_argument("-l","--logfile",help="log file ")
    args=p.parse_args()
    return args 
    
def read_oscar_json(jsonFile):
    with open(jsonFile,"r") as f:
        jtext=json.load(f)
    return jtext 

def read_oscar_web(oscarURL="https://oscar.wmo.int/surface/rest/api/search/station?"):
    r=requests.get(oscarURL)
    jtext=json.loads(r.text)
    return jtext 

def parse_json_into_dataframe(jtext):
    '''
    parses the JSON from the file wigosJsonFile
    filters the stations by wigosStationIdentifiers key in the dictionaries
    '''
    
    wigosStations=[]
    nowigosStations=[]
    for d in jtext:
        if "wigosStationIdentifiers" in d.keys():
            wigosStations.append(d)
        else:
            nowigosStations.append(d)
    
    '''
    uses only the wigos 0-20XXX-0-YYYYY (surface)
    '''
    p=re.compile("0-20\d{3}-0-\d{5}")

    fwigosStations=[]
    for d in wigosStations:
        wigosInfo=d["wigosStationIdentifiers"]
        for e in wigosInfo:
            if e["primary"]==True:
                wigosId=e["wigosStationIdentifier"]
                if p.match(wigosId):
                    wigosParts=wigosId.split("-")
                    d["wigosIdentifierSeries"]=wigosParts[0]
                    d["wigosIssuerOfIdentifier"]=wigosParts[1]
                    d["wigosIssueNumber"]=wigosParts[2]
                    d["wigosLocalIdentifierCharacter"]=wigosParts[3]
                    d["oldID"]=wigosParts[3][-5:]
                    fwigosStations.append(d)
                    
    df=pd.DataFrame(fwigosStations)
    df=df[["longitude","latitude","name","wigosStationIdentifiers","wigosIdentifierSeries","wigosIssuerOfIdentifier","wigosIssueNumber",
           "wigosLocalIdentifierCharacter","oldID"]]  
    return df

def get_ident(bid):
    '''
    gets the ident of the message by combining blockNumber and stationNumber keys from the input BUFR file
    the ident may be single valued or multivalued ( only single valued are considered further)
    
    '''
    ident=None 
    if ( codes_is_defined(bid, "blockNumber") and codes_is_defined(bid,"stationNumber") ):
        blockNumber=codes_get_array(bid,"blockNumber")
        stationNumber=codes_get_array(bid,"stationNumber")
        if len(blockNumber)==1 and len(stationNumber)==1:
            ident="{0:02d}{1:03d}".format(int(blockNumber),int(stationNumber))
        elif len(blockNumber)==1 and len(stationNumber)!=1:
            blockNumber=np.repeat(blockNumber,len(stationNumber))
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
                   if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG] 
        elif len(blockNumber)!=1 and len(stationNumber)!=1:
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
                   if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG]
        '''
    
    returnhere identonly 


the first element of 

def add_wigos_info(ident,bid,wdf,obid):
    '''
    add the wigos information to the message ident pointed by bid
    the wdf is the whole wigos dataframe and obid is the output bid
the list is returned to the main program
        this avoids lists being used in the dataframe query and breaking the logic
        '''
   
    
    if codes_is_defined(bid, "shortDelayedDescriptorReplicationFactor"isinstance(ident,list):
        shortDelayed=codes_get_array(bid,"shortDelayedDescriptorReplicationFactor")
    else:
ident=ident[0]
    return ident 


   shortDelayed=None 

    if codes_is_defined(bid, "delayedDescriptorReplicationFactor"def add_wigos_info(ident,bid,odf,obid):
    '''
     delayedDesc=codes_get_array(bid,"delayedDescriptorReplicationFactor")
    else:
        delayedDesc=None 
       add the wigos information to the message ident pointed by bid
    the odf contains the WIGOS information for ident 
    obid is the output handle
    '''
   
    
    if codes_is_defined(bid, "extendedDelayedDescriptorReplicationFactorshortDelayedDescriptorReplicationFactor"):
        extDelayedDescshortDelayed=codes_get_array(bid,"extendedDelayedDescriptorReplicationFactorshortDelayedDescriptorReplicationFactor")
    else:
        extDelayedDescshortDelayed=None 

        
    nsubsets=codes_getif codes_is_defined(bid, "numberOfSubsetsdelayedDescriptorReplicationFactor"):
    compressed    delayedDesc=codes_get_array(bid,"compressedDatadelayedDescriptorReplicationFactor")
    else:
    masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
     if masterTablesVersionNumber<28:delayedDesc=None 
        masterTablesVersionNumber=28
    if codes_is_defined(bid, "extendedDelayedDescriptorReplicationFactor"):
  
      unexpandedDescriptorsextDelayedDesc=codes_get_array(bid,"unexpandedDescriptorsextendedDelayedDescriptorReplicationFactor")
    outUD=list(unexpandedDescriptors)else:
    outUD.insert(0,301150)    extDelayedDesc=None 

        
    '''nsubsets=codes_get(bid,"numberOfSubsets")
    only treat the uncompressed messages with 1 subset compressed=codes_get(bid,"compressedData")
    
    masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
    forif masterTablesVersionNumber<28:
 future add treatment of compressed messages with moremasterTablesVersionNumber=28
 than 1 subset
    ''' 
    unexpandedDescriptors=codes_get_array(bid,"unexpandedDescriptors")
    outUD=list(unexpandedDescriptors)
   if compressed==0 and nsubsets==1:
 outUD.insert(0,301150)
        
    '''
    only treat the uncompressed IMPORTANT,messages takeswith into1 accountsubset delayed
 replications ( all possiblefor cases)future toadd accommodate
treatment of compressed messages with more than  SYNOP + TEMP messages 1 subset
        '''
    
    if shortDelayed is not Nonecompressed==0 and nsubsets==1:
        '''
    codes_set_array(obid,"inputShortDelayedDescriptorReplicationFactor",shortDelayed)
    IMPORTANT, takes into account ifdelayed delayedDescreplications is not None:
      ( all possible cases) to accommodate
        SYNOP + TEMP messages 
      codes_set_array(obid,"inputDelayedDescriptorReplicationFactor",delayedDesc)  '''
        if extDelayedDescshortDelayed is not None:
            codes_set_array(obid,"inputExtendedDelayedDescriptorReplicationFactorinputShortDelayedDescriptorReplicationFactor",extDelayedDescshortDelayed)
        if delayedDesc is not 

None:
            codes_set_array(obid,"masterTablesVersionNumberinputDelayedDescriptorReplicationFactor",masterTablesVersionNumberdelayedDesc)
         codes_set(obid,"numberOfSubsets",nsubsets)
if extDelayedDesc is not None:
           odf=wdf.query("oldID=='{0}'".format(ident)) codes_set_array(obid,"inputExtendedDelayedDescriptorReplicationFactor",extDelayedDesc)
        if not odf.empty:
  

          codes_set_array(obid, "unexpandedDescriptorsmasterTablesVersionNumber",outUDmasterTablesVersionNumber)
            codes_set(obid,"numberOfSubsets",nsubsets)
        
        
        codes_set_array(obid, "unexpandedDescriptors",outUD)
        wis=odf["wigosIdentifierSeries"].values 
            if len(wis)!=1:
                wis=wis[0]
            codes_set(obid,"wigosIdentifierSeries",int(wis))
            widwid=odf["wigosIssuerOfIdentifier"].values 
            if len(wid)!=1:
                wid=wid[0]
            codes_set(obid,"wigosIssuerOfIdentifier",int(wid))
            win=odf["wigosIssueNumber"].values 
            if len(win)!=1:
                win=win[0]
            codes_set(obid,"wigosIssueNumber",int(win))            
            wlid=odf["wigosLocalIdentifierCharacter"].values 
            wlid="{0:5}".format(wlid[0])
            logging.info(" wlid here {0}".format(wlid))
            codes_set(obid,"wigosLocalIdentifierCharacter",str(wlid))
            codes_bufr_copy_data(bid,obid)
        else:
            logging.info(" wigos skipping compressed  message id {0} is empty for ident with {1}".format(ident,odf["wigosLocalIdentifierCharacter"].values))
    else:
        logging.info(" skipping compressed  message id {0} with {1} subsets ".format(ident,nsubsets))
    
    return 
    
     

def main():
    print("ecCodes version {0}".format(codes_get_api_version()))
    args=read_cmd_line()
    logfile=args.logfile 
    logging.basicConfig(filename=logfile,level=logging.INFO,filemode="w")
    
    infile=args.input 
    
    outfile=args.output 
   
    mode=args.mode 
    if mode=="web":
        jtext=read_oscar_web()
        cdirectory=os.getcwd()
        oscarFile=os.path.join(cdirectory,"oscar.json")
        with open(oscarFile,"w") as f:
            json.dump(jtext,f)
    else:
        cdirectory=os.getcwd()
        oscarFile=os.path.join(cdirectory,"oscar.json")
        with open(oscarFile,"r") as f:
            jtext=json.load(f)
           
       
        
    wigosDf=parse_json_into_dataframe(jtext)
    
    f=open(infile,"rb")
    nmsg=codes_count_in_file(f)
    fout=open(outfile,"wb")
    for i in range(0,nmsg):
        bid=codes_bufr_new_from_file(f)
        obid=codes_clone(bid)
        codes_set(bid, 'skipExtraKeyAttributes', 1)
   
        codes_set(bid,"unpack",1)
        ident=get_ident(bid)
       
        if ident:
            logging.info (" \t message {0} ident {1} ".format(i+1,ident))

            odf=wigosDf.query("oldID=='{0}'".format(ident))                
     codes_set(bid,"unpack",1)
       if  ident=get_ident(bid)not odf.empty:
       
        if ident:add_wigos_info(ident,bid, odf,obid)
            logging.info (" \t message {0} ident {1} ".format(i+1,ident))
 codes_write(obid,fout)
              add_wigos_info(ident,bid, wigosDf,obid)
else:
                logging.info("  codes_write(obid,foutwigos {0} is empty for ident {1}".format(ident,odf["wigosLocalIdentifierCharacter"].values))
    
        else:
            logging.info ("message {0} rejected ".format(i+1))
        codes_release(obid)        
        codes_release(bid)
    f.close()    
   
    print (" finished")


if __name__ == '__main__':
    main()