Page History

'''
Created on 22 Oct 2019


# Copyright 2005-2018 ECMWF.
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction
   
This is a test program to encode Wigos Synop
requires
   
1) ecCodes version 2.814.1 or above (available at https://confluence.ecmwf.int/display/ECC/Releases)
2) python2.7python3.6.8-01
   
To run the program
   
   ./wigosTempaddWigosProg.py  -i synop_multi_subset.bufr -o out_synop_multisubset.bufr  -w WIGOS_TEMP_IDENT.csv
      
Uses BUFR version 4 template  and adds the WIGOS Identifier 301150
REQUIRES TablesVersionNumber above 28
   
Author : Roberto Ribas Garcia ECMWF 1228/0910/2019

'''Modifications
from eccodes import *
import argparse 
import json 
import re 
import pandas as pd 
import numpy as np 
import logging 
import requests 
import os 

def read_cmd_line():
    p=argparse.ArgumentParser()
    p.add_argument("-i","--input",help="input bufr file")
    p.add_argument("-o","--output",help="output bufr file with wigos")
    p.add_argument("-m","--mode",choices=["web","json"],help=" wigos source [ json file or web ]")
    p.add_argument("-l","--logfile",help="log file ")
    args=p.parse_args()
    return args 
    
def read_oscar_json(jsonFile):
    with open(jsonFile,"r") as f:
performance improvement ( uses skipExtraKeyAttributes)  and codes_clone   04/11/2019
    changes for SYNOP and TEMP messages                                       05/11/2019
    fixed codes_clone issue                                             jtext=json.load(f)
    return jtext 05/11/2019

def read_oscar_web(oscarURL="https://oscar.wmo.int/surface/rest/api/search/station?"):
    r=requests.get(oscarURL)
    jtext=json.loads(r.text)
    return jtext'''
from eccodes import *
import argparse 
import json 
import re 
import pandas as pd 
import numpy as np 
import logging 
import requests 
import os 

def parseread_jsoncmd_into_dataframeline(jtext):
    '''p=argparse.ArgumentParser()
    parses the JSON from the file wigosJsonFilep.add_argument("-i","--input",help="input bufr file")
    filters the stations by wigosStationIdentifiers key in the dictionaries
    '''
    
    wigosStations=[]
    nowigosStations=[]
    for d in jtext:
        if "wigosStationIdentifiers" in d.keys():
            wigosStations.append(d)
        elsep.add_argument("-o","--output",help="output bufr file with wigos")
    p.add_argument("-m","--mode",choices=["web","json"],help=" wigos source [ json file or web ]")
    p.add_argument("-l","--logfile",help="log file ")
    args=p.parse_args()
    return args 
    
def read_oscar_json(jsonFile):
    with open(jsonFile,"r") as f:
            nowigosStations.append(djtext=json.load(f)
    
return jtext   '''

def read_oscar_web(oscarURL="https://oscar.wmo.int/surface/rest/api/search/station?"):
    uses only the wigos 0-20XXX-0-YYYYY (surface)
    '''
    p=re.compile("0-20\d{3}-0-\d{5}")

    fwigosStations=[]
    for d in wigosStations:r=requests.get(oscarURL)
    jtext=json.loads(r.text)
    return jtext 

def parse_json_into_dataframe(jtext):
    '''
    parses the JSON from the file wigosJsonFile
    filters the stations  wigosInfo=d["wigosStationIdentifiers"]
    by wigosStationIdentifiers key in the dictionaries
    for e in wigosInfo:'''
    
    wigosStations=[]
    if e["primary"]==True:nowigosStations=[]
    for d in jtext:
        if wigosId=e["wigosStationIdentifierwigosStationIdentifiers"]
 in d.keys():
              if p.match(wigosId):wigosStations.append(d)
        else:
            wigosParts=wigosIdnowigosStations.splitappend("-"d)
    
    '''
    uses only the wigos     d["wigosIdentifierSeries"]=wigosParts[0]0-20XXX-0-YYYYY (surface)
    '''
        p=re.compile("0-20\d{3}-0-\d{5}")

        d["wigosIssuerOfIdentifier"]=wigosParts[1fwigosStations=[]
    for d in wigosStations:
             wigosInfo=d["wigosIssueNumberwigosStationIdentifiers"]=wigosParts[2]
        for e in wigosInfo:
         d   if e["wigosLocalIdentifierCharacterprimary"]=wigosParts[3]=True:
                    dwigosId=e["oldIDwigosStationIdentifier"]=wigosParts[3][-5:]
                if    fwigosStations.append(d)p.match(wigosId):
                    wigosParts=wigosId.split("-")
        df=pd.DataFrame(fwigosStations)
    df=df[["longitude","latitude","name","wigosStationIdentifiers","wigosIdentifierSeries","wigosIssuerOfIdentifier","wigosIssueNumber",
           "wigosLocalIdentifierCharacter","oldID"]]  d["wigosIdentifierSeries"]=wigosParts[0]
    return df

def get_ident(bid):
    '''
    gets the ident of the message by combining blockNumber and stationNumber keys from the input BUFR file
 d["wigosIssuerOfIdentifier"]=wigosParts[1]
              the ident may be single valued or multivalued ( only single valued are considered further)
 d["wigosIssueNumber"]=wigosParts[2]
         '''
    ident=None 
    if ( codes_is_defined(bid, "blockNumber") and codes_is_defined(bid,"stationNumber") ):
 d["wigosLocalIdentifierCharacter"]=wigosParts[3]
                    blockNumber=codes_get_array(bid,"blockNumber")d["oldID"]=wigosParts[3][-5:]
        stationNumber=codes_get_array(bid,"stationNumber")
         if len(blockNumber)==1 and lenfwigosStations.append(stationNumberd)==1:
            ident="{0:02d}{1:03d}".format(int(blockNumber),int(stationNumber))
        
  elif len(blockNumber)==1 and len(stationNumber)!=1:  df=pd.DataFrame(fwigosStations)
            blockNumber=np.repeat(blockNumber,len(stationNumber))df=df[["longitude","latitude","name","wigosStationIdentifiers","wigosIdentifierSeries","wigosIssuerOfIdentifier","wigosIssueNumber",
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) "wigosLocalIdentifierCharacter","oldID"]]  
    return df

def get_ident(bid):
    '''
    gets the ident of the message by combining blockNumber and stationNumber if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG] keys from the input BUFR file
    the ident may be single valued elifor multivalued len(blockNumber)!=1 and len(stationNumber)!=1:( only single valued are considered further)
    
    '''
    ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) None 
    if ( codes_is_defined(bid, "blockNumber") and codes_is_defined(bid,"stationNumber") ):
              blockNumber=codes_get_array(bid,"blockNumber")
     if b!=CODES_MISSING_LONG and s!stationNumber=CODEScodes_MISSING_LONG]get_array(bid,"stationNumber")
        if len(blockNumber)==1 and len(stationNumber)==1:
     
    return ident 

def add_wigos_info(ident,bid,wdf,obid): ident="{0:02d}{1:03d}".format(int(blockNumber),int(stationNumber))
    '''
    add the wigos information to the message ident pointed by bid
elif len(blockNumber)==1 and len(stationNumber)!=1:
           the wdf is the whole wigos dataframe and obid is the output bid
    '''
    
    
    if codes_is_defined(bid, "shortDelayedDescriptorReplicationFactor"):
        shortDelayed=codes_get_array(bid,"shortDelayedDescriptorReplicationFactor")
    else:
        shortDelayed=None 

    if codes_is_defined(bid, "delayedDescriptorReplicationFactor"):
        delayedDesc=codes_get_array(bid,"delayedDescriptorReplicationFactor")
    else: blockNumber=np.repeat(blockNumber,len(stationNumber))
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
                   if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG] 
        elif len(blockNumber)!=1 and len(stationNumber)!=1:
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
        delayedDesc=None 
        
   if 

b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG]
        '''
    nsubsets=codes_get(bid,"numberOfSubsets")
    compressed=codes_get(bid,"compressedData")
    
    masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
    if masterTablesVersionNumber<28:    here only the first element of the list is returned to the main program
        masterTablesVersionNumber=28
this avoids lists being used in the dataframe 
query and breaking  unexpandedDescriptors=codes_get_array(bid,"unexpandedDescriptors")the logic
    outUD=list(unexpandedDescriptors)
    outUD.insert(0,301150)'''
        
    '''if isinstance(ident,list):
    only treat the uncompressed messages with 1 subset ident=ident[0]
    forreturn futureident add


 treatment of compressed messages with more than 1 subset

def add_wigos_info(ident,bid,odf,obid):
    '''
    
add the wigos  if compressed==0 and nsubsets==1:information to the message ident pointed by bid
    the odf contains the ifWIGOS shortDelayedinformation isfor notident None:
    obid is the output handle
    codes_set_array(obid,"inputShortDelayedDescriptorReplicationFactor",shortDelayed)'''
   
    
 if delayedDesc is not Noneif codes_is_defined(bid, "shortDelayedDescriptorReplicationFactor"):
            shortDelayed=codes_setget_array(obidbid,"inputDelayedDescriptorReplicationFactorshortDelayedDescriptorReplicationFactor",delayedDesc)
    else:
        shortDelayed=None 

    if codes_is_setdefined(obidbid, "masterTablesVersionNumberdelayedDescriptorReplicationFactor",masterTablesVersionNumber):
        delayedDesc=codes_get_setarray(obidbid,"numberOfSubsetsdelayedDescriptorReplicationFactor",nsubsets)
    else:
    odf=wdf.query("oldID=='{0}'".format(ident))    delayedDesc=None 
        if
 not odf.empty:
  if codes_is_defined(bid, "extendedDelayedDescriptorReplicationFactor"):
        extDelayedDesc=codes_setget_array(obidbid, "unexpandedDescriptorsextendedDelayedDescriptorReplicationFactor",outUD)
    else:
        wis=odf["wigosIdentifierSeries"].valuesextDelayedDesc=None 

        
    if len(wis)!=1:nsubsets=codes_get(bid,"numberOfSubsets")
    compressed=codes_get(bid,"compressedData")
    
        wis=wis[0]
masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
    if masterTablesVersionNumber<28:
        codes_set(obid,"wigosIdentifierSeries",int(wis))masterTablesVersionNumber=28
        
    wid=odf["wigosIssuerOfIdentifier"].values unexpandedDescriptors=codes_get_array(bid,"unexpandedDescriptors")
    outUD=list(unexpandedDescriptors)
        if len(wid)!=1:outUD.insert(0,301150)
        
    '''
    wid=wid[0]
            codes_set(obid,"wigosIssuerOfIdentifier",int(wid))
            win=odf["wigosIssueNumber"].values only treat the uncompressed messages with 1 subset 
    for future add treatment of compressed messages with more than 1 subset
    '''
    
    if len(win)!=1compressed==0 and nsubsets==1:
        '''
        win=win[0]
            codes_set(obid,"wigosIssueNumber",int(win))IMPORTANT, takes into account delayed replications ( all possible cases) to accommodate
        SYNOP + TEMP  messages 
        '''
    wlid=odf["wigosLocalIdentifierCharacter"].values 
   if shortDelayed is not None:
         wlid="{0:5}".format(wlid[0]   codes_set_array(obid,"inputShortDelayedDescriptorReplicationFactor",shortDelayed)
        if delayedDesc is  logging.info(" wlid here {0}".format(wlid))not None:
            codes_set_array(obid,"wigosLocalIdentifierCharacterinputDelayedDescriptorReplicationFactor",str(wlid))delayedDesc)
        if extDelayedDesc is not None:
            codes_bufrset_copy_data(bid,obidarray(obid,"inputExtendedDelayedDescriptorReplicationFactor",extDelayedDesc)
        else:
    

        logging.info(" wigos {0} is empty for ident {1}".format(ident,odf["wigosLocalIdentifierCharacter"].values)codes_set(obid,"masterTablesVersionNumber",masterTablesVersionNumber)
        codes_set(obid,"numberOfSubsets",nsubsets)
    else:    
        logging.info("
 skipping compressed  message id {0} with {1} subsets ".format(ident,nsubsets) codes_set_array(obid, "unexpandedDescriptors",outUD)
    
    return obid
 wis=odf["wigosIdentifierSeries"].values 
   
     

defif mainlen(wis)!=1:
    args=read_cmd_line()
    logfile=args.logfile 
    logging.basicConfig(filename=logfile,level=logging.INFO,filemode="w")wis=wis[0]
    
    infile=args.input codes_set(obid,"wigosIdentifierSeries",int(wis))
    
    outfile=args.outputwid=odf["wigosIssuerOfIdentifier"].values 
   
     if mode=args.mode len(wid)!=1:
    if mode=="web":
        jtext=read_oscar_web()wid=wid[0]
        cdirectory=os.getcwd()codes_set(obid,"wigosIssuerOfIdentifier",int(wid))
        oscarFile=os.path.join(cdirectory,"oscar.json")win=odf["wigosIssueNumber"].values 
        withif open(oscarFile,"w") as flen(win)!=1:
            json.dump(jtext,f)
 win=win[0]
   else:
     codes_set(obid,"wigosIssueNumber",int(win))   cdirectory=os.getcwd()
         oscarFile=os.path.join(cdirectory,"oscar.json")
        with open(oscarFile,"r") as f:wlid=odf["wigosLocalIdentifierCharacter"].values 
            jtext=json.load(fwlid="{0:5}".format(wlid[0])
        logging.info(" wlid here {0}".format(wlid))
        codes_set(obid,"wigosLocalIdentifierCharacter",str(wlid))
        
    wigosDf=parse_json_into_dataframe(jtextcodes_bufr_copy_data(bid,obid)
    else:
      f=open(infile,"rb")
    nmsg=codes_count_in_file(f)
    fout=open(outfile,"wb"  logging.info(" skipping compressed  message id {0} with {1} subsets ".format(ident,nsubsets))
    for
   i in range(0,nmsg):
return 
    
     obid=codes_bufr_new_from_samples("BUFR4")

def main():
    print("ecCodes    bid=version {0}".format(codes_bufrget_newapi_from_file(fversion()))
        codes_set(bid,"unpack",1args=read_cmd_line()
     logfile=args.logfile 
  ident=get_ident(bid  logging.basicConfig(filename=logfile,level=logging.INFO,filemode="w")
    
    if ident:infile=args.input 
    
    outfile=args.output 
   
    loggingmode=args.infomode ("
 \t message {0} ident {1} ".format(i+1,ident))if mode=="web":
            add_wigos_info(ident,bid, wigosDf, obidjtext=read_oscar_web()
            codes_write(obid,foutcdirectory=os.getcwd()
    
        else:oscarFile=os.path.join(cdirectory,"oscar.json")
            logging.info ("message {0} rejected ".format(i+1))with open(oscarFile,"w") as f:
        codes_release(obid)    json.dump(jtext,f)
    else:
        codes_release(bidcdirectory=os.getcwd()
     f.close()   oscarFile=os.path.join(cdirectory,"oscar.json")
    
   
 with   print (" finished")


if __name__ == '__main__':
    main()open(oscarFile,"r") as f:
            jtext=json.load(f)
           
       
        
    wigosDf=parse_json_into_dataframe(jtext)
    
    f=open(infile,"rb")
    nmsg=codes_count_in_file(f)
    fout=open(outfile,"wb")
    for i in range(0,nmsg):
        bid=codes_bufr_new_from_file(f)
        obid=codes_clone(bid)
        codes_set(bid, 'skipExtraKeyAttributes', 1)
        codes_set(bid,"unpack",1)
        ident=get_ident(bid)
       
        if ident:
            logging.info (" \t message {0} ident {1} ".format(i+1,ident))

            odf=wigosDf.query("oldID=='{0}'".format(ident))                
            if not odf.empty:
                add_wigos_info(ident,bid, odf,obid)
                codes_write(obid,fout)
            else:
                logging.info(" wigos {0} is empty for ident {1}".format(ident,odf["wigosLocalIdentifierCharacter"].values))
    
        else:
            logging.info ("message {0} rejected ".format(i+1))
        codes_release(obid)        
        codes_release(bid)
    f.close()    
   
    print (" finished")


if __name__ == '__main__':
    main()