Page History

'''
# Copyright 2005-2018 ECMWF.
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction
   
This is a test program to encode Wigos Synop
requires
   
1) ecCodes version 2.14.1 or above (available at https://confluence.ecmwf.int/display/ECC/Releases)
2) python3.6.8-01
   
To run the program
   
   ./addWigosProg.py  -i synop_multi_subset.bufr -o out_synop_multisubset.bufr  -w WIGOS_TEMP_IDENT.csv
      
Uses BUFR version 4 template  and adds the WIGOS Identifier 301150
REQUIRES TablesVersionNumber above 28
   
Author : Roberto Ribas Garcia ECMWF 28/10/2019

Modifications
    performance improvement ( uses skipExtraKeyAttributes)  and codes_clone   04/11/2019
    changes for SYNOP and TEMP messages                                       05/11/2019

'''
from eccodes import *
import argparse 
import json 
import re 
import pandas as pd 
import numpy as np 
import logging 
import requests 
import os 

def read_cmd_line():
    p=argparse.ArgumentParser()
    p.add_argument("-i","--input",help="input bufr file")
    p.add_argument("-o","--output",help="output bufr file with wigos")
    p.add_argument("-m","--mode",choices=["web","json"],help=" wigos source [ json file or web ]")
    p.add_argument("-l","--logfile",help="log file ")
    args=p.parse_args()
    return args 
    
def read_oscar_json(jsonFile):
    with open(jsonFile,"r") as f:
        jtext=json.load(f)
    return jtext 

def read_oscar_web(oscarURL="https://oscar.wmo.int/surface/rest/api/search/station?"):
    r=requests.get(oscarURL)
    jtext=json.loads(r.text    fixed codes_clone issue                                                   05/11/2019

'''
from eccodes import *
import argparse 
import json 
import re 
import pandas as pd 
import numpy as np 
import logging 
import requests 
import os 

def read_cmd_line():
    p=argparse.ArgumentParser()
    p.add_argument("-i","--input",help="input bufr file")
    p.add_argument("-o","--output",help="output bufr file with wigos")
    p.add_argument("-m","--mode",choices=["web","json"],help=" wigos source [ json file or web ]")
    p.add_argument("-l","--logfile",help="log file ")
    args=p.parse_args()
    return jtextargs 
    
def parseread_oscar_json_into_dataframe(jtextjsonFile):
    '''
    parses the JSON from the file wigosJsonFile
    filters the stations by wigosStationIdentifiers key in the dictionarieswith open(jsonFile,"r") as f:
        jtext=json.load(f)
    '''
    
    wigosStations=[]
    nowigosStations=[]
    for d in jtext:return jtext 

def read_oscar_web(oscarURL="https://oscar.wmo.int/surface/rest/api/search/station?"):
    r=requests.get(oscarURL)
    jtext=json.loads(r.text)
    return jtext   if "wigosStationIdentifiers" in d.keys(

def parse_json_into_dataframe(jtext):
    '''
    parses the JSON  wigosStations.append(d)from the file wigosJsonFile
    filters the stations by else:
wigosStationIdentifiers key in the dictionaries
    '''
    nowigosStations.append(d)
    wigosStations=[]
    '''nowigosStations=[]
    usesfor onlyd the wigos 0-20XXX-0-YYYYY (surface)in jtext:
    '''
    p=re.compile("0-20\d{3}-0-\d{5}")

if "wigosStationIdentifiers" in d.keys():
     fwigosStations=[]
    for d in wigosStations:.append(d)
        wigosInfo=d["wigosStationIdentifiers"]else:
        for  e in wigosInfo: nowigosStations.append(d)
    
    '''
    if e["primary"]==True:
    uses only the wigos 0-20XXX-0-YYYYY (surface)
    '''
    p=re.compile("0-20\d{3}-0-\d{5}")

    wigosIdfwigosStations=e["wigosStationIdentifier"]
    for d in wigosStations:
        wigosInfo=d["wigosStationIdentifiers"]
   if p.match(wigosId):
    for e in wigosInfo:
             wigosParts=wigosId.split("-")if e["primary"]==True:
                    dwigosId=e["wigosIdentifierSerieswigosStationIdentifier"]=wigosParts[0]
                if p.match(wigosId):
     d["wigosIssuerOfIdentifier"]=wigosParts[1]               wigosParts=wigosId.split("-")
                    d["wigosIssueNumberwigosIdentifierSeries"]=wigosParts[20]
                    d["wigosLocalIdentifierCharacterwigosIssuerOfIdentifier"]=wigosParts[31]
                    d["oldIDwigosIssueNumber"]=wigosParts[3][-5:2]
                    fwigosStations.append(d)d["wigosLocalIdentifierCharacter"]=wigosParts[3]
                    
   d["oldID"]=wigosParts[3][-5:]
                    fwigosStations.append(d)
                    
    df=pd.DataFrame(fwigosStations)
    df=df[["longitude","latitude","name","wigosStationIdentifiers","wigosIdentifierSeries","wigosIssuerOfIdentifier","wigosIssueNumber",
           "wigosLocalIdentifierCharacter","oldID"]]  
    return df

def get_ident(bid):
    '''
    gets the ident of the message by combining blockNumber and stationNumber keys from the input BUFR file
    the ident may be single valued or multivalued ( only single valued are considered further)
    
    '''
    ident=None 
    if ( codes_is_defined(bid, "blockNumber") and codes_is_defined(bid,"stationNumber") ):
        blockNumber=codes_get_array(bid,"blockNumber")
        stationNumber=codes_get_array(bid,"stationNumber")
        if len(blockNumber)==1 and len(stationNumber)==1:
            ident="{0:02d}{1:03d}".format(int(blockNumber),int(stationNumber))
        elif len(blockNumber)==1 and len(stationNumber)!=1:
            blockNumber=np.repeat(blockNumber,len(stationNumber))
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
                   if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG] 
        elif len(blockNumber)!=1 and len(stationNumber)!=1:
            ident=[str("{0:02d}{1:03d}".format(b,s)) for b,s in zip(blockNumber,stationNumber) 
                   if b!=CODES_MISSING_LONG and s!=CODES_MISSING_LONG]
        '''
    
    returnhere identonly 


the first element of 

def add_wigos_info(ident,bid,wdf,obid):
    '''
    add the wigos information to the message ident pointed by bid
    the wdf is the whole wigos dataframe and obid is the output bid
the list is returned to the main program
        this avoids lists being used in the dataframe query and breaking the logic
        '''
   
    
    if codes_is_defined(bid, "shortDelayedDescriptorReplicationFactor"isinstance(ident,list):
        shortDelayed=codes_get_array(bid,"shortDelayedDescriptorReplicationFactor")
    else:ident=ident[0]
    return    shortDelayed=None ident 


    if

def codesadd_iswigos_definedinfo(ident,bid, "delayedDescriptorReplicationFactor"odf,obid):
    '''
     delayedDesc=codes_get_array(bid,"delayedDescriptorReplicationFactor")
    else:
        delayedDesc=None 
       add the wigos information to the message ident pointed by bid
    the odf contains the WIGOS information for ident 
    ifobid codes_is_defined(bid, "extendedDelayedDescriptorReplicationFactoris the output handle
    '''
   
    
    if codes_is_defined(bid, "shortDelayedDescriptorReplicationFactor"):
        extDelayedDescshortDelayed=codes_get_array(bid,"extendedDelayedDescriptorReplicationFactorshortDelayedDescriptorReplicationFactor")
    else:
        extDelayedDescshortDelayed=None 

        
    nsubsets=codes_getif codes_is_defined(bid, "numberOfSubsetsdelayedDescriptorReplicationFactor"):
      compressed  delayedDesc=codes_get_array(bid,"compressedDatadelayedDescriptorReplicationFactor")
    else:
    masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
    ifdelayedDesc=None masterTablesVersionNumber<28:
        masterTablesVersionNumber=28
    if codes_is_defined(bid, "extendedDelayedDescriptorReplicationFactor"):
    
    unexpandedDescriptorsextDelayedDesc=codes_get_array(bid,"unexpandedDescriptorsextendedDelayedDescriptorReplicationFactor")
    outUD=list(unexpandedDescriptors)
else:
     outUD.insert(0,301150)   extDelayedDesc=None 

        
    '''nsubsets=codes_get(bid,"numberOfSubsets")
    only treat the uncompressed messages with 1 subset compressed=codes_get(bid,"compressedData")
    
    masterTablesVersionNumber=codes_get(bid,"masterTablesVersionNumber")
    if masterTablesVersionNumber<28:
 for future add treatment of compressed messages withmasterTablesVersionNumber=28
 more than 1 subset
    '''
    unexpandedDescriptors=codes_get_array(bid,"unexpandedDescriptors")
    outUD=list(unexpandedDescriptors)
   if compressed==0 and nsubsets==1:
 outUD.insert(0,301150)
        
    '''
    only treat the uncompressed IMPORTANT,messages takeswith into1 accountsubset delayed
 replications ( all possiblefor cases)future toadd accommodate
treatment of compressed messages with more than  SYNOP + TEMP messages 1 subset
        '''
    
    if shortDelayed is not Nonecompressed==0 and nsubsets==1:
        '''
    codes_set_array(obid,"inputShortDelayedDescriptorReplicationFactor",shortDelayed)
    IMPORTANT, takes into account ifdelayed delayedDescreplications is( not None:
    all possible cases) to accommodate
        SYNOP + TEMP messages 
        codes_set_array(obid,"inputDelayedDescriptorReplicationFactor",delayedDesc)'''
        if extDelayedDescshortDelayed is not None:
            codes_set_array(obid,"inputExtendedDelayedDescriptorReplicationFactorinputShortDelayedDescriptorReplicationFactor",extDelayedDescshortDelayed)
        if delayedDesc is not 

None:
            codes_set_array(obid,"masterTablesVersionNumberinputDelayedDescriptorReplicationFactor",masterTablesVersionNumberdelayedDesc)
         codes_set(obid,"numberOfSubsets",nsubsets)if extDelayedDesc is not None:
        odf=wdf.query("oldID=='{0}'".format(ident))
     codes_set_array(obid,"inputExtendedDelayedDescriptorReplicationFactor",extDelayedDesc)
       if not odf.empty:
    

        codes_set_array(obid, "unexpandedDescriptorsmasterTablesVersionNumber",outUDmasterTablesVersionNumber)
            wis=odf["wigosIdentifierSeries"].values 
     codes_set(obid,"numberOfSubsets",nsubsets)
       if len(wis)!=1:
                wis=wis[0]
            codes_set_array(obid, "wigosIdentifierSeriesunexpandedDescriptors",int(wis)outUD)
            widwis=odf["wigosIssuerOfIdentifierwigosIdentifierSeries"].values 
            if len(widwis)!=1:
                wid=widwis=wis[0]
            codes_set(obid,"wigosIssuerOfIdentifierwigosIdentifierSeries",int(widwis))
            winwid=odf["wigosIssueNumberwigosIssuerOfIdentifier"].values 
            if len(winwid)!=1:
                win=winwid=wid[0]
            codes_set(obid,"wigosIssueNumberwigosIssuerOfIdentifier",int(winwid))            
            wlidwin=odf["wigosLocalIdentifierCharacterwigosIssueNumber"].values 
            wlid="{0:5}".format(wlid[0])if len(win)!=1:
            logging.info(" wlid here {0}".format(wlid))
    win=win[0]
        codes_set(obid,"wigosLocalIdentifierCharacterwigosIssueNumber",strint(wlidwin))
            codes_bufr_copy_data(bid,obid)
        else:
 wlid=odf["wigosLocalIdentifierCharacter"].values 
           logging.info(" wigos {0} is empty for ident {1wlid="{0:5}".format(ident,odf["wigosLocalIdentifierCharacter"].values))
    else:wlid[0])
        logging.info(" skippingwlid compressedhere  message id {0} with {1} subsets ".format(ident,nsubsetswlid))
        codes_set(obid,"wigosLocalIdentifierCharacter",str(wlid))
    
    return codes_bufr_copy_data(bid,obid)
    else:
     

def main():
    print("ecCodes version {0}".format(   logging.info(" skipping compressed  message id {0} with {1} subsets ".format(ident,nsubsets))
    
    return 
    
     

def main():
    print("ecCodes version {0}".format(codes_get_api_version()))
    args=read_cmd_line()
    logfile=args.logfile 
    logging.basicConfig(filename=logfile,level=logging.INFO,filemode="w")
    
    infile=args.input 
    
    outfile=args.output 
   
    mode=args.mode 
    if mode=="web":
        jtext=read_oscar_web()
        cdirectory=os.getcwd()
        oscarFile=os.path.join(cdirectory,"oscar.json")
        with open(oscarFile,"w") as f:
            json.dump(jtext,f)
    else:
        cdirectory=os.getcwd()
        oscarFile=os.path.join(cdirectory,"oscar.json")
        with open(oscarFile,"r") as f:
            jtext=json.load(f)
           
       
        
    wigosDf=parse_json_into_dataframe(jtext)
    
    f=open(infile,"rb")
    nmsg=codes_count_in_file(f)
    fout=open(outfile,"wb")
    for i in range(0,nmsg):
        bid=codes_bufr_new_from_file(f)
        obid=codes_clone(bid)
        codes_set(bid, 'skipExtraKeyAttributes', 1)
        codes_set(bid,"unpack",1)
        ident=get_ident(bid)
       
        if ident:
            logging.info (" \t message {0} ident {1} ".format(i+1,ident))

            add_wigos_info(ident,bid, wigosDf,obid)
odf=wigosDf.query("oldID=='{0}'".format(ident))            codes_write(obid,fout)
    
        else:
    if not odf.empty:
      logging.info ("message {0} rejected ".format(i+1))
        codesadd_wigos_release(info(ident,bid, odf,obid)
        
        codes_releasewrite(bidobid,fout)
       f.close()     else:
    
           print logging.info(" finished")


if __name__ == '__main__':
    main()

wigos {0} is empty for ident {1}".format(ident,odf["wigosLocalIdentifierCharacter"].values))
    
        else:
            logging.info ("message {0} rejected ".format(i+1))
        codes_release(obid)        
        codes_release(bid)
    f.close()    
   
    print (" finished")


if __name__ == '__main__':
    main()