- Created by Axel Bonet, last modified on Apr 08, 2021
Among reasons to use ecFlow is certainly its simplicity with simple Python clients.
Tkinter, as the graphical interface available with Python, may be used then, to customize simple GUIs.
Some time may be dedicated learning the basics (Tkinter)
Application skeleton starts with:
from Tkinter import * root = Tk() w = Label(root, text="Hello, world!") w.pack() root.mainloop()
There may be the remembering of the 'overview' interface: it displays current tasks in submitted, active and aborted states from a server.
To do so with ecFlow, Python and Tkinter, we may use a ScrolledList widget to list such tasks.
It is trivial to add an update button so that user can request most recent content.
A dedicated class is also created to cause regular refresh (PaceKeeper)
Queue module shall then be used to deal with the asynchronous aspect.
Overview Expand source
import Tkinter as tk
import ecflow as ec
from threading import Thread
import Queue
import sys
PROGRAM_NAME = "ecflowview-overview"
BUTTON_FONT = ('times', 12)
MONO_FONT = ('lucidatypewriter', 14, 'bold')
class MenuBar(tk.Frame):
def __init__(self, parent):
tk.Frame.__init__(self, parent)
self.__helpButton = self.__createHelp()
self.__helpButton.grid(row=0, column=3)
self.__updateButton = tk.Button(self, text='Update',
font= BUTTON_FONT,
command= parent.update)
self.__updateButton.grid(row=0, column=2)
self.__quitButton = tk.Button(
self, text='Quit',
font= BUTTON_FONT, command= self.quit)
self.__quitButton.grid(row=0, column=0)
def __createHelp(self):
mb = tk.Menubutton(self, font=BUTTON_FONT,
relief= tk.RAISED,
text= 'Help')
menu = tk.Menu(mb)
mb['menu'] = menu
return mb
from scrolledlist import ScrolledList
class TaskList(tk.Frame):
NAME_WIDTH = 80
NAME_LINES = 22
def __init__(self, parent, kind):
tk.Frame.__init__(self, parent)
self.__kind = kind
self.__callback = None
self.__label = tk.Label(self, font=BUTTON_FONT,
text= kind)
rowx = 0
self.__label.grid(row=rowx, column=0, sticky= tk.W)
rowx += 1
self.__scrolledList = ScrolledList(
self,
width= self.NAME_WIDTH,
height= self.NAME_LINES,
callback= self.__callback)
self.__scrolledList.grid(row=rowx, column=0)
def insert(self, node):
self.__scrolledList.append(node.get_abs_node_path())
def clear(self):
self.__scrolledList.clear()
running = [True]
class PaceKeeper():
PACE = 60
def __init__(self, item, queue):
thr = Thread(target=self.process,
args=(queue, running))
self._item = item
thr.start()
def process(self, queue, running):
import time
while running:
queue.put(self._item.update)
time.sleep(self.PACE)
def run(self): self.update()
def update(self, verbose=False):
import time
while True:
print time.clock()
self._item.update()
time.sleep(self.PACE)
class Application(tk.Frame):
def __init__(self, master=None, client=None, queue=None):
tk.Frame.__init__(self, master)
self.__client = client
self.__queue = queue
# if client is None: self.__client = ec.Client("localhost", 31415)
width = 400
height = 300
self.canvas = tk.Canvas(width=width, height=height, bg='black')
self.grid()
self.createWidgets()
self.canvas.after(50, self.check_queue)
def createWidgets(self):
if 0:
self.quitButton = tk.Button(self, text='Quit',
command=self.quit)
self.quitButton.grid(row=1, column=1)
self.__menuBar = MenuBar(self)
self.__menuBar.grid(row=0, column=0, columnspan=3,
sticky=tk.W)
self.__wins = dict()
rowx = 1
colx = 0
for kind in ("active", "aborted", "submitted"):
self.__wins[kind] = TaskList(self, kind)
self.__wins[kind].grid(row=rowx, column= colx)
colx += 1
self.update()
def check_queue(self):
try:
self.__queue.get(block=False)
except Queue.Empty:
pass
else:
self.update()
self.canvas.after(50, self.check_queue)
def process_node(self, node):
for kind, win in self.__wins.items():
status = "%s" % node.get_state()
if status != kind: continue
win.insert(node)
print node.get_abs_node_path(), status
def process_nc(self, node):
for node in node.nodes:
if not isinstance(node, ec.Task):
self.process_nc(node)
else:
self.process_node(node)
def update(self):
self.__client.sync_local() # get changes,
defs = self.__client.get_defs()
if defs is None: raise BaseException("empty content")
for kind, win in self.__wins.items():
win.clear()
print
for suite in defs.suites:
self.process_nc(suite)
def get_username():
return pwd.getpwuid( os.getuid() )[ 0 ]
if __name__ == '__main__':
host = "localhost"
port = 31415
if len(sys.argv) > 2:
host = sys.argv[1]
port = int(sys.argv[2])
print "# " + PROGRAM_NAME + ": contact is %s %d" % ( host, port)
queue = Queue.Queue()
if port < 65535 or sys.argv[-1] != "sms":
client = ec.Client(host, port)
app = Application(client=client, queue=queue)
else: app = AppSms(host, port, queue)
app.master.title(PROGRAM_NAME)
PaceKeeper(app, queue)
app.mainloop()
It is not really far ahead to provide a treeview display of the server content, using ttk.Treeview widget.
ecFlow is "destination agnostic": it just submits local commands that can lead to remote jobs submission. The Treeview interface may bridge the ecFlow tree understanding of the current situation with the system side. It can display the target node (HPC, linux_cluster, localhost, workstation), the id of the active job, or the identification of the submitted job (qid). Check command is then expected to validate and match the server information with the query from the queuing systems, or remote machine.
A contextual menu is added to display task manual, script, job, or output, when available.
Treeview display Expand source
import ecflow as ec
from threading import Thread
import Queue
import sys
try:
import Tkinter as tk
import tkFont
except ImportError: # 3.x
import tkinter as tk # Tkinter
import tkinter.font as tkFont
import ttk
PROGRAM_NAME = "ecflowview-treeview"
BUTTON_FONT = ('times', 12)
MONO_FONT = ('lucidatypewriter', 14, 'bold')
tree_columns = (# 'status', # 'since',
'path', "host", "id", "checked", "node")
colors = { 'complete': "yellow",
'active': "green",
'aborted': "red",
'submitted': "turquoise",
"shutdown": "pink",
"halted": "violet",
"queued": "lightblue",
"unknown": "grey",
"suspended": "orange" }
displays = dict()
running = True
PACE = 30
def guess_host(node, kind=None):
if node is None: return ""
value = None
for var in node.variables:
name = var.name()
if kind is None and name == "ECF_JOB_CMD":
if "%WSHOST%" in var.value():
return guess_host(node, "WSHOST")
elif "%SCHOST%" in var.value():
return guess_host(node, "SCHOST")
elif "%HOST%" in var.value():
return guess_host(node, "HOST")
else: value = var.value()
elif kind == "SCHOST" and name == "SCHOST":
return var.value()
elif kind == "WSHOST" and name == "WSHOST":
return var.value()
elif kind == "HOST" and name == "HOST":
return var.value()
else: pass
if value is None and node.get_parent():
return guess_host(node.get_parent(), kind)
return "???"
def guess_qid(node, path=None):
if node is None: return ""
if path is None and not isinstance(node, ec.Task): return ""
if 0:
content = "%s" % node
for line in content.split("\n"):
print line
if "edit ECF_JOB " in line:
dummy, name, value = line.split()
print line, dummy, name, value
return value
elif 0:
for var in node.variables:
if var.name() == "ECF_JOB": return var.value() + ".sub"
elif path is None:
return guess_qid(node.get_parent(), node.get_abs_node_path())
else: pass
loc = None
for var in node.variables:
if var.name() == "ECF_HOME": loc = var.value() + path; break
if loc is None and node.get_parent() is not None:
return guess_qid(node.get_parent(), path)
elif loc is None: return "???"
import commands
a = commands.getstatusoutput("ls -ltr %s*.sub | awk '{print $8}'" % loc)
item =a[1].split('\n')
print path, item[0]
fd = open(item[0])
line = "???"
for line in fd.readlines():
if "has been submitted" in line:
full = line.split()
if "llsubmit:" in line: return full[3]
return full[2]
print "# last: ", line
return line
def guess_check(master, item, node, host, qid):
SLURM_ROOT="/usr/local/apps/slurm/current/bin"
qstat="/usr/local/apps/propbs/bin/qstat" # PBS
llq="/usr/lpp/LoadL/full/bin/llq -f %id %jn %o %c %st %nh"
import commands
if "lxa" in host or "lxb" in host or host=="linux_cluster":
print node, host, qid
SGE_ROOT="/usr/local/apps/sge"
SGE_ROOT="/usr/local/apps/sge/sge6_2u5"
sge="/usr/local/apps/sge/sge6_2u5/bin/lx24-amd64/qstat -u emos -f"
cmd = "ssh lxab 'SGE_ROOT=%s %s' | grep %d" % (SGE_ROOT, sge, int(qid))
a = commands.getstatusoutput(cmd)
print cmd, a
if a[0] != 0: return
full = a[1].split()
if len(full) > 6 and full[4]=='r':
print "#YES %s is running since %s %s" % (node, full[5], full[6])
master._tree.set(item, column="checked", value="%s" % full[6])
else: print full
elif "c2a" in host or "c2b" in host or "ecg" in host:
print node, host, qid
a = commands.getstatusoutput("rsh %s %s %s | grep %s" % (host, llq, qid, qid))
print a
elif "cc" in host:
print node, host, qid
a = commands.getstatusoutput("ssh %s qstat %s" % (host, qid))
print a
else: print node, host, qid
def textw(clnt, fullpath, kind, node=None):
# create child window
win = tk.Toplevel()
# display message
if kind == "variables":
msg = ""
for var in node.variables:
msg += "%-20s: %s\n" %(var.name(), var.value())
message = msg
elif kind == "info":
message = "%s" % node
else: message = clnt.client.get_file(fullpath, kind)
# text = tk.Label(win, text=message)
win.title(fullpath + " - " + kind)
s = tk.Scrollbar(win)
T = tk.Text(win)
T.focus_set()
s.pack(side=tk.RIGHT, fill=tk.Y)
T.pack(side=tk.LEFT, fill=tk.Y)
s.config(command=T.yview)
T.config(yscrollcommand=s.set)
for i in message.split("\n"):
T.insert(tk.END, i + "\n")
T.yview(tk.MOVETO, 1.0)
# ysb = ttk.Scrollbar(text, orien='vertical',
# command=text.yview)
# xsb = ttk.Scrollbar(self, orien='horizontal',
# command=text.xview)
# text.configure(yscrollcommand=ysb.set,
# xscrollcommand=xsb.set)
# text.pack()
# quit child window and return to root window
# the button is optional here, simply use the corner x of the child window
# tk.Button(win, text='close', command=win.destroy).pack()
class PaceKeeper():
def __init__(self, master, queue):
self.thr = Thread(target=self.process,
args=(queue, running))
self._master = master
self.thr.start()
def process(self, queue, running):
import time
if not running: self.thr.stop(); return
queue.put(self._master.update)
time.sleep(PACE)
def run(self): self.update()
def update(self, verbose=False):
import time
if 1: # while running:
print time.clock()
self._master.update()
time.sleep(PACE)
class ContextMenu(object):
def __init__(self, dad):
self.dad = dad
self.cmn = tk.Menu(dad, tearoff=0)
self.cmn.add_command(label="Info", command=dad.info)
self.cmn.add_command(label="Script", command=dad.script)
self.cmn.add_command(label="Manual", command=dad.manual)
self.cmn.add_command(label="Job", command=dad.job)
self.cmn.add_command(label="Output", command=dad.jobout)
self.cmn.add_command(label="Why", command=dad.why)
self.cmn.add_command(label="TimeLine", command=dad.timeline)
self.cmn.add_command(label="Variables", command=dad.variables)
self.cmn.add_command(label="Messages", command=dad.messages)
class MenuBar(tk.Frame):
def __init__(self, parent):
tk.Frame.__init__(self, parent)
self.__helpButton = self.__createHelp()
self.__updateButton = tk.Button(self, text='Update',
font= BUTTON_FONT,
command= parent.update)
self.__checkButton = tk.Button(self, text='Check',
font= BUTTON_FONT,
command= parent.check)
self.__quitButton = tk.Button(
self, text='Quit',
font= BUTTON_FONT, command= parent.quit)
col = 0
self.__quitButton.grid(row=0, column=col)
col += 1
self.__updateButton.grid(row=0, column=col)
col += 1
self.__checkButton.grid(row=0, column=col)
col += 1
self.__helpButton.grid(row=0, column=col)
col += 1
timed = tk.Label(self, font=BUTTON_FONT,
textvariable= parent.last_update, )
timed.grid(row=0, column=col)
self.__createCheck(col+1)
def __createHelp(self):
mb = tk.Menubutton(self, font=BUTTON_FONT,
relief= tk.RAISED,
text= 'Help')
menu = tk.Menu(mb)
mb['menu'] = menu
menu.add_command(command= self.__url,
label="confluence tutorial?")
menu.add_command(command= self.__url2,
label="tkinter?")
return mb
def __url2(self): self.__url(1)
def __url(self, num=0):
import os
url = "https://confluence.ecmwf.int/display/ECFLOW/Documentation"
if num == 1: url = "http://effbot.org/tkinterbook/"
os.system("firefox " + url)
def __createCheck(self, col):
col=4
keys = colors.keys()
keys.append("unfold")
for kind in sorted(keys):
var = tk.IntVar()
displays[kind] = var
check = tk.Checkbutton(self, text= kind, variable= var,
onvalue= 1, offvalue= 0,)
if kind in ("active", "aborted", "submitted", "unfold"):
check.select()
check.grid(row=0, column=col)
col += 1
def sortby(tree, col, desc, node=''):
data = []
for item in tree.get_children(node):
for kid in tree.get_children(item):
data.append((tree.set(kid, col), kid, item))
data.sort(reverse=desc)
for idx, item in enumerate(data):
tree.move(item[1], item[2], idx)
tree.heading(
col, command= lambda col= col: sortby(tree, col, int(not desc)))
class Client(object):
def __init__(self, host, port):
self.client = ec.Client(host, port)
self.host = host
self.port = port
def sync(self, top, parent):
self.client.sync_local()
load = self.defs()
if load is None: print "empty server %s %d" %(self.host, self.port)
for item in load.suites:
self.process(top, parent, item)
def defs(self): return self.client.get_defs()
def process(self, top, parent, item):
status = "%s" % item.get_state()
fullpath = item.get_abs_node_path()
if not displays[status].get(): return
unfold = displays["unfold"].get()
if isinstance(item, ec.Task) or not unfold:
name = item.name()
if isinstance(item, ec.Task):
host = guess_host(item)
checked = "nope"
qid = guess_qid(item)
else: host= ""; qid = ""; checked = ""
if unfold: name = fullpath; fullpath = ""
dad = top._tree.insert(parent, 'end',
values= (fullpath,
host,
qid,
checked,
),
tags= status,
text=name, open=unfold)
else: dad = parent
if 0:
ilen = tkFont.Font().measure(fullpath)
if ilen > 80: self._tree.column(1, width=ilen)
else: pass
if not isinstance(item, ec.Task):
for kid in item.nodes: self.process(top, dad, kid)
else: pass
class Application(tk.Frame):
def __init__(self, master=None, client=None, queue=None):
width = 768
height = 480
tk.Frame.__init__(self, master, width=width, height=height,)
self.__queue = queue
if client is None:
HOST = "vsms1";
self.__client = [ ]
PORT = 43333; self.__client.append(Client(HOST, PORT))
PORT = 32222; self.__client.append(Client(HOST, PORT))
PORT = 31415; self.__client.append(Client(HOST, PORT))
print self.__client
else: self.__client = [client ]
# self.canvas = tk.Canvas(width=width, height=height, bg='black')
self.grid()
self.last_update = tk.StringVar()
self.last_update.set("00:00")
self.createWidgets()
self.cwn = None
# self.canvas.after(50, self.check_queue)
def createWidgets(self):
self.__menuBar = MenuBar(self)
row = 0
self.__menuBar.grid(row=row, column=0, columnspan=3,
sticky=tk.W)
row += 1
self.__wins = dict()
rowx = 1
colx = 0
if 1:
self._tree = ttk.Treeview(self, columns=tree_columns,
selectmode='extended',
# justify="right",
# show="headings" # path disappear
)
ysb = ttk.Scrollbar(self, orien='vertical',
command=self._tree.yview)
xsb = ttk.Scrollbar(self, orien='horizontal',
command=self._tree.xview)
self._tree.configure(yscrollcommand=ysb.set,
# justify="right",
xscrollcommand=xsb.set)
row=1
self._tree.grid(row=row, column=0,
sticky='nsew', in_=self)
ysb.grid(row=row, column=len(tree_columns)+1, sticky='wens')
xsb.grid(sticky='ewns', columnspan=len(tree_columns),)
self.grid_columnconfigure(0, weight=1,)
self.grid_rowconfigure(row, weight=1)
pos = 0
self.process()
for kind, color in colors.items():
self._tree.tag_configure(kind, background=color)
self._tree.bind("<Double-1>", self.OnDoubleClick)
# create a popup menu
self.cmn = ContextMenu(self)
self._tree.bind("<Button-3>", self.popup)
for col in tree_columns:
self._tree.heading(
col, text=col.title(),
command= lambda c=col: sortby(self._tree, c, 0))
width = tkFont.Font().measure(col.title())
if col == "status":
ilen = tkFont.Font().measure(kind)
if ilen > width: width = ilen
else: width = 80
self._tree.column(col, width= width, stretch= True, anchor=tk.E)
pos += 1
self.grid()
self.pack(fill='both', expand=True)
self.update()
def info(self): return self.cmgen("info")
def manual(self): return self.cmgen("manual")
def script(self): return self.cmgen("script")
def job(self): return self.cmgen("job")
def jobout(self): return self.cmgen("jobout")
def why(self): return self.cmgen("why")
def timeline(self): return self.cmgen("timeline")
def variables(self): return self.cmgen("variables")
def messages(self): return self.cmgen("messages")
def cmgen(self, kind="info"):
item = self.cwn
full = self._tree.item(item,"values")
path = self._tree.item(item,"text")
print path, full, item
print "#hello:" # , node.get_abs_node_path()
upp = item; print self._tree.parent(upp)
while self._tree.parent(upp) != '':
upp = self._tree.parent(upp)
idx = self._tree.index(upp)
if path[0] == '/': fullpath = path
else: fullpath = full[0]
if idx < 0 or idx >= len(self.__client): return
clnt = self.__client[idx]
defs = clnt.defs()
node = defs.find_abs_node(fullpath)
if kind in ("script", "manual", "job", "output", "jobout"):
textw(clnt, fullpath, kind)
elif kind in ("variables",):
textw(clnt, fullpath, kind, node)
elif kind in ("info"):
textw(clnt, fullpath, kind, node)
else:
print "# not yet", node
def popup(self, event):
item = self._tree.identify('row',event.x,event.y)
if item is None: print "???"; return
print "context for", self._tree.item(item,"values"), \
self._tree.item(item,"text")
self.cwn = item
self.cmn.cmn.post(event.x_root, event.y_root)
def OnDoubleClick(self, event):
item = self._tree.identify('row',event.x,event.y)
if item is None: print "???"; return
print "you clicked on", self._tree.item(item,"values"), \
self._tree.item(item,"text")
def check_queue(self):
try:
self.__queue.get(block=False)
except Queue.Empty:
pass
else: self.update()
self.canvas.after(50, self.check_queue)
def unfold(self, item=None):
if item is None:
for item in self._tree.get_children(''):
self.unfold(item)
return
item.open()
for kid in self._tree.get_children(item):
self.unfold(kid)
def quit(self):
global running
running = False
self.destroy()
sys.exit(0)
def check(self, item=None):
if item is None:
for item in self._tree.get_children(''): self.check(item)
else: pass
else:
for kid in self._tree.get_children(item): self.check(kid)
full = self._tree.item(item, 'values'),
text = self._tree.item(item, 'text'),
if len(full[0]) > 3:
full = full[0]
if full[1] != "???":
unfold = displays["unfold"].get()
if unfold: path = text
else: path = full[0]
guess_check(self, item, path, full[1], full[2])
def process(self):
for clt in self.__client:
parent = self._tree.insert('', 'end',
text="%s@%s" % (clt.host,
clt.port),
open= True)
clt.sync(self, parent)
def update(self):
import time
now = time.localtime(time.time())
hhmmss = "%02d:%02d:%02d" % (now.tm_hour, now.tm_min, now.tm_sec)
self.last_update.set(hhmmss)
for item in self._tree.get_children(''): self._tree.delete(item)
self.process()
if __name__ == '__main__':
if len(sys.argv) > 2:
HOST = sys.argv[1]
PORT = int(sys.argv[2])
print "# " + PROGRAM_NAME + ": contact %s %d" % (HOST, PORT)
client = Client(HOST, PORT)
else: client = None
queue = Queue.Queue()
app = Application(client=client, queue=queue)
app.master.title(PROGRAM_NAME)
PaceKeeper(app, queue)
app.mainloop()
 |  |
|---|
- No labels