site_graphlogo
  -   Terms of Use and Privacy
Demos | Example Code | Meta | Operation
rss
site_graphlogo
  -   Terms of Use and Privacy
Demos | Example Code | Meta | Operation
rss

<<   <   >   >>

2021-05-02 | Example Code | Triple Viewer

This program will listen to the localhost MQTT broker on the allorg/dfd/full topic and dynamically update the graph when local refresh events come across:

#!/usr/bin/python3
# coding=utf-8
# The person who associated a work with this deed has dedicated
# the work to the public domain by waiving all of his or her rights
# to the work worldwide under copyright law, including all related
# and neighboring rights, to the extent allowed by law.
# You can copy, modify, distribute and perform the work, even for
# commercial purposes, all without asking permission.
# In no way are the patent or trademark rights of any person affected by
# CC0, nor are the rights that other persons may have in the work or in
# how the work is used, such as publicity or privacy rights.
# Unless expressly stated otherwise, the person who associated a work with
# this deed makes no warranties about the work, and disclaims liability
# for all uses of the work, to the fullest extent permitted by applicable law.
# When using or citing the work, you should not imply endorsement by the 
# author or the affirmer.
# https://creativecommons.org/publicdomain/zero/1.0/

import paho.mqtt.client as mqttc
import paho.mqtt.subscribe as subscribe
import paho.mqtt.publish as publish
import wx
import sys
import os 
import base64
import wx.html2
import pickle
import time
import logging
from rfc5424logging import Rfc5424SysLogHandler
import re
from natsort import natsorted
import os, tempfile
import pygraphviz as pgv
import subprocess
import pprint
import uuid
from datetime import datetime,date
from wxasync import WxAsyncApp, AsyncBind,StartCoroutine
import asyncio
from asyncio.events import get_event_loop
from queue import Queue
f=tempfile.mkstemp(suffix='.md', prefix='temptpv', dir=None, text=True)
ed='/usr/local/Typora/Typora'
aid=sys.argv[1].rstrip()
base='https://example.com/'
#aid no spaces - designation of application/user
mqttbroker=sys.argv[2].rstrip()
graph='0'
try:
   gvizlay={'0':sys.argv[3]}
except:
   gvizlay={'0':'sfdp'}
try:
   rootnode=sys.argv[4]
except:
   rootnode='1'
q=Queue()
rootdfd='http://localhost:4026/'
dfd={}
dfd[graph]= pgv.AGraph()
accel=False
last_icon='x'
mouse_up=True

colr={'orange':'#EE7733','blue':'#0077BB','cyan':'#33BBEE','magenta':'#EE3377',
      'red':'#CC3311','teal':'#009988','grey':'#BBBBBB','black':'#000000'}
#palette for color blindness from https://personal.sron.nl/~pault/
beghtml='''<!doctype html>
<html lang='en'>
<head>
<title>Viewer</title>
<meta HTTP-EQUIV='Content-Type' CONTENT='text/html; charset=utf-8'>
<link rel="stylesheet" type="text/css" href="/css/style.css" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="/svg-pan-zoom.min.js"></script>
</head>
<body>'''
midhtml='<embed id="Graph" type="image/svg+xml" style="width: 100%; height: 1000px;" src="'
endhtml='''" />
    <script>
      // Don't use window.onLoad like this in production, because it can only listen to one function.
      window.onload = function() {
        svgPanZoom('#Graph', {
          zoomEnabled: true,
          controlIconsEnabled: false
        });
      };
    </script>
</body></html>'''
class CButton(wx.Button):
   def __init__(self,parent,txt,):
      wx.Button.__init__(self,parent,-1,txt,size=wx.Size(35,35),style=wx.BU_EXACTFIT|wx.BORDER_NONE)
      self.Bind(wx.EVT_LEFT_DOWN, self.OnLeftDown)
      self.Bind(wx.EVT_LEFT_UP, self.OnLeftUp)
      self.Bind(wx.EVT_LEFT_DCLICK, self.OnDClick)
   def OnLeftDown(self, event):
      global last_icon
      global mouse_up
      if mouse_up:
         last_icon=self.GetLabel() 
         self.SetLabel('✨')
      mouse_up=False
      event.Skip()
   def OnLeftUp(self, event):
      global mouse_up
      self.SetLabel(last_icon)
      mouse_up=True
      event.Skip()
   def OnDClick(self, event):
      global accel
      self.SetLabel(last_icon)
      accel=True
      event.Skip()
class logwindow(wx.Frame):
   def __init__(self, parent,txt):
      wx.Frame.__init__( self,parent, size=(503,320), style=wx.DEFAULT_FRAME_STYLE )
      panel=wx.Panel(self, -1)
      log_txt = wx.TextCtrl(self,size=(500,310),style=wx.TE_MULTILINE)
      log_txt.write(txt)
      self.Centre()
      self.Show()
class grp_frm(wx.Frame):
   def __init__(self):
      wx.Frame.__init__( self,
         None, -1, "Viewer",
         style=wx.DEFAULT_FRAME_STYLE )
      splitter = wx.SplitterWindow(self, style=wx.SP_LIVE_UPDATE|wx.SP_3D, size = (958,1053))
      p1=wx.Panel(splitter)
      p2=wx.Panel(splitter)
      mai_web = wx.html2.WebView.New(p2)
      fld_chk = wx.CheckBox(p2)
      lvl_txt = wx.TextCtrl(p2,size=wx.Size(-1,35),style=wx.TE_PROCESS_ENTER)
      sub_txt = wx.TextCtrl(p2,size=wx.Size(-1,35),style=wx.TE_PROCESS_ENTER)
      oplist=['Super-processes','Sub-processes','Users of Super-processes','Users of Sub-processes','Clear Results']
      opt_cho = wx.Choice(p2,size=wx.Size(-1,35),choices=oplist)
      prd_btn = CButton(p2,'🔗')
      obj_txt = wx.TextCtrl(p2,size=wx.Size(-1,35),style=wx.TE_PROCESS_ENTER)
      nav_tre = wx.TreeCtrl(p1, -1, wx.DefaultPosition, size=(90,-1))
      fwd_btn = CButton(p2,'→')
      bak_btn = CButton(p2,'←')
      lay_btn = CButton(p2,'')
      pub_btn = CButton(p2,'đŸ“Ŗ')
      exp_btn = CButton(p2,'📤')
      imp_btn = CButton(p2,'đŸ“Ĩ')
      obj_txt.Show()
      opt_cho.Hide()
      grp_vsz = wx.BoxSizer( wx.VERTICAL )
      brw_vsz = wx.BoxSizer( wx.VERTICAL )
      btn_hsz = wx.BoxSizer( wx.HORIZONTAL)
      btn_hsz.Add(lay_btn,0)
      btn_hsz.Add(fld_chk,0,wx.LEFT|wx.RIGHT, 1)
      btn_hsz.Add(lvl_txt,1,wx.EXPAND| wx.ALL,1)
      btn_hsz.Add(bak_btn,0)
      btn_hsz.Add(fwd_btn,0)
      btn_hsz.Add(sub_txt,1,wx.EXPAND| wx.ALL,1)
      btn_hsz.Add(prd_btn,0)
      btn_hsz.Add(obj_txt,9,wx.EXPAND| wx.ALL,1)
      btn_hsz.Add(opt_cho,9,wx.EXPAND| wx.ALL,1)
      btn_hsz.Add(pub_btn,0)
      btn_hsz.Add(exp_btn,0)
      btn_hsz.Add(imp_btn,0)
      grp_vsz.Add( nav_tre, 9,wx.EXPAND)
      font = wx.SystemSettings.GetFont(wx.SYS_SYSTEM_FONT)
      font.SetPointSize(20)
      self.SetFont(font)
      brw_vsz.Add( btn_hsz,0)
      brw_vsz.Add( mai_web, 1, wx.EXPAND)
      lvl_txt.SetValue(graph)
      font.SetPointSize(12)
      lvl_txt.SetFont(font)
      sub_txt.SetFont(font)
      obj_txt.SetFont(font)
      opt_cho.SetFont(font)
      root = nav_tre.AddRoot('0')
      p2.Bind(wx.EVT_BUTTON,lambda event: pred(),prd_btn)
      p2.Bind(wx.EVT_BUTTON,lambda event: mai_web.CanGoBack() and mai_web.GoBack(), bak_btn)
      p2.Bind(wx.EVT_BUTTON,lambda event: mai_web.CanGoForward() and  mai_web.GoForward(), fwd_btn)
      p2.Bind(wx.html2.EVT_WEBVIEW_NAVIGATING, lambda event: clickety_click(event), mai_web)
      p2.Bind(wx.EVT_BUTTON,lambda event: autoroute(),lay_btn)
      p2.Bind(wx.EVT_BUTTON,lambda event: publish_bar(),pub_btn)
      #publish works for all predicates.(❓ is just a visual query, not a predicate)
      p2.Bind(wx.EVT_TEXT_ENTER,lambda event: query(),sub_txt)
      #enter in subject triggers query with options listed when visual query ❓
      p2.Bind(wx.EVT_TEXT_ENTER,lambda event: change_level(),lvl_txt)
      p1.Bind(wx.EVT_TREE_ITEM_ACTIVATED,lambda event: leaf(event),nav_tre)
      p2.Bind(wx.EVT_BUTTON,lambda event: export_dfd(),exp_btn)
      p2.Bind(wx.EVT_BUTTON,lambda event: import_dfd(),imp_btn)
      p1.SetSizer(grp_vsz)
      p2.SetSizer(brw_vsz)
      splitter.SplitVertically(p1, p2, 100)
      splitter.SetMinimumPaneSize(40)
      self.Fit()
      mai_web.EnableContextMenu(enable=False)
      from SPARQLWrapper import SPARQLWrapper, JSON
      sparql = SPARQLWrapper('http://localhost/sparql')
      sparql.addDefaultGraph(base)
      sparql.setReturnFormat(JSON)
      def publish_bar():
         ts=datetime.utcnow().isoformat(sep='T', timespec='milliseconds').replace(':','').replace('-','').replace(' ','')+'Z'
         if prd_btn.GetLabel()=='đŸšĢ' and sub_txt.GetValue()=='a':
            publish.single("allorg/dfd/full",'ℹī¸ '+ts+' '+aid+' 0 a đŸšĢ a',hostname=mqttbroker)
         else:
            publish.single("allorg/dfd/full",'ℹī¸ '+ts+' '+aid+' '+graph+' '+sub_txt.GetValue()+' '+prd_btn.GetLabel()+' '+obj_txt.GetValue(),hostname=mqttbroker)
      def highlight(IRI):
         subject=IRI[len(base):-1].strip()
         if subject.find('/')==-1:
            nid=subject
            graph='0'
         else:
            nid=subject[subject.rfind('/')+1:]
            graph=subject[:subject.rfind('/')].replace('/','.')
         try:
            dfd[graph].get_node(nid).attr['penwidth']="7"
         except:
            print('Node:'+nid+' not part of graph.')
         refresh_graph()
      def query():
         if prd_btn.GetLabel()=='❓':
            skip=False
            if opt_cho.GetCurrentSelection()==4:
               skip=True
               for graph in dfd:
                  for node in dfd[graph]:
                     dfd[graph].get_node(node).attr['penwidth']="2"
               refresh_graph()
            elif opt_cho.GetCurrentSelection()==0:
               sparql.setQuery('''
SELECT ?res WHERE { <'''+base+sub_txt.GetValue()+'''/>
<https://w3id.org/dfd#subProcessOf> ?res 
OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) . }''')
            elif opt_cho.GetCurrentSelection()==1:
               sparql.setQuery('''
SELECT ?res WHERE { ?res 
<https://w3id.org/dfd#subProcessOf>
<'''+base+sub_txt.GetValue()+'''/>
OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) . }
 ''')
            elif opt_cho.GetCurrentSelection()==2:
               sparql.setQuery('''
SELECT DISTINCT ?res WHERE 
{{
?res <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Interface> .
?dataflow <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#DataFlow> .
?process <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Process> .
?dataflow <https://w3id.org/dfd#from> ?res .
?dataflow <https://w3id.org/dfd#to> ?process .
<'''+base+sub_txt.GetValue()+'''/> <https://w3id.org/dfd#subProcessOf> ?process OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) .
}
UNION
{
?res <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Interface> .
?dataflow <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#DataFlow> .
?process <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Process> .
?dataflow <https://w3id.org/dfd#to> ?res .
?dataflow <https://w3id.org/dfd#from> ?process .
<'''+base+sub_txt.GetValue()+'''/> <https://w3id.org/dfd#subProcessOf> ?process OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) .
}
}
''')
            elif opt_cho.GetCurrentSelection()==3:
               sparql.setQuery('''
SELECT DISTINCT ?res WHERE 
{{
?res <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Interface> .
?dataflow <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#DataFlow> .
?process <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Process> .
?dataflow <https://w3id.org/dfd#from> ?res .
?dataflow <https://w3id.org/dfd#to> ?process .
?process <https://w3id.org/dfd#subProcessOf> <'''+base+sub_txt.GetValue()+'''/> OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) .
}
UNION
{
?res <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Interface> .
?dataflow <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#DataFlow> .
?process <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://w3id.org/dfd#Process> .
?dataflow <https://w3id.org/dfd#to> ?res .
?dataflow <https://w3id.org/dfd#from> ?process .
?process <https://w3id.org/dfd#subProcessOf> <'''+base+sub_txt.GetValue()+'''/> OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) .
}
}
''')
            if not skip:
               subres = sparql.query().convert()
               for subre in subres['results']['bindings']:
                  try:
                     highlight(subre['res']['value'])
                  except:
                     print('Assuming this is not on the graph')
      def inc_button():
         global gvizlay
         if lay_btn.GetLabel() == '🎡':
            lay_btn.SetLabel('đŸ”ĩ')
            gvizlay[graph]='dot'
         elif lay_btn.GetLabel() == 'đŸ”ĩ':
            lay_btn.SetLabel('đŸĨ§')
            gvizlay[graph]='twopi'
         elif lay_btn.GetLabel() == 'đŸĨ§':
            lay_btn.SetLabel('⭕')
            gvizlay[graph]='circo'
         elif lay_btn.GetLabel() == '⭕':
            lay_btn.SetLabel('⚛ī¸')
            gvizlay[graph]='neato'
         else:
            lay_btn.SetLabel('🎡')
            gvizlay[graph]='sfdp'
         self.SetTitle(gvizlay[graph])
      def find_button():
         if graph in gvizlay:
            if gvizlay[graph]=='neato':
               lay_btn.SetLabel('⚛ī¸')
            elif gvizlay[graph]=='dot':
               lay_btn.SetLabel('đŸ”ĩ')
            elif gvizlay[graph]=='twopi':
               lay_btn.SetLabel('đŸĨ§')
            elif gvizlay[graph]=='circo':
               lay_btn.SetLabel('⭕')
            else:
               lay_btn.SetLabel('🎡')
         else:
            gvizlay[graph]='sfdp'
            lay_btn.SetLabel('🎡')
      find_button()
      def pred():
         if prd_btn.GetLabel() == '🔗':
            prd_btn.SetLabel('đŸ’Ŧ')
         elif prd_btn.GetLabel() =='đŸ’Ŧ':
            obj_txt.Hide()
            opt_cho.Show()
            prd_btn.SetLabel('❓')
            p2.Layout()
         elif prd_btn.GetLabel() =='❓':
            obj_txt.Show()
            opt_cho.Hide()
            p2.Layout()
            prd_btn.SetLabel('đŸšĢ')
         else:
            prd_btn.SetLabel('🔗')
      def update_tre():
         nav_tre.DeleteChildren(root)
         for i in natsorted(dfd):
            if i !='0':
               nav_tre.AppendItem(root,i)
               nav_tre.SortChildren(root) 
      def find_selection():   
         if nav_tre.GetSelection()!=graph:
            (child, cookie) = nav_tre.GetFirstChild(nav_tre.RootItem)
            while child.IsOk():
               if nav_tre.GetItemText(child)==graph:
                  nav_tre.SetFocusedItem(child)         
               (child, cookie) = nav_tre.GetNextChild(nav_tre.RootItem, cookie)
      def change_level():
         global dfd
         global graph
         graph=lvl_txt.GetValue()
         if graph not in dfd:
            dfd[graph]= pgv.AGraph()
            update_tre()
         find_selection()
         find_button()
         autoroute()
      def clickety_click(event):
         bits=re.compile(re.escape(rootdfd)+'(.+)')
         curr=event.GetURL()
         m = bits.match(curr)
         try:
            attempt=m.group(1)
            if graph=='0':
               aurl=attempt
            else:
               aurl=graph+'.'+attempt
            if aurl in dfd:
               lvl_txt.SetValue(aurl) 
               change_level() 
               event.Veto()
               autoroute()
         except:
            six=9

      def leaf(evt):
         item=evt.GetItem()
         selected=nav_tre.GetItemText(item)
         if selected!=graph:
            lvl_txt.SetValue(selected) 
            change_level()
      def gane_sarson_node_label(node):
         lbl=node.attr['tlabel']
         nid=node.name
         mo=re.search('\d|\.',nid)
         if nid.find('D')!=-1 and mo:
            clbl=''
            for c in nid:
               clbl+=c
            node.attr['label']='<f0> '+clbl+'|<f1> '+lbl
         elif mo:
            node.attr['label']='{<f0> '+nid+'|<f1>'+lbl+'''   

   }'''
         if not(nid.find('D')!=-1 and mo) and not mo:
            node.attr['label']=lbl
      def gane_sarson_node(node):
         nid=node.name
         node.attr['fontname']='NotoSans-Bold'
         mo=re.search('\d|\.',nid)
         if nid.find('D')!=-1 and mo:
            node.attr['shape']='record'
            node.attr['color']=colr['orange']
         elif mo:
            node.attr['shape']='Mrecord'
            if node.attr['level']=='đŸ”Ĩ':
               node.attr['color']=colr['magenta']
               node.attr['penwidth']="6"
               node.attr['tooltip']=node.attr['datetime']+': '+node.attr['message']
            elif node.attr['level']=='🌊':
               node.attr['color']=colr['blue']
               node.attr['penwidth']="2"
               node.attr['tooltip']=node.attr['datetime']+': alarm cleared'
            else:
               node.attr['color']=colr['blue']
            if node.attr['url']==None:
               node.attr['href']=nid
            elif node.attr['url']!=None:
               node.attr['href']=node.attr['url']
         if not(nid.find('D')!=-1 and mo) and not mo:
            node.attr['shape']='box'
            node.attr['color']=colr['black']
      async def process_queue():
         global dfd
         global graph
         while not q.empty():
            mess=q.get()
            b=mess.split(' ')
            if b[3] not in dfd:
               dfd[b[3]]= pgv.AGraph()
               update_tre()
            if b[0]=='đŸ”Ĩ' and b[5]==('🗝ī¸'):
               m=mess[mess.rfind('🗝ī¸')+3:]
               logger = logging.getLogger('tpv')
               sh = Rfc5424SysLogHandler(address=('127.0.0.1', 514),msg_as_utf8=False,facility=19,appname='tp-'+aid)
               logger.addHandler(sh)
               logger.critical(m)
               logger.removeHandler(sh)
               dfd[b[3]].add_node(b[4],level=b[0],datetime=b[1],message=m)
               autoroute()
            elif b[0]=='🌊' and b[5]==('🗝ī¸'):
               m=mess[mess.rfind('🗝ī¸')+3:]
               logger = logging.getLogger('tpv')
               logger.setLevel(logging.INFO)
               sh = Rfc5424SysLogHandler(address=('127.0.0.1', 514),msg_as_utf8=False,facility=19,appname='tp-'+aid)
               logger.addHandler(sh)
               logger.info(m)
               logger.removeHandler(sh)
               dfd[b[3]].add_node(b[4],level=b[0],datetime=b[1],message=m)
               autoroute()
            elif b[5]==('🎨'):
               lvl_txt.SetValue(b[6])
               change_level()
               autoroute()
            elif b[5] in ('↔ī¸','➡ī¸','âŦ…ī¸'):
               dfd[b[3]].add_node(b[4],penwidth="2")
               dfd[b[3]].add_node(b[6],penwidth="2")
               if b[5]=='↔ī¸':
                  dfd[b[3]].add_edge(b[4],b[6],dir='both',color=colr['black'],penwidth="1")
               if b[5]=='➡ī¸':
                  dfd[b[3]].add_edge(b[4],b[6],dir='forward',color=colr['black'],penwidth="1")
               if b[5]=='âŦ…ī¸':
                  dfd[b[3]].add_edge(b[4],b[6],dir='back',color=colr['black'],penwidth="1")
            elif b[5]=='🏷ī¸' and b[4].find('DataFlow')!=-1:
               c=b[4].split('DataFlow')
               dfd[b[3]].add_edge(c[0],c[1],label=mess[mess.rfind('🏷ī¸')+3:])
            elif b[5]=='🏷ī¸' and b[4].find('DataFlow')==-1:
               dfd[b[3]].add_node(b[4],tlabel=mess[mess.rfind('🏷ī¸')+3:])
               # The problem with label as an attribute is that it is customized for Gane and Sarson.
               # tlabel is just the label part, before gane_sarson_node_label hacks it up.
               gane_sarson_node_label(dfd[b[3]].get_node(b[4]))
            elif b[5]=='đŸ’Ŧ' and b[4].find('DataFlow')!=-1:
               c=b[4].split('DataFlow')
               dfd[b[3]].add_edge(c[0],c[1],tooltip=mess[mess.rfind('đŸ’Ŧ')+2:])
            elif b[5]=='đŸ’Ŧ' and b[4].find('DataFlow')==-1:
               dfd[b[3]].add_node(b[4],tooltip=mess[mess.rfind('đŸ’Ŧ')+2:])
            elif b[5]=='🔗':
               dfd[b[3]].add_node(b[4],url=mess[mess.rfind('🔗')+2:])
            elif b[5]=='đŸšĢ':
               if b[4]=='a':
                  reset()
               else:
                  if b[4]==b[6]:
                     dfd[b[3]].remove_node(b[4])
                  else:
                     dfd[b[3]].remove_edge(b[4],b[6])
            await asyncio.sleep(0.1)
      def reset():
         global dfd
         global gvizlay
         global graph
         global accel
         prd_btn.SetLabel('🔗')
         dfd.clear()
         dfd['0']= pgv.AGraph()
         nav_tre.CollapseAll()
         nav_tre.DeleteChildren(root)
         sub_txt.SetValue('')
         gvizlay={'0':'sfdp'}
         graph='0'
         accel=False
         lvl_txt.SetValue('0')
         gvizlay[graph]=='sfdp'
         find_button()
         change_level()
         autoroute()
      def populate_nodes():
         for n in dfd[graph].nodes_iter():
            gane_sarson_node(n)
      def refresh_graph():
         os.system('rm -rf /home/divine/.cache/tpv.py')
         os.system('rm /home/divine/sync/websites/site/tex/*.svg')
         # A bit lazy using os.system... need to refactor later.  Technical debt.  I also need to 
         # figure out Webkit caching and/or use a different rendering program, but I like the
         # full web presentation via webkit.
         prefix=str(uuid.uuid4())
         #I cannot figure out how to keep svg files from caching, so 
         #I am using the UUID hack.
         if fld_chk.GetValue()==True:
            labelalldfd= pgv.AGraph()
            for n in allgraph(labelalldfd).nodes_iter():
               n.attr['label']=n.name+'\\n'+n.attr['tlabel']
            labelalldfd.draw('/home/divine/sync/websites/site/tex/alldfd.dot',prog=gvizlay[graph])

         if nav_tre.IsExpanded(nav_tre.GetRootItem()):
            dfd[graph].graph_attr.update(fontname="NotoSans-Bold",sep="+10",root=rootnode, overlap="false",splines="true")
            dfd[graph].graph_attr.update(concentrate="false",pad=".5")
            dfd[graph].draw('/home/divine/sync/websites/site/tex/'+prefix+'.svg',prog=gvizlay[graph])
         else:
            alldfd= pgv.AGraph()
            allgraph(alldfd).draw('/home/divine/sync/websites/site/tex/'+prefix+'.svg',prog=gvizlay[graph])
         with open('/home/divine/sync/websites/site/tex/index.html','w') as f:
            f.write(beghtml+midhtml+prefix+'.svg'+endhtml)
         os.system('cp /home/divine/sync/websites/site/tex/'+prefix+'.svg /home/divine/sync/websites/site/tex/files/full.svg')
         mai_web.LoadURL('http://localhost:4026/')
      def on_message(client, userdata, mess):
         q.put(mess.payload.decode("utf-8"))
         res=(StartCoroutine(process_queue,self))
      client = mqttc.Client()         
      client.connect(mqttbroker, 1883, 60)
      client.on_message = on_message
      client.subscribe("allorg/dfd/full")
      client.loop_start()
      def import_dfd():
         with wx.FileDialog(self, "Import DFD", wildcard="*.dfd;*.dfd.trim",
            style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST) as fileDialog:

            if fileDialog.ShowModal() == wx.ID_CANCEL:
               return     
            pathname = fileDialog.GetPath()
            try:
               with open(pathname, 'rb') as f:
                  for l in f.read().splitlines():
                     q.put(l.decode("utf-8"))
            except IOError:
               wx.LogError("Cannot open file '%s'." % newfile)
         ts=datetime.utcnow().isoformat(sep='T', timespec='milliseconds').replace(':','').replace('-','').replace(' ','')+'Z'
         publish.single("allorg/dfd/full",'🐜 '+ts+' '+
            aid+' '+graph+' '+aid+' 🛎ī¸ '+aid,hostname=mqttbroker)
      def export_dfd():
         logfin=[]
         with wx.FileDialog(self, "Export DFD", wildcard="*.dfd.*;*dfd.trim",
                       style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as fileDialog:
            if fileDialog.ShowModal() == wx.ID_CANCEL:
               return
            pathname = fileDialog.GetPath()
            try:
               with open(pathname, 'w+') as f:
                  if pathname[-9:]=='.dfd.trim':
                     for l in log:
                        b=l.split(' ')
                        mess=b[3]+' '+b[4]+' '+b[5]+l[l.rfind(b[5]+' ')+1:]  
                        if b[0]!='🐜' and b[5]!='📐' and mess not in logfin: 
                           f.write(l+'\n')
                           logfin.append(mess)
                  else:
                     for l in log:
                        f.write(l+'\n')
            except IOError:
               wx.LogError("Cannot save current data in file '%s'." % pathname)

      def allgraph(alldfd):
         alldfd.graph_attr.update(fontname="NotoSans-Bold",sep="+10",root=rootnode, overlap="false",splines="true")
         alldfd.graph_attr.update(concentrate="false",pad=".5")
         for g in dfd:
            for e in dfd[g].edges_iter():
               gane_sarson_node(e[0])
               gane_sarson_node(e[1])
               if g=='0':
                  alldfd.add_edge(g+'.'+e[0],g+'.'+e[1])
                  alldfd.get_edge(g+'.'+e[0],g+'.'+e[1]).attr['dir']=e.attr['dir']
               else:
                  alldfd.add_edge('0.'+g+'.'+e[0],'0.'+g+'.'+e[1])
                  alldfd.get_edge('0.'+g+'.'+e[0],'0.'+g+'.'+e[1]).attr['dir']=e.attr['dir']
            for n in dfd[g].nodes_iter():
               if n.find('DataFlow')==-1:
                  if g=='0':
                     alldfd.add_node(g+'.'+n)
                     for a in ('height','width','url','href','fixedsize','fontname','label','shape','color',
                        'penwidth','tooltip','tlabel'):
                        alldfd.get_node(g+'.'+n).attr[a]=dfd[g].get_node(n).attr[a]
                  else:
                     alldfd.add_node('0.'+g+'.'+n)
                     for a in ('height','width','url','href','fixedsize','fontname','label','shape','color',
                        'penwidth','tooltip','tlabel'):
                        alldfd.get_node('0.'+g+'.'+n).attr[a]=dfd[g].get_node(n).attr[a]
            for n in dfd[g].nodes_iter():
               mo=re.search('\d|\.',n)
               if n.find('D')==-1 and mo:
                  if g!='0':
                     alldfd.get_node('0.'+g+'.'+n).attr['label']=dfd[g].get_node(n).attr['label'].replace('<f0> ','<f0> '+g+'.') 
                     b=n.split('.')
                     last=g
                     for bs in b:
                        alldfd.add_edge('0.'+last,'0.'+last+'.'+bs,dir='both')
                        last=bs
         return alldfd
      def autoroute():
         global accel
         global gvizlay
         populate_nodes()
         if accel:
            accel=False
            inc_button()
         else:
            refresh_graph()
if __name__ == '__main__':
   app = WxAsyncApp()
   frame = grp_frm()
   frame.Show()
   app.SetTopWindow(frame)
   loop = get_event_loop()
   loop.run_until_complete(app.MainLoop())

It is set up to alarm with đŸ”Ĩ as a syslog priority and deals with KVPs via 🗝ī¸, with the structured message as the object.

It also has embedded queries that use inference on the subProcessOf relations, which for the DFD tracks users and datastores associated with particular processes and their subprocesses.

scripts python analysis