diff --git a/helper.py b/helper.py new file mode 100644 index 0000000000000000000000000000000000000000..fac9f882a818aae8ad89ac12dd8b0eb7fdf2b795 --- /dev/null +++ b/helper.py @@ -0,0 +1,515 @@ + +################################################################################## +#Helper functions to read raw input files and identify best combinations # +# # +#Ali Mohammed, University of Basel # +#<ali.mohammed@unibas.ch> # +#All rights reserved, 2020 # +################################################################################## + + +import glob +import re +import numpy as np +import csv +import math +import matplotlib as mpl +from matplotlib import rc +from matplotlib import pyplot as plt +import os +from collections import namedtuple + + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +#application times regex dictionary +regexTime = {} +regexTime["declare"] = "Throughput \(declare/offer\): " +regexTime["withdraw"] = "Throughput \(withdraw/dispose\): " +regexTime["require"] = "Throughput \(declare/require\): " + + + + +units = {} +units["us"] = 1 +units["ms"] = 1000 +units["s"] = 1000000 + +unitText = {} +unitText["us"] = "$\mu$s" +unitText["ms"] = "$ms$" +unitText["s"] = "$s$" + + +floatregex = "[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?" +intregex = "\d+" +processregex = "t\d+" +nameregex = "\w+:\d+" + +legend_labels = [] + +event = namedtuple("event", "name start duration processID componentID parameter") + +eventsDict = {} +eventsDict["mstro_*_init"] = "mstro_.*_init\(" +eventsDict["mstro_*_finalize"] = "mstro_.*finalize\(" +eventsDict["mstro_cdo_declare"] = "mstro_cdo_declare.*\(" +eventsDict["mstro_cdo_offer"] = "mstro_cdo_offer.*\(" +eventsDict["mstro_cdo_demand"] = "mstro_cdo_demand.*\(" +eventsDict["mstro_cdo_withdraw"] = "mstro_cdo_withdraw.*\(" +eventsDict["mstro_cdo_dispose"] = "mstro_cdo_dispose.*\(" +eventsDict["mstro_pm_attach"] = "mstro_pm_attach.*\(" +eventsDict["mstro_pc_handle_msg"] = "mstro_pc_handle_msg.*\(" +eventsDict["mstro_pm_handle_msg"] = "mstro_pm_handle_msg.*\(" +eventsDict["mstro_pc_app_befriend"] = "mstro_pc_app_befriend\(" +eventsDict["mstro_pc__handle_initiate_transfer"] = "mstro_pc__handle_initiate_transfer\(" + + +#mstro_pc__handle_initiate_transfer\(.*\) Initiating transfer from .*dst app \d+ of CDO \w+ +# mstro_pc__handle_initiate_transfer source ...dst ... +# mstro_ofi__select_endpoint ... too noisy +# mstro_pc__handle_poolop_ack ...operation + + +eventsColors = {} +eventsColors["mstro_*_init"] = 'gray' +eventsColors["mstro_*_finalize"] = 'black' +eventsColors["mstro_cdo_declare"] = "#bcd4e6" # blue +eventsColors["mstro_cdo_offer"] = '#6cbfee' # blue +eventsColors["mstro_cdo_demand"] = '#1f75fe' # blue +eventsColors["mstro_cdo_withdraw"] = '#00008b' #blue +eventsColors["mstro_cdo_dispose"] = '#1f305e' #blue + + +#https://htmlcolorcodes.com/colors/shades-of-green/ +eventsColors["mstro_pm_attach"] = '#7FFFD4' #green +eventsColors["mstro_pc_handle_msg"] = '#088F8F' #green +eventsColors["mstro_pm_handle_msg"] = '#d20538' #red +eventsColors["mstro_pc_app_befriend"] = '#454B1B' #green +eventsColors["mstro_pc__handle_initiate_transfer"] = '#AAFF00' #green + + +""" +Amber #FFBF00 rgb(255, 191, 0) +Apricot #FBCEB1 rgb(251, 206, 177) +Bisque #F2D2BD rgb(242, 210, 189) +Bronze #CD7F32 rgb(205, 127, 50) +Buff #DAA06D rgb(218, 160, 109) +Burnt Orange #CC5500 rgb(204, 85, 0) +Burnt Sienna #E97451 rgb(233, 116, 81) +Butterscotch #E3963E rgb(227, 150, 62) +""" + +#telemetry events +eventsColors["JOIN"] = '#FFBF00' #Amber +eventsColors["LEAVE"] = '#FBCEB1' #Apricot +eventsColors["DECLARE"] = '#F2D2BD' #Bisque +eventsColors["SEAL"] = '#CD7F32' #Bronze + +parameters = {} +parameters["cdo_name"] = r".*CDO `(?P<parameter>.*)'" +parameters["msg_type"] = r".* ... unpacked msg: type mstro.pool.MstroMsg \((?P<parameter>.*)\)" +parameters["friends"] = r".*\) (?P<parameter>.*)" +parameters["dst_app_cdo"] = r".*\) Initiating transfer.*(?P<parameter>dst app \d+ of CDO \w+)" + +mpl.rcParams.update({'font.size': 14}) +bar_width = 0.2 + +def initPlot(components, title="", zoom=""): + #print(len(components)) + y_pos = np.arange(len(components)) # the width + if zoom != "": + fig, [ax, ax1] = plt.subplots(2, 1, figsize=(40, 12), gridspec_kw={'height_ratios': [1, 3]}) + ax1.set_title("Zoom", fontsize=46, y = 1.015) + data = [0] * len(components) + ax1.barh(y_pos, data , height = bar_width, align='center', color='gray') + ax1.set_yticks(y_pos) + ax1.set_yticklabels(components) + ax1.set_ylim(0,len(components)+1) + ax.set_ylim(0,len(components)+6) + axes = [ax, ax1] + else: + fig, ax = plt.subplots(figsize=(100, 4)) + axes = ax + ax.set_ylim(0,len(components)+6) + if title != "": + ax.set_title(title, y = 1.015) + + data = [0] * len(components) + results = ax.barh(y_pos, data , height = bar_width, align='center', color='gray') + + ax.set_yticks(y_pos) + ax.set_yticklabels(components) + + fig.tight_layout() + + return axes + + +def fetchComponentID(processID, components, componentsList): + componentName = components[processID] + componentID = componentsList.index(componentName) + return componentID + + +def plotEvents(axes, events, componentsList, plotStart=0, tlimits = [], listAxes=0): + if listAxes == 1: + for ax in axes: + plotEventsCore(ax, events, componentsList, plotStart, tlimits) + else: + plotEventsCore(axes, events, componentsList, plotStart, tlimits) + + +def plotEventsCore(ax, events, componentsList, plotStart=0, tlimits = []): + + for event in events: + #print(event) + if event ==[]: + continue + + plotEventCore(ax, event, componentsList, plotStart, tlimits) + +def plotEvent(axes, event, componentsList, plotStart=0, tlimits = [], listAxes=0): + if listAxes == 1: + for ax in axes: + plotEventCore(ax, event, componentsList, plotStart, tlimits) + else: + plotEventCore(axes, event, componentsList, plotStart, tlimits) + +def plotEventCore(ax, event, componentsList, plotStart=0, tlimits = []): + + + #if tlimits != []: + # if ((event.start- plotStart) < (tlimits[0])) or ((event.start - plotStart+ event.duration) > (tlimits[1])): + # return + + if event.duration == 0: # nothing to do there + return + + + y_pos = np.arange(len(componentsList)) + + + #print(event) + #auto detected events + if event.name == "": + ax.barh(y_pos[event.componentID], event.duration, left=(event.start - plotStart) ,height = bar_width, align='center', color='gray') + #ax.text((event.start - plotStart) + event.duration/2, y_pos[event.componentID] + bar_width/2, event.parameter , ha='center', va='bottom', rotation=90, fontsize = 6, color='gray') + x = (event.start - plotStart) + event.duration/2 + y = y_pos[event.componentID] + bar_width/2 + ax.annotate(event.parameter , xy =(x, y), xytext =(x, y), ha='center', va='bottom', rotation=90, fontsize = 6, color='gray') + return + + if event.name in legend_labels: + ax.barh(y_pos[event.componentID], event.duration, left=(event.start - plotStart) ,height = bar_width, align='center', color=eventsColors[event.name]) + else: + ax.barh(y_pos[event.componentID], event.duration, left=(event.start - plotStart) ,height = bar_width, align='center', color=eventsColors[event.name], label = event.name) + legend_labels.append(event.name) + + if event.parameter != "": + x = (event.start - plotStart) + event.duration/2 + y = y_pos[event.componentID] + bar_width/2 + ax.annotate(event.parameter , xy =(x, y), xytext =(x, y), ha='center', va='bottom', rotation=90, fontsize = 12, color=eventsColors[event.name]) + #ax.text((event.start - plotStart) + event.duration/2, y_pos[event.componentID] + bar_width/2, event.parameter , ha='center', va='bottom', rotation=90, fontsize = 12, color=eventsColors[event.name]) + +def getStartTime(filename, processID): + for line in open(filename): + match = re.search( processID + " .* " + r"(?P<time>"+ intregex + "): " + "mstro_log__init\(" , line) + if match: + #print(line) + time = float(match.group("time")) + break + + return time + + +def getAllEvents(filename, processID, components, componentsList, pattern="", exclude = ""): + temp = [] + + events = [] + + componentID = fetchComponentID(processID, components, componentsList) + + + + #[I:cdo] C1 t11018273 (C02FQ3K9MD6T 59229) 804367159223000: mstro_cdo_declare_core(cdo.c:435) Declared CDO `Test CDO name üÜöÖäÄ' + eventPattern = "mstro_\w+" + if pattern != "": + eventPattern = pattern + + for line in open(filename): + match = re.search( processID + " .* " + r"(?P<time>"+ intregex + "): " + r"(?P<eventName>"+eventPattern+")\(", line) + #match = re.search(eventsDict[eventName] + ""+ parameter, line) + if match: + name = match.group("eventName") + if name in exclude: + continue + t =float(match.group("time")) + print(name) + + #fake a very small duration to register the event + duration = 100000 + events.append(event(name = "", start = t, duration = duration, processID= processID, componentID = componentID, parameter=name)) + + else: #try match with component name + match = re.search( components[processID]+" .* " + r"(?P<time>"+ intregex + "): " + r"(?P<eventName>"+eventPattern+")\(", line) + if match: + name = match.group("eventName") + if name in exclude: + continue + t =float(match.group("time")) + print(name) + + #fake a very small duration to register the event + duration = 100000 + events.append(event(name = "", start = t, duration = duration, processID= processID, componentID = componentID, parameter=name)) + + + return events + + + +def getEventsWithParameters(filename, processID, eventName, components, componentsList, parameter, oneofakind=0): + temp = [] + times = {} + + componentID = fetchComponentID(processID, components, componentsList) + events = [] + #[I:cdo] C1 t11018273 (C02FQ3K9MD6T 59229) 804367159223000: mstro_cdo_declare_core(cdo.c:435) Declared CDO `Test CDO name üÜöÖäÄ' + + for line in open(filename): + match = re.search( processID + " .* " + r"(?P<time>"+ intregex + "): " + eventsDict[eventName] + parameter , line) + #match = re.search(eventsDict[eventName] + ""+ parameter, line) + if match: + param = match.group("parameter") + t =float(match.group("time")) + #print(param) + if oneofakind == 1: + # create the event directly + duration = 100000 + events.append(event(name = eventName, start = t, duration = duration, processID= processID, componentID = componentID, parameter=param)) + elif not param in times: + temp = [] + temp.append(t) + times[param] = temp + else: + times[param].append(t) + + else: #try match with component name + match = re.search( components[processID]+" .* " + r"(?P<time>"+ intregex + "): " + eventsDict[eventName] + parameter , line) + if match: + param = match.group("parameter") + t =float(match.group("time")) + #print(param) + if oneofakind == 1: + # create the event directly + duration = 100000 + events.append(event(name = eventName, start = t, duration = duration, processID= processID, componentID = componentID, parameter=param)) + elif not param in times: + temp = [] + temp.append(t) + times[param] = temp + else: + times[param].append(t) + + + + #create an event for every key + for key in times: + if times[key] == []: + start = 0 + duration = 0 + else: + start = np.min(times[key]) + end = np.max(times[key]) + duration = end - start + if duration == 0: + #fake a very small duration to register the event + duration = 100000 + events.append(event(name = eventName, start = start, duration = duration, processID= processID, componentID = componentID, parameter=key)) + + return events + + +def getComponentListFromTelemetry(filename): + componentsList = [] + + for line in open(filename): + match = re.search(r"(?P<time>"+ intregex +r"),(?P<type>\w+),\d+, { (?P<appID>\d+),\"(?P<name>.*)\"\s?}", line ) + if match: + eventName = match.group("type") + + if eventName == "JOIN": + componentsList.append(match.group("name")) + + + return componentsList + +def readTelemetryEvents(filename, componentsList=[]): + + if componentsList == []: + componentsList = getComponentListFromTelemetry(filename) + + apps = {} + events = [] + for line in open(filename): + match = re.search(r"(?P<time>"+ intregex +r"),(?P<type>\w+),\d+, { (?P<appID>\d+)(,\"(?P<name>.*)\")?\s?}", line ) + if match: + print(line) + + # create dict apps with componentID, appID -- with JOIN events + eventName = match.group("type") + appID = match.group("appID") + start = float(match.group("time")) + #fake a very small duration to register the event + duration = 100000 + + if eventName == "JOIN": + name = match.group("name") + #componentID = componentsList.index(name + r"(:0)?") + componentID = [i for i, item in enumerate(componentsList) if re.search(name + r"(:\d+)?", item)] + apps[appID] = name + + events.append(event(name = eventName, start = start, duration = duration, processID= 0, componentID = componentID, parameter= "app " + appID)) + + if eventName == "LEAVE": + name = apps[appID] + componentID = [i for i, item in enumerate(componentsList) if re.search(name + r"(:\d+)?", item)] + + events.append(event(name = eventName, start = start, duration = duration, processID= 0, componentID = componentID, parameter= "app " + appID)) + + + if eventName == "DECLARE" or eventName == "SEAL": + cdoName = match.group("name") + name = apps[appID] + componentID = [i for i, item in enumerate(componentsList) if re.search(name + r"(:\d+)?", item)] + + events.append(event(name = eventName, start = start, duration = duration, processID= 0, componentID = componentID, parameter= cdoName)) + + return events + +def getEvent(filename, processID, eventName, components, componentsList): + temp = [] + #[D:cdo] C2 PC (C02FQ3K9MD6T 59230) 804367160004000: + + for line in open(filename): + match = re.search( processID + " .* " + r"(?P<time>"+ intregex + "): " + eventsDict[eventName] , line) + if match: + #print(line) + temp.append(float(match.group("time"))) + else: #try match with component name + match = re.search( components[processID]+" .* " + r"(?P<time>"+ intregex + "): " + eventsDict[eventName] , line) + if match: + #print(line) + temp.append(float(match.group("time"))) + + #print(str(event.name) + " " + str(temp) ) + if temp == []: + start = 0 + duration = 0 + else: + start = np.min(temp) + end = np.max(temp) + duration = end - start + if duration == 0: + #fake a very small duration to register the event + duration = 100000 + + componentID = fetchComponentID(processID, components, componentsList) + + return event(name = eventName, start = start, duration = duration, processID= processID, componentID = componentID, parameter="") + +def getValueFromFile(filename, processID, funPattern): + + #t11018272 (C02FQ3K9MD6T 59230) 804367140962000: mstro_ofi_init( + temp = [] + + for line in open(filename): + match = re.search( processID + " \(.*\) " + r"(?P<time>"+ intregex + "): " + funPattern , line) + if match: + #print(line) + temp.append(float(match.group("time"))) + #else: + #print(line) + return temp + + +def getComponents(filename): + + d = getProcessIDs(filename) + d = getComponentNames(filename, d) + + return d + +def getProcessIDs(filename): + + d = {} + #[D:core] t11018272 + for line in open(filename): + match = re.search("\[.*\] :\d+ \d+ " + r"(?P<ProcessID>"+ processregex + ") ", line) + if match: + ProcessID = match.group("ProcessID") + #print(line) + if not ProcessID in d: + d[ProcessID] = "" + #print("added " + ProcessID + " to dictionary") + + return d + +def getComponentNames(filename, dictIDs): + + for key in dictIDs.keys(): + for line in open(filename): + match = re.search("\[.*\] " + r"(?P<componentName>"+ nameregex + ") \d+ " + key, line) + if match: + componentName = match.group("componentName") + dictIDs[key] = componentName + print("found id: " + str(key) + " name: " + str(componentName)) + break + + return dictIDs + + +def formatValue(value): + magnitude = 1024 + + label = "" + if value >= magnitude: + value = value/magnitude + label = "k" + + if value >= magnitude: + value = value/magnitude + label = "M" + + if value >= magnitude: + value = value/magnitude + label = "G" + + return str(int(value)) + label + + + +#convert dict to list +def dict2List(dict): + + dictlist = [] + + for key in dict.keys(): + + dictlist.append(dict[key]) + + return dictlist + + diff --git a/vis.py b/vis.py new file mode 100644 index 0000000000000000000000000000000000000000..c17da520ee82d6bf76997048e2a0a9fab2db5521 --- /dev/null +++ b/vis.py @@ -0,0 +1,173 @@ +import math +import numpy as np +import glob +import re +from matplotlib.font_manager import FontProperties +from collections import namedtuple +import helper as hp +import matplotlib.pyplot as plt +import csv +import os + + +import sys +from optparse import OptionParser + + +import matplotlib + +""" +python3 vis.py --path check_pm_declare.sh.log --zoom 30000000,300000000 + +python3 vis.py --path check_pm_interlock.sh.log --zoom 35000000,80000000 + +""" + +parser = OptionParser() + +parser.add_option("-f", "--path", dest="path", + help="data file path", default=".", metavar="FILE") + +parser.add_option("-o", "--outpath", dest="outpath", + help="figure directory path", default="figure.pdf", metavar="FILE") + + +parser.add_option("--zoom", dest="tlimits", + help="start-time,end-time", default="", metavar="FILE") + +(options, args) = parser.parse_args() + + + +listAxes = 0 +tlimits = [] +if options.tlimits != "": + tlimits = [int(x) for x in options.tlimits.split(",")] + listAxes = 1 + +#print(tlimits) + +components = hp.getComponents(options.path) +print(components) + + + +componentsList = hp.dict2List(components) +#print(componentsList) + + + + +ax = hp.initPlot(componentsList, zoom=options.tlimits) + +#skip PM start time ...take the minimum of the components +s = [] +for i in {1: len(components)}: + s.append(hp.getStartTime(options.path, list(components.keys())[i])) +startTime = np.min(s) +print("startTime: " + str(startTime)) + + + +for key in components.keys(): + # plot mstro_*_init + if key == list(components.keys())[0]: #skip PM + continue + init = hp.getEvent(options.path, key, "mstro_*_init", components, componentsList) + #print(init.duration) + hp.plotEvent(ax, init, componentsList, startTime, tlimits, listAxes) + #print(init.start - startTime - tlimits[0]) + + + #plot "mstro_cdo_declare" + events = hp.getEventsWithParameters(options.path, key, "mstro_cdo_declare", components, componentsList, hp.parameters["cdo_name"]) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + #plot "mstro_cdo_offer" + events = hp.getEventsWithParameters(options.path, key, "mstro_cdo_offer", components, componentsList, hp.parameters["cdo_name"]) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + #plot "mstro_cdo_withdraw" + events = hp.getEventsWithParameters(options.path, key, "mstro_cdo_withdraw", components, componentsList, hp.parameters["cdo_name"]) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + #plot "mstro_cdo_demand" + events = hp.getEventsWithParameters(options.path, key, "mstro_cdo_demand", components, componentsList, hp.parameters["cdo_name"]) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + #plot "mstro_cdo_dispose" + events = hp.getEventsWithParameters(options.path, key, "mstro_cdo_dispose", components, componentsList, hp.parameters["cdo_name"]) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + events = hp.getEventsWithParameters(options.path, key, "mstro_pc_handle_msg", components, componentsList, hp.parameters["msg_type"], oneofakind=1) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + events = hp.getEventsWithParameters(options.path, key, "mstro_pc__handle_initiate_transfer", components, componentsList, hp.parameters["dst_app_cdo"], oneofakind=1) + #print(events) + hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + + #events = hp.getEventsWithParameters(options.path, key, "mstro_pc_app_befriend", components, componentsList, hp.parameters["friends"], oneofakind=1) + #print(events) + #hp.plotEvents(ax, events, componentsList, startTime) + + + + fini = hp.getEvent(options.path, key, "mstro_*_finalize", components, componentsList) + #print(fini) + hp.plotEvent(ax, fini, componentsList, startTime, tlimits, listAxes) + #print("fini") + #print(fini.start + fini.duration - startTime - tlimits[1]) + + attach = hp.getEvent(options.path, key, "mstro_pm_attach", components, componentsList) + #print(fini) + hp.plotEvent(ax, attach, componentsList, startTime, tlimits, listAxes) + + + +#c1_events = hp.getAllEvents(options.path, list(components.keys())[1], components, componentsList, pattern = "mstro_\w+") +#tlimits = [startTime, fini.start ] +#p.plotEvents(ax, c1_events, componentsList, startTime, tlimits) + +#get all PM events +#pm_events = hp.getAllEvents(options.path, list(components.keys())[0], components, componentsList, pattern = "mstro_pm__handle\w+") +#tlimits = [startTime, fini.start ] +#hp.plotEvents(ax, pm_events, componentsList, startTime, tlimits) + + +events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm_handle_msg", components, componentsList, hp.parameters["msg_type"], oneofakind=1) +#print(events) +hp.plotEvents(ax, events, componentsList, startTime, tlimits, listAxes) + + +#read telemetry events +teleEvents = hp.readTelemetryEvents(options.path, componentsList) +hp.plotEvents(ax, teleEvents, componentsList, startTime, tlimits, listAxes) + + +legendcols = 7 +if tlimits != []: + #ax.set_xlim(tlimits) + #ax.set_xlim(xmin = tlimits[0], xmax=tlimits[1]) + ax[1].set_xbound(tlimits) + ax[0].legend(ncol = legendcols, fontsize = 12, bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand", borderaxespad=0.) + #plt.xlim(tlimits) +else: + ax.legend(ncol = legendcols, fontsize = 12, bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand", borderaxespad=0.) +#plt.show() + + +plt.savefig(options.outpath, bbox_inches='tight') +plt.close() \ No newline at end of file