diff --git a/visualise/helper.py b/visualise/helper.py
index 7a44080646b11c6270615872af16d369014daf57..39512a3fdbba680f8917285cd809dc48204f29eb 100644
--- a/visualise/helper.py
+++ b/visualise/helper.py
@@ -1,13 +1,40 @@
-
##################################################################################
#Helper functions to read raw input files and identify best combinations #
+##################################################################################
+# #
+# Copyright (C) 2018-2020 Cray Computer GmbH #
+# Copyright (C) 2021 HPE Switzerland GmbH #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are #
+# met: #
+# #
+# 1. Redistributions of source code must retain the above copyright #
+# notice, this list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright #
+# notice, this list of conditions and the following disclaimer in the #
+# documentation and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its #
+# contributors may be used to endorse or promote products derived from #
+# this software without specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
# #
-#Ali Mohammed, University of Basel #
-#<ali.mohammed@unibas.ch> #
-#All rights reserved, 2020 #
##################################################################################
-
import glob
import re
import numpy as np
@@ -18,6 +45,7 @@ from matplotlib import rc
from matplotlib import pyplot as plt
import os
from collections import namedtuple
+from dataclasses import dataclass, field
@@ -39,6 +67,171 @@ regexTime["withdraw"] = "Throughput \(withdraw/dispose\): "
regexTime["require"] = "Throughput \(declare/require\): "
+regex = {}
+regex["gid"] = "\w+-\w+-\w+-\w+-\w+."
+regex["declare"] = "mstro_pm_cdo_registry_declare.*"
+regex["offer"] = "mstro_pm_cdo_registry__set_state.*to 16 \(OFFERED,POOLED\)"
+regex["require"] = "mstro_pm_cdo_registry__set_state.*to 32 \(REQUIRED,POOLED\)"
+regex["demand"] = "mstro_pm_cdo_registry__set_state.*DEMANDED"
+regex["withdraw"] = "mstro_pm_cdo_registry__set_state.*WITHDRAWN"
+regex["trans-init"] = ".*Sent initiate-transfer for.*"
+regex["trans-comp"] = ".*TRANSFER COMPLETION from app.*"
+
+
+
+
+
+@dataclass(unsafe_hash=True)
+class CDO:
+ '''Class for keeping track of an item in inventory.'''
+ gid: str
+ declare: float = field(init=False, default=0)
+ offer: float = field(init=False, default=0)
+ demand: float = field(init=False, default=0)
+ transport_init: float = field(init=False, default=0)
+ transport_complete: float = field(init=False, default=0)
+ withdraw: float = field(init=False, default=0)
+ require: float = field(init=False, default=0)
+
+
+ def require_demand(self) -> float:
+ return self.demand - self.require
+
+ def declare_require(self) -> float:
+ return self.require - self.declare
+
+ def demand_to_transport(self) -> float:
+ return self.transport_init - self.demand
+
+ def demand_to_transportComplete(self) -> float:
+ return self.transport_complete - self.demand
+
+ def offer_to_withdraw(self) -> float:
+ return self.withdraw - self.offer
+
+ def lifetime(self) -> float:
+ return self.withdraw - self.declare
+
+
+def calculateRequireDemand(cdos):
+ require_demand = []
+ for key, cdo in cdos.items():
+ v = cdo.require_demand()
+ if v > 0:
+ require_demand.append(v)
+
+ return require_demand
+
+def calculateDeclareRequire(cdos):
+ declare_require = []
+ for key, cdo in cdos.items():
+ v = cdo.declare_require()
+ if v > 0:
+ declare_require.append(v)
+
+ return declare_require
+
+def calculateDemandTransport(cdos):
+ demand_to_transport = []
+ for key, cdo in cdos.items():
+ v = cdo.demand_to_transport()
+ if v > 0:
+ demand_to_transport.append(v)
+
+ return demand_to_transport
+
+def calculateDemandTransportComplete(cdos):
+ demand_to_transportComplete = []
+ for key, cdo in cdos.items():
+ v = cdo.demand_to_transportComplete()
+ if v > 0:
+ demand_to_transportComplete.append(v)
+
+ return demand_to_transportComplete
+
+
+def calculateLifeTime(cdos):
+ lifetime = []
+ for key, cdo in cdos.items():
+ v = cdo.lifetime()
+ if v > 0:
+ lifetime.append(cdo.lifetime())
+
+ return lifetime
+
+def getCDOs(filename):
+
+ all_gid = []
+ cdos = {}
+ i = 0
+ n_lines = -1
+ for line in open(filename, errors='ignore'):
+ n_lines = n_lines + 1
+ match = re.match( ".*" + r"(?P<gid>"+ regex["gid"] + ")" + ".*", line)
+ if match:
+ #print(line)
+ gid = match.group("gid")
+
+ if gid not in all_gid:
+ i = i + 1
+ cdos[gid] = CDO(gid = gid)
+ all_gid.append(gid)
+ if i%1000 == 0:
+ print("read " +str(i)+ " cdos after " +str(n_lines) + " lines")
+
+ #get delare time
+ if cdos[gid].declare == 0:
+ declare_time = getTime(line, regex["declare"])
+ if declare_time:
+ cdos[gid].declare = declare_time
+ continue
+ #get offer time
+ if cdos[gid].offer == 0:
+ offer_time = getTime(line, regex["offer"])
+ if offer_time:
+ cdos[gid].offer = offer_time
+ continue
+ #get demand time
+ if cdos[gid].demand == 0:
+ demand_time = getTime(line,regex["demand"])
+ if demand_time:
+ cdos[gid].demand = demand_time
+ continue
+ #get transport_init
+ if cdos[gid].transport_init == 0:
+ transport_init = getTime(line, regex["trans-init"])
+ if transport_init:
+ cdos[gid].transport_init= transport_init
+ continue
+ #get transport_compelete
+ if cdos[gid].transport_complete == 0 :
+ transport_complete = getTime(line, regex["trans-comp"])
+ if transport_complete:
+ cdos[gid].transport_complete = transport_complete
+ continue
+ #get withdraw time
+ if cdos[gid].withdraw == 0 :
+ withdraw_time = getTime(line, regex["withdraw"])
+ if withdraw_time:
+ cdos[gid].withdraw = withdraw_time
+ continue
+ #get require time
+ if cdos[gid].require == 0 :
+ require_time = getTime(line, regex["require"])
+ if require_time:
+ cdos[gid].require = require_time
+ return cdos
+
+
+def getTime(line, regex):
+ match = re.match(".*\(\w+ \d+\) "+r"(?P<time>"+ intregex + "): " + regex, line)
+ if match:
+ time = float(match.group("time"))
+ #print(time)
+ else:
+ time = None
+
+ return time
units = {}
@@ -76,7 +269,9 @@ eventsDict["mstro_pc_app_befriend"] = "mstro_pc_app_befriend\("
eventsDict["mstro_pc__handle_initiate_transfer"] = "mstro_pc__handle_initiate_transfer\("
eventsDict["mstro_pm__handle_join"] = "mstro_pm__handle_join\("
eventsDict["mstro_pm_app_register"] = "mstro_pm_app_register\("
-eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_\w+\("
+#eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_\w+\("
+eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_.*\("
+eventsDict["mstro_pm__handle_transfer_completed"] = "mstro_pm__handle_transfer_completed\("
@@ -104,6 +299,7 @@ eventsColors["mstro_pm_handle_msg"] = '#d20538' #red
eventsColors["mstro_pm__handle_join"] = '#d20538' #red
eventsColors["mstro_pm_app_register"] = '#d20538' #red
eventsColors["mstro_pm__handle_"] = '#d20538' #red
+eventsColors["mstro_pm__handle_transfer_completed"] = '#454B1B' #green
eventsColors["mstro_pc_app_befriend"] = '#454B1B' #green
@@ -127,9 +323,11 @@ eventsColors["LEAVE"] = '#FBCEB1' #Apricot
eventsColors["DECLARE"] = '#F2D2BD' #Bisque
eventsColors["SEAL"] = '#CD7F32' #Bronze
+
parameters = {}
parameters["cdo_name"] = r".*CDO `(?P<parameter>.*)'"
parameters["all"] = r".*\) (?P<parameter>.*)"
+parameters["fromapp"] = r".*\) TRANSFER COMPLETION (?P<parameter>from app \d+)"
parameters["caller"] = r".*Caller (?P<parameter>.*:\d+)"
parameters["registered_app"] = r".* (?P<parameter>Registered app \d+ for .*:\d+)"
parameters["msg_type"] = r".* ... unpacked msg: type mstro.pool.MstroMsg \((?P<parameter>.*)\)"
@@ -190,9 +388,11 @@ def plotEvents(axes, events, componentsList, plotStart=0, listAxes=0):
def plotEventsCore(ax, events, componentsList, plotStart=0):
-
+ i = 0
for event in events:
- #print(event)
+ if i%1000 == 0:
+ print("plotting event " + str(i))
+ i = i + 1
if event ==[]:
continue
@@ -211,7 +411,7 @@ def plotEventCore(ax, event, componentsList, plotStart=0):
if event.duration == 0: # nothing to do there
return
-
+ print(event)
y_pos = np.arange(len(componentsList))
@@ -293,6 +493,44 @@ def getAllEvents(filename, processID, components, componentsList, pattern="", ex
return events
+def scaleDatabyFactor(data, factor):
+ scaled = []
+ scaled.append([float(x) / factor for x in data])
+ return scaled
+
+
+def scaleDataUnit(data):
+
+
+ unit = r' $(ns)$'
+ factor = 1
+
+ maxvalue = np.max(data, initial=0)
+
+ #print(maxvalue)
+
+ if maxvalue >= 1000:
+ maxvalue = maxvalue/1000
+ unit = r' $(\mu s)$'
+ factor *= 1000
+
+ if maxvalue >= 1000:
+ maxvalue = maxvalue/1000
+ unit = r' $(ms)$'
+ factor *= 1000
+
+ if maxvalue >= 1000:
+ maxvalue = maxvalue/1000
+ unit = r' $(s)$'
+ factor *= 1000
+
+ #print(magnitude)
+ scaledData = scaleDatabyFactor(data, factor)
+
+ return scaledData, unit
+
+
+
def getEventsWithParameters(filename, processID, eventName, components, componentsList, parameter, oneofakind=0):
temp = []
diff --git a/visualise/hist.py b/visualise/hist.py
new file mode 100644
index 0000000000000000000000000000000000000000..267cf57a5ac8eb8effe36ce4a39f382bc0d72595
--- /dev/null
+++ b/visualise/hist.py
@@ -0,0 +1,171 @@
+##################################################################################
+# Extract information from maestro-core logs and create histograms #
+##################################################################################
+# #
+# Copyright (C) 2018-2020 Cray Computer GmbH #
+# Copyright (C) 2021 HPE Switzerland GmbH #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are #
+# met: #
+# #
+# 1. Redistributions of source code must retain the above copyright #
+# notice, this list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright #
+# notice, this list of conditions and the following disclaimer in the #
+# documentation and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its #
+# contributors may be used to endorse or promote products derived from #
+# this software without specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+# #
+##################################################################################
+
+import math
+import numpy as np
+import glob
+import re
+import matplotlib
+#matplotlib.use('MacOSX')
+#matplotlib.use('PS')
+from matplotlib.font_manager import FontProperties
+from collections import namedtuple
+import helper as hp
+import matplotlib.pyplot as plt
+import csv
+import os
+
+
+import sys
+from optparse import OptionParser
+
+
+import matplotlib
+
+parser = OptionParser()
+
+parser.add_option("-f", "--path", dest="path",
+ help="data file path", default=".", metavar="FILE")
+
+parser.add_option("-o", "--outpath", dest="outpath",
+ help="figure directory path", default="figure.pdf", metavar="FILE")
+
+
+parser.add_option("--zoom", dest="tlimits",
+ help="start-time,end-time", default="", metavar="FILE")
+
+(options, args) = parser.parse_args()
+
+
+
+
+cdos = hp.getCDOs(options.path)
+print("completed reading cdos")
+
+require_demand = hp.calculateRequireDemand(cdos)
+print("completed calculating require_demand")
+
+declare_require = hp.calculateDeclareRequire(cdos)
+print("completed calculating declare_require")
+
+demand_to_transport = hp.calculateDemandTransport(cdos)
+print("completed calculating demand_to_transport")
+
+demand_to_transportComplete = hp.calculateDemandTransportComplete(cdos)
+print("completed calculating demand_to_transportComplete")
+
+lifetime = hp.calculateLifeTime(cdos)
+print("completed calculating lifetime")
+
+# print(cdos)
+# print("require_demand")
+# print(require_demand)
+
+# print("declare_require")
+# print(declare_require)
+
+# print("lifetime")
+# print(lifetime)
+
+n_cdos = len(cdos)
+
+print("#CDOs = " + str(n_cdos))
+##################################################################################################
+matplotlib.rcParams.update({'font.size': 20})
+fig2, ax2 = plt.subplots()
+organized_data = []
+labels = []
+organized_data.append(require_demand)
+organized_data.append(declare_require)
+organized_data.append(demand_to_transport)
+organized_data.append(demand_to_transportComplete)
+organized_data.append(lifetime)
+
+labels.append("require-demand(#" +str(len(require_demand))+")")
+labels.append("declare-require(#" +str(len(demand_to_transport))+")")
+labels.append("demand-transport init(#" +str(len(demand_to_transport))+")")
+labels.append("demand-transport Complete(#" +str(len(demand_to_transportComplete))+")")
+labels.append("life time(#" +str(len(lifetime))+")")
+
+bd = ax2.boxplot(organized_data,whis=np.inf)
+for patch in bd['boxes']:
+ patch.set(linewidth=3)
+for patch in bd['medians']:
+ patch.set(linewidth=3)
+ax2.set_xticklabels(labels)
+plt.xticks(rotation=90, ha='center')
+ax2.set_xlabel("#CDOs = " + str(n_cdos))
+ax2.set_ylabel('Time (ns)')
+#ax2.set_autoscaley_on(False)
+#ax2.tick_params(axis = 'x', labelsize = 16)
+
+#plt.autoscale(True)
+plt.savefig(options.path+'_box.pdf', bbox_inches='tight')
+################################################################################################
+matplotlib.rcParams.update({'font.size': 20})
+i = -1
+for data in organized_data:
+ i = i +1
+ fig2, ax2 = plt.subplots()
+ scaledData, unit = hp.scaleDataUnit(data)
+ print(labels[i] + ": Average: " + str(np.mean(scaledData)) + " median: " + str(np.median(scaledData)) + unit)
+ bd = ax2.boxplot(scaledData, showfliers=False) # whis=inf
+ for patch in bd['boxes']:
+ patch.set(linewidth=3)
+ for patch in bd['medians']:
+ patch.set(linewidth=3)
+ ax2.set_xticklabels([labels[i]])
+ #plt.xticks(rotation=90, ha='center')
+ ax2.set_xlabel("#CDOs = " + str(n_cdos))
+ ax2.set_ylabel('Time ' + unit)
+ plt.savefig(options.path+'_box_'+labels[i]+'.pdf', bbox_inches='tight')
+################################################################################################
+matplotlib.rcParams.update({'font.size': 20})
+plt.style.use('ggplot')
+i = -1
+for data in organized_data:
+ i = i +1
+ scaledData, unit = hp.scaleDataUnit(data)
+ fig2, ax2 = plt.subplots()
+ bd = ax2.hist(scaledData,bins='auto') #bins='fd'
+ #ax2.set_xticklabels([labels[i]])
+ plt.xticks(rotation=90, ha='center')
+ ax2.set_title(labels[i])
+ ax2.set_xlabel("Time " + unit)
+ #ax2.set_ylabel('Time (ns)')
+ plt.savefig(options.path+'_hist_'+labels[i]+'.pdf', bbox_inches='tight')
+
\ No newline at end of file
diff --git a/visualise/vis.py b/visualise/vis.py
index b87b70110274d7b8a78b79f2c97054492c06bc8e..177dbfafbe700e82c74674fe99fe436a36e79b29 100644
--- a/visualise/vis.py
+++ b/visualise/vis.py
@@ -1,7 +1,52 @@
+##################################################################################
+# Visualize maestro-core logs #
+##################################################################################
+# #
+# Copyright (C) 2018-2020 Cray Computer GmbH #
+# Copyright (C) 2021 HPE Switzerland GmbH #
+# #
+# Redistribution and use in source and binary forms, with or without #
+# modification, are permitted provided that the following conditions are #
+# met: #
+# #
+# 1. Redistributions of source code must retain the above copyright #
+# notice, this list of conditions and the following disclaimer. #
+# #
+# 2. Redistributions in binary form must reproduce the above copyright #
+# notice, this list of conditions and the following disclaimer in the #
+# documentation and/or other materials provided with the distribution. #
+# #
+# 3. Neither the name of the copyright holder nor the names of its #
+# contributors may be used to endorse or promote products derived from #
+# this software without specific prior written permission. #
+# #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
+# #
+# #
+##################################################################################
+
+
+
+
+
+
import math
import numpy as np
import glob
import re
+import matplotlib
+#matplotlib.use('MacOSX')
+#matplotlib.use('PS')
from matplotlib.font_manager import FontProperties
from collections import namedtuple
import helper as hp
@@ -157,24 +202,30 @@ for key in components.keys():
# print(events)
# hp.plotEvents(ax, events, componentsList, startTime, listAxes)
-events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm_app_register", components, componentsList, hp.parameters["registered_app"], oneofakind=1)
-print(events)
-hp.plotEvents(ax, events, componentsList, startTime, listAxes)
+# events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm_app_register", components, componentsList, hp.parameters["registered_app"], oneofakind=1)
+# print(events)
+# hp.plotEvents(ax, events, componentsList, startTime, listAxes)
#events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_join", components, componentsList, hp.parameters["caller"], oneofakind=1)
#print(events)
#hp.plotEvents(ax, events, componentsList, startTime, listAxes)
+events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_transfer_completed", components, componentsList, hp.parameters["fromapp"], oneofakind=1)
+print(events)
+print(len(events))
+hp.plotEvents(ax, events, componentsList, startTime, listAxes)
+
events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_", components, componentsList, hp.parameters["all"], oneofakind=1)
print(events)
+print(len(events))
hp.plotEvents(ax, events, componentsList, startTime, listAxes)
-#read telemetry events
-teleEvents = hp.readTelemetryEvents(options.path, componentsList)
-hp.plotEvents(ax, teleEvents, componentsList, startTime, listAxes)
+# #read telemetry events
+# teleEvents = hp.readTelemetryEvents(options.path, componentsList)
+# hp.plotEvents(ax, teleEvents, componentsList, startTime, listAxes)
legendcols = 7
@@ -187,7 +238,9 @@ if tlimits != []:
#plt.xlim(tlimits)
else:
ax.legend(ncol = legendcols, fontsize = 12, bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand", borderaxespad=0.)
-#plt.show()
+
+print("Drawing ...")
+# plt.show()
plt.savefig(options.outpath, bbox_inches='tight')