Skip to content
Snippets Groups Projects
Commit ede34602 authored by Ali Mohammed's avatar Ali Mohammed
Browse files

add hist.py to calculate histogram of cdo operations from logs

parent 42c074bb
No related branches found
No related tags found
1 merge request!41Draft: Resolve "visualize maestro logs"
##################################################################################
#Helper functions to read raw input files and identify best combinations #
##################################################################################
# #
# Copyright (C) 2018-2020 Cray Computer GmbH #
# Copyright (C) 2021 HPE Switzerland GmbH #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are #
# met: #
# #
# 1. Redistributions of source code must retain the above copyright #
# notice, this list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright #
# notice, this list of conditions and the following disclaimer in the #
# documentation and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its #
# contributors may be used to endorse or promote products derived from #
# this software without specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
# #
#Ali Mohammed, University of Basel #
#<ali.mohammed@unibas.ch> #
#All rights reserved, 2020 #
##################################################################################
import glob
import re
import numpy as np
......@@ -18,6 +45,7 @@ from matplotlib import rc
from matplotlib import pyplot as plt
import os
from collections import namedtuple
from dataclasses import dataclass, field
......@@ -39,6 +67,171 @@ regexTime["withdraw"] = "Throughput \(withdraw/dispose\): "
regexTime["require"] = "Throughput \(declare/require\): "
regex = {}
regex["gid"] = "\w+-\w+-\w+-\w+-\w+."
regex["declare"] = "mstro_pm_cdo_registry_declare.*"
regex["offer"] = "mstro_pm_cdo_registry__set_state.*to 16 \(OFFERED,POOLED\)"
regex["require"] = "mstro_pm_cdo_registry__set_state.*to 32 \(REQUIRED,POOLED\)"
regex["demand"] = "mstro_pm_cdo_registry__set_state.*DEMANDED"
regex["withdraw"] = "mstro_pm_cdo_registry__set_state.*WITHDRAWN"
regex["trans-init"] = ".*Sent initiate-transfer for.*"
regex["trans-comp"] = ".*TRANSFER COMPLETION from app.*"
@dataclass(unsafe_hash=True)
class CDO:
'''Class for keeping track of an item in inventory.'''
gid: str
declare: float = field(init=False, default=0)
offer: float = field(init=False, default=0)
demand: float = field(init=False, default=0)
transport_init: float = field(init=False, default=0)
transport_complete: float = field(init=False, default=0)
withdraw: float = field(init=False, default=0)
require: float = field(init=False, default=0)
def require_demand(self) -> float:
return self.demand - self.require
def declare_require(self) -> float:
return self.require - self.declare
def demand_to_transport(self) -> float:
return self.transport_init - self.demand
def demand_to_transportComplete(self) -> float:
return self.transport_complete - self.demand
def offer_to_withdraw(self) -> float:
return self.withdraw - self.offer
def lifetime(self) -> float:
return self.withdraw - self.declare
def calculateRequireDemand(cdos):
require_demand = []
for key, cdo in cdos.items():
v = cdo.require_demand()
if v > 0:
require_demand.append(v)
return require_demand
def calculateDeclareRequire(cdos):
declare_require = []
for key, cdo in cdos.items():
v = cdo.declare_require()
if v > 0:
declare_require.append(v)
return declare_require
def calculateDemandTransport(cdos):
demand_to_transport = []
for key, cdo in cdos.items():
v = cdo.demand_to_transport()
if v > 0:
demand_to_transport.append(v)
return demand_to_transport
def calculateDemandTransportComplete(cdos):
demand_to_transportComplete = []
for key, cdo in cdos.items():
v = cdo.demand_to_transportComplete()
if v > 0:
demand_to_transportComplete.append(v)
return demand_to_transportComplete
def calculateLifeTime(cdos):
lifetime = []
for key, cdo in cdos.items():
v = cdo.lifetime()
if v > 0:
lifetime.append(cdo.lifetime())
return lifetime
def getCDOs(filename):
all_gid = []
cdos = {}
i = 0
n_lines = -1
for line in open(filename, errors='ignore'):
n_lines = n_lines + 1
match = re.match( ".*" + r"(?P<gid>"+ regex["gid"] + ")" + ".*", line)
if match:
#print(line)
gid = match.group("gid")
if gid not in all_gid:
i = i + 1
cdos[gid] = CDO(gid = gid)
all_gid.append(gid)
if i%1000 == 0:
print("read " +str(i)+ " cdos after " +str(n_lines) + " lines")
#get delare time
if cdos[gid].declare == 0:
declare_time = getTime(line, regex["declare"])
if declare_time:
cdos[gid].declare = declare_time
continue
#get offer time
if cdos[gid].offer == 0:
offer_time = getTime(line, regex["offer"])
if offer_time:
cdos[gid].offer = offer_time
continue
#get demand time
if cdos[gid].demand == 0:
demand_time = getTime(line,regex["demand"])
if demand_time:
cdos[gid].demand = demand_time
continue
#get transport_init
if cdos[gid].transport_init == 0:
transport_init = getTime(line, regex["trans-init"])
if transport_init:
cdos[gid].transport_init= transport_init
continue
#get transport_compelete
if cdos[gid].transport_complete == 0 :
transport_complete = getTime(line, regex["trans-comp"])
if transport_complete:
cdos[gid].transport_complete = transport_complete
continue
#get withdraw time
if cdos[gid].withdraw == 0 :
withdraw_time = getTime(line, regex["withdraw"])
if withdraw_time:
cdos[gid].withdraw = withdraw_time
continue
#get require time
if cdos[gid].require == 0 :
require_time = getTime(line, regex["require"])
if require_time:
cdos[gid].require = require_time
return cdos
def getTime(line, regex):
match = re.match(".*\(\w+ \d+\) "+r"(?P<time>"+ intregex + "): " + regex, line)
if match:
time = float(match.group("time"))
#print(time)
else:
time = None
return time
units = {}
......@@ -76,7 +269,9 @@ eventsDict["mstro_pc_app_befriend"] = "mstro_pc_app_befriend\("
eventsDict["mstro_pc__handle_initiate_transfer"] = "mstro_pc__handle_initiate_transfer\("
eventsDict["mstro_pm__handle_join"] = "mstro_pm__handle_join\("
eventsDict["mstro_pm_app_register"] = "mstro_pm_app_register\("
eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_\w+\("
#eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_\w+\("
eventsDict["mstro_pm__handle_"] = "mstro_pm__handle_.*\("
eventsDict["mstro_pm__handle_transfer_completed"] = "mstro_pm__handle_transfer_completed\("
......@@ -104,6 +299,7 @@ eventsColors["mstro_pm_handle_msg"] = '#d20538' #red
eventsColors["mstro_pm__handle_join"] = '#d20538' #red
eventsColors["mstro_pm_app_register"] = '#d20538' #red
eventsColors["mstro_pm__handle_"] = '#d20538' #red
eventsColors["mstro_pm__handle_transfer_completed"] = '#454B1B' #green
eventsColors["mstro_pc_app_befriend"] = '#454B1B' #green
......@@ -127,9 +323,11 @@ eventsColors["LEAVE"] = '#FBCEB1' #Apricot
eventsColors["DECLARE"] = '#F2D2BD' #Bisque
eventsColors["SEAL"] = '#CD7F32' #Bronze
parameters = {}
parameters["cdo_name"] = r".*CDO `(?P<parameter>.*)'"
parameters["all"] = r".*\) (?P<parameter>.*)"
parameters["fromapp"] = r".*\) TRANSFER COMPLETION (?P<parameter>from app \d+)"
parameters["caller"] = r".*Caller (?P<parameter>.*:\d+)"
parameters["registered_app"] = r".* (?P<parameter>Registered app \d+ for .*:\d+)"
parameters["msg_type"] = r".* ... unpacked msg: type mstro.pool.MstroMsg \((?P<parameter>.*)\)"
......@@ -190,9 +388,11 @@ def plotEvents(axes, events, componentsList, plotStart=0, listAxes=0):
def plotEventsCore(ax, events, componentsList, plotStart=0):
i = 0
for event in events:
#print(event)
if i%1000 == 0:
print("plotting event " + str(i))
i = i + 1
if event ==[]:
continue
......@@ -211,7 +411,7 @@ def plotEventCore(ax, event, componentsList, plotStart=0):
if event.duration == 0: # nothing to do there
return
print(event)
y_pos = np.arange(len(componentsList))
......@@ -293,6 +493,44 @@ def getAllEvents(filename, processID, components, componentsList, pattern="", ex
return events
def scaleDatabyFactor(data, factor):
scaled = []
scaled.append([float(x) / factor for x in data])
return scaled
def scaleDataUnit(data):
unit = r' $(ns)$'
factor = 1
maxvalue = np.max(data, initial=0)
#print(maxvalue)
if maxvalue >= 1000:
maxvalue = maxvalue/1000
unit = r' $(\mu s)$'
factor *= 1000
if maxvalue >= 1000:
maxvalue = maxvalue/1000
unit = r' $(ms)$'
factor *= 1000
if maxvalue >= 1000:
maxvalue = maxvalue/1000
unit = r' $(s)$'
factor *= 1000
#print(magnitude)
scaledData = scaleDatabyFactor(data, factor)
return scaledData, unit
def getEventsWithParameters(filename, processID, eventName, components, componentsList, parameter, oneofakind=0):
temp = []
......
##################################################################################
# Extract information from maestro-core logs and create histograms #
##################################################################################
# #
# Copyright (C) 2018-2020 Cray Computer GmbH #
# Copyright (C) 2021 HPE Switzerland GmbH #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are #
# met: #
# #
# 1. Redistributions of source code must retain the above copyright #
# notice, this list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright #
# notice, this list of conditions and the following disclaimer in the #
# documentation and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its #
# contributors may be used to endorse or promote products derived from #
# this software without specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
# #
##################################################################################
import math
import numpy as np
import glob
import re
import matplotlib
#matplotlib.use('MacOSX')
#matplotlib.use('PS')
from matplotlib.font_manager import FontProperties
from collections import namedtuple
import helper as hp
import matplotlib.pyplot as plt
import csv
import os
import sys
from optparse import OptionParser
import matplotlib
parser = OptionParser()
parser.add_option("-f", "--path", dest="path",
help="data file path", default=".", metavar="FILE")
parser.add_option("-o", "--outpath", dest="outpath",
help="figure directory path", default="figure.pdf", metavar="FILE")
parser.add_option("--zoom", dest="tlimits",
help="start-time,end-time", default="", metavar="FILE")
(options, args) = parser.parse_args()
cdos = hp.getCDOs(options.path)
print("completed reading cdos")
require_demand = hp.calculateRequireDemand(cdos)
print("completed calculating require_demand")
declare_require = hp.calculateDeclareRequire(cdos)
print("completed calculating declare_require")
demand_to_transport = hp.calculateDemandTransport(cdos)
print("completed calculating demand_to_transport")
demand_to_transportComplete = hp.calculateDemandTransportComplete(cdos)
print("completed calculating demand_to_transportComplete")
lifetime = hp.calculateLifeTime(cdos)
print("completed calculating lifetime")
# print(cdos)
# print("require_demand")
# print(require_demand)
# print("declare_require")
# print(declare_require)
# print("lifetime")
# print(lifetime)
n_cdos = len(cdos)
print("#CDOs = " + str(n_cdos))
##################################################################################################
matplotlib.rcParams.update({'font.size': 20})
fig2, ax2 = plt.subplots()
organized_data = []
labels = []
organized_data.append(require_demand)
organized_data.append(declare_require)
organized_data.append(demand_to_transport)
organized_data.append(demand_to_transportComplete)
organized_data.append(lifetime)
labels.append("require-demand(#" +str(len(require_demand))+")")
labels.append("declare-require(#" +str(len(demand_to_transport))+")")
labels.append("demand-transport init(#" +str(len(demand_to_transport))+")")
labels.append("demand-transport Complete(#" +str(len(demand_to_transportComplete))+")")
labels.append("life time(#" +str(len(lifetime))+")")
bd = ax2.boxplot(organized_data,whis=np.inf)
for patch in bd['boxes']:
patch.set(linewidth=3)
for patch in bd['medians']:
patch.set(linewidth=3)
ax2.set_xticklabels(labels)
plt.xticks(rotation=90, ha='center')
ax2.set_xlabel("#CDOs = " + str(n_cdos))
ax2.set_ylabel('Time (ns)')
#ax2.set_autoscaley_on(False)
#ax2.tick_params(axis = 'x', labelsize = 16)
#plt.autoscale(True)
plt.savefig(options.path+'_box.pdf', bbox_inches='tight')
################################################################################################
matplotlib.rcParams.update({'font.size': 20})
i = -1
for data in organized_data:
i = i +1
fig2, ax2 = plt.subplots()
scaledData, unit = hp.scaleDataUnit(data)
print(labels[i] + ": Average: " + str(np.mean(scaledData)) + " median: " + str(np.median(scaledData)) + unit)
bd = ax2.boxplot(scaledData, showfliers=False) # whis=inf
for patch in bd['boxes']:
patch.set(linewidth=3)
for patch in bd['medians']:
patch.set(linewidth=3)
ax2.set_xticklabels([labels[i]])
#plt.xticks(rotation=90, ha='center')
ax2.set_xlabel("#CDOs = " + str(n_cdos))
ax2.set_ylabel('Time ' + unit)
plt.savefig(options.path+'_box_'+labels[i]+'.pdf', bbox_inches='tight')
################################################################################################
matplotlib.rcParams.update({'font.size': 20})
plt.style.use('ggplot')
i = -1
for data in organized_data:
i = i +1
scaledData, unit = hp.scaleDataUnit(data)
fig2, ax2 = plt.subplots()
bd = ax2.hist(scaledData,bins='auto') #bins='fd'
#ax2.set_xticklabels([labels[i]])
plt.xticks(rotation=90, ha='center')
ax2.set_title(labels[i])
ax2.set_xlabel("Time " + unit)
#ax2.set_ylabel('Time (ns)')
plt.savefig(options.path+'_hist_'+labels[i]+'.pdf', bbox_inches='tight')
\ No newline at end of file
##################################################################################
# Visualize maestro-core logs #
##################################################################################
# #
# Copyright (C) 2018-2020 Cray Computer GmbH #
# Copyright (C) 2021 HPE Switzerland GmbH #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are #
# met: #
# #
# 1. Redistributions of source code must retain the above copyright #
# notice, this list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright #
# notice, this list of conditions and the following disclaimer in the #
# documentation and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its #
# contributors may be used to endorse or promote products derived from #
# this software without specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS #
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED #
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A #
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT #
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, #
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, #
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY #
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
# #
##################################################################################
import math
import numpy as np
import glob
import re
import matplotlib
#matplotlib.use('MacOSX')
#matplotlib.use('PS')
from matplotlib.font_manager import FontProperties
from collections import namedtuple
import helper as hp
......@@ -157,24 +202,30 @@ for key in components.keys():
# print(events)
# hp.plotEvents(ax, events, componentsList, startTime, listAxes)
events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm_app_register", components, componentsList, hp.parameters["registered_app"], oneofakind=1)
print(events)
hp.plotEvents(ax, events, componentsList, startTime, listAxes)
# events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm_app_register", components, componentsList, hp.parameters["registered_app"], oneofakind=1)
# print(events)
# hp.plotEvents(ax, events, componentsList, startTime, listAxes)
#events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_join", components, componentsList, hp.parameters["caller"], oneofakind=1)
#print(events)
#hp.plotEvents(ax, events, componentsList, startTime, listAxes)
events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_transfer_completed", components, componentsList, hp.parameters["fromapp"], oneofakind=1)
print(events)
print(len(events))
hp.plotEvents(ax, events, componentsList, startTime, listAxes)
events = hp.getEventsWithParameters(options.path, list(components.keys())[0], "mstro_pm__handle_", components, componentsList, hp.parameters["all"], oneofakind=1)
print(events)
print(len(events))
hp.plotEvents(ax, events, componentsList, startTime, listAxes)
#read telemetry events
teleEvents = hp.readTelemetryEvents(options.path, componentsList)
hp.plotEvents(ax, teleEvents, componentsList, startTime, listAxes)
# #read telemetry events
# teleEvents = hp.readTelemetryEvents(options.path, componentsList)
# hp.plotEvents(ax, teleEvents, componentsList, startTime, listAxes)
legendcols = 7
......@@ -187,6 +238,8 @@ if tlimits != []:
#plt.xlim(tlimits)
else:
ax.legend(ncol = legendcols, fontsize = 12, bbox_to_anchor=(0., 1.02, 1., .102), loc=3, mode="expand", borderaxespad=0.)
print("Drawing ...")
# plt.show()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment