diff --git a/ImHex_Patterns/LinkTest_2-1-18_SIONlib_1-7-6.hexpat b/ImHex_Patterns/LinkTest_2-1-18_SIONlib_1-7-6.hexpat
new file mode 100644
index 0000000000000000000000000000000000000000..d143f5f70e16f3ce31185f6381963e89cd5c578d
--- /dev/null
+++ b/ImHex_Patterns/LinkTest_2-1-18_SIONlib_1-7-6.hexpat
@@ -0,0 +1,220 @@
+#include <std/sys.pat>
+#include <std/io.pat>
+#include <std/cint.pat>
+
+// Valid for LinkTest Version 2.1.17
+// Valid for SIONlib Version 1.7.6 or 1.7.7
+// SIONlib bug: 1.7.7 forgot to increment the patch level saved to the file
+
+// ImHex Bugs:
+// 1: Attributes applied to an array do not inherit to its elements.
+// This makes conversion of the displayed values in case of endinaness changes impossible.
+
+// TODO: Currently only works if there is only one SION block in total and one SION chunk per task.
+
+std::size_t offset;
+std::size_t numIter;
+bool bes; //If true SION headers are big endian
+bool bel; //If true LinkTest data is big-endian
+
+struct SION_Header_{
+ if(bes){
+ be u32 Version;
+ be u32 Patchlevel;
+ be u32 File_version;
+ be u32 Block_Size;
+ be u32 Num_Tasks;
+ be u32 Num_Files;
+ be u32 File_Num;
+ }else{
+ le u32 Version;
+ le u32 Patchlevel;
+ le u32 File_version;
+ le u32 Block_Size;
+ le u32 Num_Tasks;
+ le u32 Num_Files;
+ le u32 File_Num;
+ }
+ padding[16];
+ char Filename[1024];
+ if(bes){
+ be u64 GlobalRank[Num_Tasks];
+ be u64 Chunk_Size[Num_Tasks];
+ be u32 Max_Chunks;
+ be u64 Var_Start;
+ }else{
+ le u64 GlobalRank[Num_Tasks];
+ le u64 Chunk_Size[Num_Tasks];
+ le u32 Max_Chunks;
+ le u64 Var_Start;
+ }
+ if(Block_Size>1){
+ padding[Block_Size-$];
+ }
+};
+
+struct LinkTest_Header_{
+ if(bel){
+ be u32 Major_Version;
+ be u32 Minor_Version;
+ be u32 Patchlevel;
+ }else{
+ le u32 Major_Version;
+ le u32 Minor_Version;
+ le u32 Patchlevel;
+ }
+ char Git_Hash[41];
+ if(bel){
+ be u32 Mode_Length [[hidden]];
+ }else{
+ le u32 Mode_Length [[hidden]];
+ }
+ char Mode[Mode_Length];
+ u8 A2A_Flag;
+ u8 Bidirectional_Flag;
+ u8 Unidirectional_Flag;
+ u8 Bisection_Flag;
+ u8 Step_Randomization_Flag;
+ u8 Serialize_Testing_Flag;
+ u8 No_SION_File_Flag;
+ u8 Parallel_SION_File_Flag;
+ u8 GPU_Memory_Flag;
+ u8 Multi_Buffer_Flag;
+ u8 Randomized_Buffer_Flag;
+ u8 Check_Buffer_Flag;
+ u8 Memory_Allocator_Type;
+ if(bel){
+ be u64 Num_Messages;
+ be u64 Size_Messages;
+ be u64 Num_WarmUp_Messages;
+ be u64 Num_Serial_Retests;
+ be u64 Num_Multi_Buffer;
+ be u64 Buffer_Randomization_Seed;
+ be u64 Num_Randomized_Task;
+ be u64 Task_Randomization_Seed;
+ be u64 Task_Randomization_Seed;
+ }else{
+ le u64 Num_Messages;
+ le u64 Size_Messages;
+ le u64 Num_WarmUp_Messages;
+ le u64 Num_Serial_Retests;
+ le u64 Num_Multi_Buffer;
+ le u64 Buffer_Randomization_Seed;
+ le u64 Num_Randomized_Task;
+ le u64 Task_Randomization_Seed;
+ le u64 Step_Randomization_Seed;
+ }
+ char END_HEADER[10] [[hidden]];
+ std::assert(END_HEADER=="END_HEADER","LinkTest header did not end with END_HEADER");
+};
+
+struct Host_{
+ if(bel){
+ be u32 Hostname_Length [[hidden,transform("Change_Endianness_u32")]];
+ }else{
+ le u32 Hostname_Length [[hidden]];
+ }
+ char Hostname[Hostname_Length];
+ if(bel){
+ be u32 Core_ID [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+ }else{
+ le u32 Core_ID;
+ }
+};
+
+struct IterationData_{
+ if(rank==0){
+ char Start_Time[32];
+ if(bel){
+ be double Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ if(LinkTest_Header.A2A_Flag){
+ be double A2A_Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double A2A_Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double A2A_Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ }
+ be double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be u64 Retests_From[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+ be u64 Retests_To[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+ }else{
+ le double Minimum_Time;
+ le double Average_Time;
+ le double Maximum_Time;
+ if(LinkTest_Header.A2A_Flag){
+ le double A2A_Minimum_Time;
+ le double A2A_Average_Time;
+ le double A2A_Maximum_Time;
+ }
+ le double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+ le double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+ le u64 Retests_From[LinkTest_Header.Num_Serial_Retests];
+ le u64 Retests_To[LinkTest_Header.Num_Serial_Retests];
+ }
+ char End_Time[32];
+ }
+ if(bel){
+ be double Timings[SION_Header.Num_Tasks];
+ be u64 AccessPattern[SION_Header.Num_Tasks];
+ if(LinkTest_Header.A2A_Flag){
+ be double A2A_Time;
+ }
+ }else{
+ le double Timings[SION_Header.Num_Tasks];
+ le u64 AccessPattern[SION_Header.Num_Tasks];
+ if(LinkTest_Header.A2A_Flag){
+ le double A2A_Time;
+ }
+ }
+};
+
+struct Data_{
+ Host_ Host;
+ IterationData_ Data[numIter];
+ char END_BLOCK[9] [[hidden]];
+ std::assert(END_BLOCK=="END_BLOCK","LinkTest per-rank data did not end with END_BLOCK");
+ rank=rank+1;
+};
+
+char SION_String[4] @ 0x0 [[hidden]];
+std::assert(SION_String=="sion","File does not start with sion!");
+u8 Endianness_Flags[4] @ $ [[hidden]];
+offset=$;
+std::assert(Endianness_Flags[0]==Endianness_Flags[3],"Endianness Flag not for SIONlib data not symmetric");
+if(Endianness_Flags[0]==1){
+ bes=true;
+}else{
+ bes=false;
+}
+std::assert(Endianness_Flags[1]==Endianness_Flags[2],"Endianness Flag not for LinkTest data not symmetric");
+if(Endianness_Flags[1]==1){
+ bel=true;
+}else{
+ bel=false;
+}
+$=offset;
+SION_Header_ SION_Header @ $ [[comment("SIONlib File Header"), name("SIONlib Header")]];
+offset=$;
+std::assert(SION_Header.Version==1007,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Patchlevel==6,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Num_Files==1,"No support for multi-file SIONlib data!");
+std::assert(SION_Header.Num_Files==1,"No support for multiple SION chunks!");
+$=offset;
+char ID_String[8] @ $ [[hidden]];
+offset=$;
+std::assert_warn(ID_String=="LinkTest","Failed to match LinkTest!");
+$=offset;
+LinkTest_Header_ LinkTest_Header @ $ [[comment("LinkTest File Header"), name("LinkTest Header")]];
+offset=$;
+std::assert_warn(LinkTest_Header.No_SION_File_Flag==0,"No-SION-File Flag set!");
+if(LinkTest_Header.Num_Randomized_Task!=0){
+ numIter=LinkTest_Header.Num_Randomized_Task;
+}else{
+ numIter=1;
+}
+u64 rank;
+$=offset;
+Data_ Data[SION_Header.Num_Tasks] @ $;
+
+
diff --git a/ImHex_Patterns/LinkTest_2-1-19_SIONlib_1-7-6.hexpat b/ImHex_Patterns/LinkTest_2-1-19_SIONlib_1-7-6.hexpat
new file mode 100644
index 0000000000000000000000000000000000000000..0bdf2b334d39c9f395f84ed65136abc85bfcf572
--- /dev/null
+++ b/ImHex_Patterns/LinkTest_2-1-19_SIONlib_1-7-6.hexpat
@@ -0,0 +1,219 @@
+#include <std/sys.pat>
+#include <std/io.pat>
+#include <std/cint.pat>
+
+// Valid for LinkTest Version 2.1.17
+// Valid for SIONlib Version 1.7.6 or 1.7.7
+// SIONlib bug: 1.7.7 forgot to increment the patch level saved to the file
+
+// ImHex Bugs:
+// 1: Attributes applied to an array do not inherit to its elements.
+// This makes conversion of the displayed values in case of endinaness changes impossible.
+
+// TODO: Currently only works if there is only one SION block in total and one SION chunk per task.
+
+std::size_t offset;
+std::size_t numIter;
+bool bes; //If true SION headers are big endian
+bool bel; //If true LinkTest data is big-endian
+
+struct SION_Header_{
+ if(bes){
+ be u32 Version;
+ be u32 Patchlevel;
+ be u32 File_version;
+ be u32 Block_Size;
+ be u32 Num_Tasks;
+ be u32 Num_Files;
+ be u32 File_Num;
+ }else{
+ le u32 Version;
+ le u32 Patchlevel;
+ le u32 File_version;
+ le u32 Block_Size;
+ le u32 Num_Tasks;
+ le u32 Num_Files;
+ le u32 File_Num;
+ }
+ padding[16];
+ char Filename[1024];
+ if(bes){
+ be u64 GlobalRank[Num_Tasks];
+ be u64 Chunk_Size[Num_Tasks];
+ be u32 Max_Chunks;
+ be u64 Var_Start;
+ }else{
+ le u64 GlobalRank[Num_Tasks];
+ le u64 Chunk_Size[Num_Tasks];
+ le u32 Max_Chunks;
+ le u64 Var_Start;
+ }
+ if(Block_Size>1){
+ padding[Block_Size-$];
+ }
+};
+
+struct LinkTest_Header_{
+ if(bel){
+ be u32 Major_Version;
+ be u32 Minor_Version;
+ be u32 Patchlevel;
+ }else{
+ le u32 Major_Version;
+ le u32 Minor_Version;
+ le u32 Patchlevel;
+ }
+ char Git_Hash[41];
+ if(bel){
+ be u32 Mode_Length [[hidden]];
+ }else{
+ le u32 Mode_Length [[hidden]];
+ }
+ char Mode[Mode_Length];
+ u8 A2A_Flag;
+ u8 Bidirectional_Flag;
+ u8 Unidirectional_Flag;
+ u8 Bisection_Flag;
+ u8 Step_Randomization_Flag;
+ u8 Serialize_Testing_Flag;
+ u8 No_SION_File_Flag;
+ u8 Parallel_SION_File_Flag;
+ u8 GPU_Memory_Flag;
+ u8 Multi_Buffer_Flag;
+ u8 Randomized_Buffer_Flag;
+ u8 Check_Buffer_Flag;
+ u8 Group_Hostname_Flag;
+ u8 Memory_Allocator_Type;
+ if(bel){
+ be u64 Num_Messages;
+ be u64 Size_Messages;
+ be u64 Num_WarmUp_Messages;
+ be u64 Num_Serial_Retests;
+ be u64 Num_Multi_Buffer;
+ be u64 Buffer_Randomization_Seed;
+ be u64 Num_Randomized_Task;
+ be u64 Task_Randomization_Seed;
+ be u64 Step_Randomization_Seed;
+ }else{
+ le u64 Num_Messages;
+ le u64 Size_Messages;
+ le u64 Num_WarmUp_Messages;
+ le u64 Num_Serial_Retests;
+ le u64 Num_Multi_Buffer;
+ le u64 Buffer_Randomization_Seed;
+ le u64 Num_Randomized_Task;
+ le u64 Task_Randomization_Seed;
+ le u64 Step_Randomization_Seed;
+ }
+ char END_HEADER[10] [[hidden]];
+ std::assert(END_HEADER=="END_HEADER","LinkTest header did not end with END_HEADER");
+};
+
+struct Host_{
+ if(bel){
+ be u32 Hostname_Length [[hidden,transform("Change_Endianness_u32")]];
+ }else{
+ le u32 Hostname_Length [[hidden]];
+ }
+ char Hostname[Hostname_Length];
+ if(bel){
+ be u32 Core_ID [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+ }else{
+ le u32 Core_ID;
+ }
+};
+
+struct IterationData_{
+ if(rank==0){
+ char Start_Time[32];
+ if(bel){
+ be double Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ if(LinkTest_Header.A2A_Flag){
+ be double A2A_Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double A2A_Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double A2A_Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ }
+ be double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+ be u64 Retests_From[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+ be u64 Retests_To[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+ }else{
+ le double Minimum_Time;
+ le double Average_Time;
+ le double Maximum_Time;
+ if(LinkTest_Header.A2A_Flag){
+ le double A2A_Minimum_Time;
+ le double A2A_Average_Time;
+ le double A2A_Maximum_Time;
+ }
+ le double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+ le double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+ le u64 Retests_From[LinkTest_Header.Num_Serial_Retests];
+ le u64 Retests_To[LinkTest_Header.Num_Serial_Retests];
+ }
+ char End_Time[32];
+ }
+ if(bel){
+ be double Timings[SION_Header.Num_Tasks];
+ be u64 AccessPattern[SION_Header.Num_Tasks];
+ if(LinkTest_Header.A2A_Flag){
+ be double A2A_Time;
+ }
+ }else{
+ le double Timings[SION_Header.Num_Tasks];
+ le u64 AccessPattern[SION_Header.Num_Tasks];
+ if(LinkTest_Header.A2A_Flag){
+ le double A2A_Time;
+ }
+ }
+};
+
+struct Data_{
+ Host_ Host;
+ IterationData_ Data[numIter];
+ char END_BLOCK[9] [[hidden]];
+ std::assert(END_BLOCK=="END_BLOCK","LinkTest per-rank data did not end with END_BLOCK");
+ rank=rank+1;
+};
+
+char SION_String[4] @ 0x0 [[hidden]];
+std::assert(SION_String=="sion","File does not start with sion!");
+u8 Endianness_Flags[4] @ $ [[hidden]];
+offset=$;
+std::assert(Endianness_Flags[0]==Endianness_Flags[3],"Endianness Flag not for SIONlib data not symmetric");
+if(Endianness_Flags[0]==1){
+ bes=true;
+}else{
+ bes=false;
+}
+std::assert(Endianness_Flags[1]==Endianness_Flags[2],"Endianness Flag not for LinkTest data not symmetric");
+if(Endianness_Flags[1]==1){
+ bel=true;
+}else{
+ bel=false;
+}
+$=offset;
+SION_Header_ SION_Header @ $ [[comment("SIONlib File Header"), name("SIONlib Header")]];
+offset=$;
+std::assert(SION_Header.Version==1007,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Patchlevel==6,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Num_Files==1,"No support for multi-file SIONlib data!");
+std::assert(SION_Header.Num_Files==1,"No support for multiple SION chunks!");
+$=offset;
+char ID_String[8] @ $ [[hidden]];
+offset=$;
+std::assert_warn(ID_String=="LinkTest","Failed to match LinkTest!");
+$=offset;
+LinkTest_Header_ LinkTest_Header @ $ [[comment("LinkTest File Header"), name("LinkTest Header")]];
+offset=$;
+std::assert_warn(LinkTest_Header.No_SION_File_Flag==0,"No-SION-File Flag set!");
+if(LinkTest_Header.Num_Randomized_Task!=0){
+ numIter=LinkTest_Header.Num_Randomized_Task;
+}else{
+ numIter=1;
+}
+u64 rank;
+$=offset;
+Data_ Data[SION_Header.Num_Tasks] @ $;
diff --git a/benchmark/.gitignore b/benchmark/.gitignore
index 827abf02e0d9316a8b561e22cb8a5b31bb206b19..fe3166c57d06a42cc310cdb5f346f43d5b7544ca 100644
--- a/benchmark/.gitignore
+++ b/benchmark/.gitignore
@@ -1 +1,10 @@
+# Ignore generated source files
cuda_kernels.cc
+# Ignore executables
+**/linktest
+**/linktest.mpi
+**/linktest.tcp
+**/linktest.cuda
+**/linktest.psm2
+**/linktest.ucp
+**/linktest.ibverbs
\ No newline at end of file
diff --git a/benchmark/benchmark.cc b/benchmark/benchmark.cc
index 6717b033c9d9f06868839f4b775dea46772416d2..d913e122f1b125af49e322e56134c72f2f5fb4fa 100644
--- a/benchmark/benchmark.cc
+++ b/benchmark/benchmark.cc
@@ -28,6 +28,7 @@
#include <vector>
#include <math.h>
#include <iostream>
+#include <sstream>
#include <numeric>
#include <algorithm>
#include <cstdlib>
@@ -44,6 +45,26 @@ int Benchmark::size() const{
return cl->size();
}
+int Benchmark::numHostRanks() const{
+ return cl->numHostRanks();
+}
+
+int Benchmark::numLocalRanks() const{
+ return cl->numLocalRanks();
+}
+
+int Benchmark::hostRank() const{
+ return cl->hostRank();
+}
+
+int Benchmark::localRank() const{
+ return cl->localRank();
+}
+
+std::shared_ptr<int[]> Benchmark::hostLocalRanks() const{
+ return cl->hostLocalRanks();
+}
+
void Benchmark::barrier() const{
auto err = cl->barrier();
if (unlikely(err)) {
@@ -101,7 +122,11 @@ inline unsigned int Benchmark::getTo(int i, int j, int step) const {
}
inline unsigned int Benchmark::getNumSteps() const {
- return args->do_bisection ? size()/2 : size()-1;
+ if(args->do_bisection!=0){
+ if(args->do_group_processes_by_hostname!=0) return numHostRanks()*numLocalRanks()/2;
+ else return size()/2;
+ }else if(args->do_group_processes_by_hostname!=0) return (numHostRanks()-1)*numLocalRanks();
+ else return size()-1;
}
const int INVALID_RANK = -8; // Random negative Number
@@ -115,16 +140,14 @@ void Benchmark::computeRankStepMappings() {
std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
rootWatch->start();
- std::iota(stepPermutation.begin(), stepPermutation.end(), 0);
if(args->do_mix) {
- std::time_t t = 0;
if( rank() == 0 ) {
- t = std::time(0);
+ std::iota(stepPermutation.begin(), stepPermutation.end(), 0);
+ std::shuffle(stepPermutation.begin(), stepPermutation.end(), randomNumberEngineSteps);
}
- cl->bcast(0, &t, 1);
-
- std::srand(t);
- std::random_shuffle(stepPermutation.begin(), stepPermutation.end());
+ cl->bcast(0, stepPermutation.data(), getNumSteps());
+ }else{
+ std::iota(stepPermutation.begin(), stepPermutation.end(), 0);
}
rootWatch->stop();
printTimingIfRoot(cl->rank(), "[randvec]", rootWatch->getDuration());
@@ -133,24 +156,42 @@ void Benchmark::computeRankStepMappings() {
std::fill(getRankFromStep.begin(), getRankFromStep.end(), INVALID_RANK);
std::fill(getStepFromRank.begin(), getStepFromRank.end(), INVALID_STEP);
if(args->do_bisection) {
- mapWithBisection();
+ mapWithBisection(args->num_randomize_tasks > 0, args->do_group_processes_by_hostname != 0);
} else {
- mapWith1FactorAlgorithm(args->num_randomize_tasks > 0);
+ mapWith1FactorAlgorithm(args->num_randomize_tasks > 0, args->do_group_processes_by_hostname != 0);
}
rootWatch->stop();
printTimingIfRoot(cl->rank(), "[getpart]", rootWatch->getDuration());
- #ifdef DEBUG_PRINT_COMMUNICATION_SCHEME
- for(int i = 0; i < size(); i++) {
- barrier();
- if( rank() == i) {
- std::cout << rank() << ": getRankFromStep: " << getRankFromStep << "\n";
- std::cout << rank() << ": getStepFromRank: " << getStepFromRank << "\n";
- std::cout << rank() << ": stepPermutation: " << stepPermutation << "\n";
+#ifdef DEBUG_PRINT_COMMUNICATION_SCHEME
+ if( rank() == 0) {
+ std::stringstream buf;
+ buf << "getRankFromStep: " << getRankFromStep;
+ debug("[%3d] %s", 0, buf.str().c_str());buf.str(std::string());
+ buf << "getStepFromRank: " << getStepFromRank;
+ debug("[%3d] %s", 0, buf.str().c_str());buf.str(std::string());
+ buf << "stepPermutation: " << stepPermutation;
+ debug("[%3d] %s", 0, buf.str().c_str());buf.str(std::string());
+ std::vector<int> buf1;buf1.resize(getNumSteps());
+ std::vector<int> buf2;buf2.resize(size() );
+ for(int i = 1; i < size(); i++) {
+ cl->recv(i, buf1.data(), buf1.size());
+ buf << "getRankFromStep: " << buf1;
+ debug("[%3d] %s", i, buf.str().c_str());buf.str(std::string());
+ cl->recv(i, buf2.data(), buf2.size());
+ buf << "getStepFromRank: " << buf2;
+ debug("[%3d] %s", i, buf.str().c_str());buf.str(std::string());
+ cl->recv(i, buf1.data(), buf1.size());
+ buf << "stepPermutation: " << buf1;
+ debug("[%3d] %s", i, buf.str().c_str());buf.str(std::string());
}
+ } else {
+ cl->send(0, getRankFromStep.data(), getRankFromStep.size());
+ cl->send(0, getStepFromRank.data(), getStepFromRank.size());
+ cl->send(0, stepPermutation.data(), stepPermutation.size());
}
barrier();
- #endif
+#endif
}
std::vector<int> Benchmark::oneFactorAlgorithm(const int size, const int rank) {
@@ -179,44 +220,98 @@ std::vector<int> Benchmark::oneFactorAlgorithm(const int size, const int rank) {
return rankFromStep;
}
-void Benchmark::mapWith1FactorAlgorithm(bool randomizeTasks) {
+void Benchmark::mapWith1FactorAlgorithm(bool randomizeTasks, bool group_processes_by_hostname) {
// We use the 1-factor algorithm for parallel communications to reduces congestion
// Implementation (including names) taken from
// https://en.wikipedia.org/wiki/All-to-all_(parallel_pattern)#1-factor_algorithm
- std::vector<int> randomrankFromRank(size());
- std::vector<int> rankFromRandomrank(size());
+ const int numRanks=(group_processes_by_hostname?numHostRanks():size());
+
+ std::vector<int> randomrankFromRank(numRanks);
+ std::vector<int> randomLocalRankFromLocalRank;
+ if(group_processes_by_hostname) randomLocalRankFromLocalRank.resize(numLocalRanks());
if(randomizeTasks) {
if(rank() == 0){
std::iota(randomrankFromRank.begin(), randomrankFromRank.end(), 0);
- std::shuffle(randomrankFromRank.begin(), randomrankFromRank.end(), randomNumberEngine);
+ std::shuffle(randomrankFromRank.begin(), randomrankFromRank.end(), randomNumberEngineTasks);
+ if(group_processes_by_hostname){
+ std::iota(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), 0);
+ std::shuffle(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), randomNumberEngineTasks);
+ }
}
- cl->bcast(0, randomrankFromRank.data(), size());
+ cl->bcast(0, randomrankFromRank.data(), numRanks);
+ if(group_processes_by_hostname) cl->bcast(0, randomLocalRankFromLocalRank.data(), numLocalRanks());
}else{
std::iota(randomrankFromRank.begin(), randomrankFromRank.end(), 0);
+ if(group_processes_by_hostname) std::iota(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), 0);;
}
+ std::vector<int> rankFromRandomrank(numRanks);
for(std::size_t i = 0; i < randomrankFromRank.size(); i++) {
rankFromRandomrank[randomrankFromRank[i]] = i;
}
- auto randomRankFromStep= oneFactorAlgorithm(size(), randomrankFromRank[rank()]);
- for (unsigned int step = 0; step < getNumSteps(); step++) {
- const int partnerRank = rankFromRandomrank[randomRankFromStep[step]];
- getRankFromStep.at(step) = partnerRank;
- getStepFromRank.at(partnerRank) = step;
+ if(group_processes_by_hostname){
+ auto randomRankFromStep = oneFactorAlgorithm(numRanks, randomrankFromRank[hostRank()]);
+ int step=0;
+ for (int hRank = 0; hRank < numRanks-1; hRank++) { //Iterate through host ranks
+ const int partnerHostRank = rankFromRandomrank[randomRankFromStep[hRank]];
+ for (int lRank = 0; lRank < numLocalRanks(); lRank++) { //Iterate through local ranks
+ const int partnerRank = hostLocalRanks().get()[partnerHostRank*numLocalRanks()+(numLocalRanks()+(localRank()+(partnerHostRank>hostRank()?1:-1)*randomLocalRankFromLocalRank[lRank]))%numLocalRanks()];
+ getRankFromStep.at(step) = partnerRank;
+ getStepFromRank.at(partnerRank) = step;
+ step++;
+ }
+ }
+ }else{
+ auto randomRankFromStep= oneFactorAlgorithm(size(), randomrankFromRank[rank()]);
+ for (int step = 0; step < numRanks - 1; step++) {
+ const int partnerRank = rankFromRandomrank[randomRankFromStep[step]];
+ getRankFromStep.at(step) = partnerRank;
+ getStepFromRank.at(partnerRank) = step;
+ }
}
}
-void Benchmark::mapWithBisection() {
- const int n = size()/2;
- const int r = rank();
- for( auto i=0; i<n; i++ ){
- if( r < n ){
- getRankFromStep.at(i) = n + (n - 1 - r + i) % n;
- } else {
- getRankFromStep.at(i) = (2*n - 1 - r + i) % n;
+void Benchmark::mapWithBisection(bool randomizeTasks, bool group_processes_by_hostname) {
+ const int n = (group_processes_by_hostname ? numHostRanks()/2 : size()/2);
+ const int r = (group_processes_by_hostname ? hostRank() : rank());
+ std::vector<int> randomRankFromRank(n);
+ std::vector<int> randomLocalRankFromLocalRank;
+ if(group_processes_by_hostname) randomLocalRankFromLocalRank.resize(numLocalRanks());
+ if(randomizeTasks){
+ if( rank() == 0 ){
+ std::iota(randomRankFromRank.begin(), randomRankFromRank.end(), 0);
+ std::shuffle(randomRankFromRank.begin(), randomRankFromRank.end(), randomNumberEngineTasks);
+ if(group_processes_by_hostname){
+ std::iota(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), 0);
+ std::shuffle(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), randomNumberEngineTasks);
+ }
+ }
+ cl->bcast(0, randomRankFromRank.data(), n);
+ if(group_processes_by_hostname) cl->bcast(0, randomLocalRankFromLocalRank.data(), numLocalRanks());
+ }else{
+ std::iota(randomRankFromRank.begin(), randomRankFromRank.end(), 0);
+ if(group_processes_by_hostname) std::iota(randomLocalRankFromLocalRank.begin(), randomLocalRankFromLocalRank.end(), 0);
+ }
+ if(group_processes_by_hostname){
+ int k=0;
+ for(int hRank=0; hRank<n; hRank++ ){
+ const int partnerHostRank=(r<n?n+(n-1-r+randomRankFromRank[hRank])%n:(2*n -1-r+randomRankFromRank[hRank])%n);
+ for(int lRank=0; lRank<numLocalRanks(); lRank++ ){
+ getRankFromStep.at(k)=hostLocalRanks().get()[partnerHostRank*numLocalRanks()+(numLocalRanks()+(localRank()+(partnerHostRank>hostRank()?1:-1)*randomLocalRankFromLocalRank[lRank]))%numLocalRanks()];
+ getStepFromRank.at(getRankFromStep.at(k)) = k;
+ k++;
+ }
+ }
+ }else{
+ for( auto i=0; i<n; i++ ){
+ if( r < n ){
+ getRankFromStep.at(i) = n + ( n - 1 - r + randomRankFromRank[i]) % n;
+ } else {
+ getRankFromStep.at(i) = (2*n - 1 - r + randomRankFromRank[i]) % n;
+ }
+ getStepFromRank.at(getRankFromStep.at(i)) = i;
}
- getStepFromRank.at(getRankFromStep.at(i)) = i;
}
}
@@ -339,21 +434,42 @@ int Benchmark::work_alltoall(){
int Benchmark::work_pingpong_parallel(const int partner,const int sign, double* const time_per_msg){
double tmp1, tmp2;
- barrier();
-
auto from = (sign < 0) ? partner : rank();
auto to = (sign < 0) ? rank() : partner;
+ barrier();
+#ifdef DEBUG_KERNEL_SYNCHRONIZATION
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(rank());
+ duration_t tBeforeBarrier;
+ rootWatch->start();
+#endif
EXEC_NOFAIL(kernel(from, to, &tmp1, true));
-
+#ifdef DEBUG_KERNEL_SYNCHRONIZATION
+ rootWatch->stop();
+ tBeforeBarrier=rootWatch->getDuration();
+#endif
barrier();
-
+#ifdef DEBUG_KERNEL_SYNCHRONIZATION
+ rootWatch->stop();
+ printTimingIfRoot(rank(), "[Kernel A->B Before Barrier]", tBeforeBarrier );
+ printTimingIfRoot(rank(), "[Kernel A->B After Barrier]", rootWatch->getDuration());
+ barrier(); //Additional barrier to reduce desynchronization due to printing
+ rootWatch->start();
+#endif
EXEC_NOFAIL(kernel(to, from, &tmp2, true));
+#ifdef DEBUG_KERNEL_SYNCHRONIZATION
+ rootWatch->stop();
+ tBeforeBarrier=rootWatch->getDuration();
+#endif
+ barrier();
+#ifdef DEBUG_KERNEL_SYNCHRONIZATION
+ rootWatch->stop();
+ printTimingIfRoot(rank(), "[Kernel B->A Before Barrier]", tBeforeBarrier );
+ printTimingIfRoot(rank(), "[Kernel B->A After Barrier]", rootWatch->getDuration());
+#endif
*time_per_msg = (sign > 0) ? tmp1 : tmp2;
- barrier();
-
return SUCCESS;
}
@@ -729,9 +845,19 @@ int Benchmark::init() {
}
if(args->num_randomize_tasks > 0) {
- randomNumberEngine = std::mt19937(args->seed_randomize_tasks);
+ randomNumberEngineTasks = std::mt19937(args->seed_randomize_tasks);
} else {
- randomNumberEngine = std::mt19937(0);
+ randomNumberEngineTasks = std::mt19937(0);
+ }
+
+ if(args->num_randomize_tasks > 0) {
+ randomNumberEngineSteps = std::mt19937(args->seed_randomize_steps);
+ } else {
+ randomNumberEngineSteps = std::mt19937(0);
+ }
+
+ if(args->do_group_processes_by_hostname > 0) {
+ cl->getHostAndLocalRank();
}
return SUCCESS;
@@ -823,12 +949,13 @@ void Benchmark::run_Measuringtest() {
const std::size_t NUM_ITERATIONS = randomizeTasks() ? args->num_randomize_tasks : 1;
newStatsVec.resize(NUM_ITERATIONS);
- double timeWork=0.0, timeTotal=0.0; // Total time = work time + all-to-all time (+ overhead)
+ double timeWork, timeTotal=0.0; // Total time = work time + all-to-all time (+ overhead) for each iteration
for(std::size_t iteration = 1; iteration <= NUM_ITERATIONS; iteration++) {
if (0 == cl->rank()) {
std::printf("\n\nIteration:%4ld of %4ld\n", iteration, NUM_ITERATIONS);
std::printf("Preparing\n");
}
+ timeWork=0;
computeRankStepMappings();
stats = &newStatsVec.at( iteration - 1 );
stats->initialize(cl.get(), args);
diff --git a/benchmark/benchmark.h b/benchmark/benchmark.h
index e6990278a1a38abbc110fbfec593939dc9899845..2c68f8ff5879c897fc7e14a04496bb0abfa9d01a 100644
--- a/benchmark/benchmark.h
+++ b/benchmark/benchmark.h
@@ -40,8 +40,13 @@ namespace linktest{
Benchmark(Benchmark&&) = delete;
int main_cmdline();
int benchmark(); // Run the main benchmark
- [[nodiscard]] int rank() const;
- [[nodiscard]] int size() const;
+ [[nodiscard]] int rank() const;
+ [[nodiscard]] int size() const;
+ [[nodiscard]] int numHostRanks() const;
+ [[nodiscard]] int numLocalRanks() const;
+ [[nodiscard]] int hostRank() const;
+ [[nodiscard]] int localRank() const;
+ [[nodiscard]] std::shared_ptr<int[]> hostLocalRanks() const;
void barrier() const;
const struct linktest_args* args;
std::unique_ptr<VirtualCluster> cl;
@@ -55,7 +60,8 @@ namespace linktest{
std::unique_ptr<MemoryBuffer> buf_all2all; //Much larger buffer for all-to-all testing
std::unique_ptr<MemoryBufferMulti> buf_multi; //Pointer to array of pointers to buffers
private:
- std::mt19937 randomNumberEngine;
+ std::mt19937 randomNumberEngineSteps;
+ std::mt19937 randomNumberEngineTasks;
LinktestStats* stats = nullptr;
std::vector<LinktestStats> newStatsVec;
std::vector<int> stepPermutation; // permutation of the step numbers
@@ -68,8 +74,8 @@ namespace linktest{
void prepareBuffers(); // Resize and fill all data Buffers that will be used.
void computeRankStepMappings(); // compute stepPermutation, getRankFromStep and getStepFromRank
std::vector<int> oneFactorAlgorithm(const int size, const int rank);
- void mapWith1FactorAlgorithm(bool randomizeTasks);
- void mapWithBisection();
+ void mapWith1FactorAlgorithm(bool randomizeTasks, bool combineNodes);
+ void mapWithBisection(bool randomizeTasks, bool combineNodes);
int work_alltoall();
/**
* Executes semi-, bi- or unidirectional tests
diff --git a/benchmark/cconfig.h b/benchmark/cconfig.h
index 325ef6a991ff71a8ce6a7eaa52eea7bc3cf2e9d7..f8e6ca60a33b6e47b0c36245e96678d57713cc8e 100644
--- a/benchmark/cconfig.h
+++ b/benchmark/cconfig.h
@@ -29,7 +29,7 @@
#define LINKTEST_VERSION 2
#define LINKTEST_VERSION_SUB 1
-#define LINKTEST_VERSION_PATCHLEVEL 17
+#define LINKTEST_VERSION_PATCHLEVEL 19
#define GIT_HASH_LEN 41
#ifndef GIT_HASH
#define GIT_HASH "UNKNOWN"
diff --git a/benchmark/cmdline.cc b/benchmark/cmdline.cc
index 6adab296a1df33b5c804875938bd12d24af3f6a0..0baea8eaa20dfedb8e79768ab43422f053888e08 100644
--- a/benchmark/cmdline.cc
+++ b/benchmark/cmdline.cc
@@ -240,6 +240,16 @@ auto linktest_all_cmdline_argdefs = {
.help = "Randomize step execution order",
.p = &cmdline_args.do_mix
},
+ Argument{
+ .type = Argument::kInteger,
+ .nargs = 1,
+ .optshort = "",
+ .optlong = "seed-randomize-steps",
+ .defaultval = "9876543210",
+ .required = false,
+ .help = "Seed for step randomization",
+ .p = &cmdline_args.seed_randomize_steps
+ },
Argument{
.type = Argument::kBoolean,
.nargs = 0,
@@ -307,7 +317,7 @@ auto linktest_all_cmdline_argdefs = {
.optlong = "num-randomize-tasks",
.defaultval = "0",
.required = false,
- .help = "Use num different randomly assigned task ids for communication scheme",
+ .help = "Use num different randomly assigned task IDs for communication scheme",
.p = &cmdline_args.num_randomize_tasks
},
Argument{
@@ -319,6 +329,16 @@ auto linktest_all_cmdline_argdefs = {
.required = false,
.help = "Seed for task randomization",
.p = &cmdline_args.seed_randomize_tasks
+ },
+ Argument{
+ .type = Argument::kBoolean,
+ .nargs = 0,
+ .optshort = "",
+ .optlong = "group-processes-by-hostname",
+ .defaultval = "0",
+ .required = false,
+ .help = "Group processes by hostnames and ensure that all processes from one group communicate with all processes from another, i.e. the communication pattern is applied to groups",
+ .p = &cmdline_args.do_group_processes_by_hostname
}
};
@@ -688,26 +708,29 @@ const struct linktest_args *bcast_cmdline_args(VirtualCluster* cl,
* processing elements but not with different endianess.
*/
- cl->bcast(root,&tmp->do_alltoall ,1);
- cl->bcast(root,&tmp->do_bidir ,1);
- cl->bcast(root,&tmp->do_unidir ,1);
- cl->bcast(root,&tmp->do_mix ,1);
- cl->bcast(root,&tmp->do_serial ,1);
- cl->bcast(root,&tmp->do_nosion ,1);
- cl->bcast(root,&tmp->do_sion_par ,1);
- cl->bcast(root,&tmp->num_msg ,1);
- cl->bcast(root,&tmp->num_warmup_msg ,1);
- cl->bcast(root,&tmp->len_msg ,1);
- cl->bcast(root,&tmp->min_walltime ,1);
- cl->bcast(root,&tmp->min_iterations ,1);
- cl->bcast(root,&tmp->max_stest ,1);
- cl->bcast(root,&tmp->do_use_gpus ,1);
- cl->bcast(root,&tmp->do_bisection ,1);
- cl->bcast(root,&tmp->use_multi_buf ,1);
- cl->bcast(root,&tmp->num_multi_buf ,1);
- cl->bcast(root,&tmp->randomize_buffers,1);
- cl->bcast(root,&tmp->buf_mt_seed ,1);
- cl->bcast(root,&tmp->check_buffers ,1);
+ cl->bcast(root,&tmp->do_alltoall ,1);
+ cl->bcast(root,&tmp->do_bidir ,1);
+ cl->bcast(root,&tmp->do_unidir ,1);
+ cl->bcast(root,&tmp->do_mix ,1);
+ cl->bcast(root,&tmp->do_serial ,1);
+ cl->bcast(root,&tmp->do_nosion ,1);
+ cl->bcast(root,&tmp->do_sion_par ,1);
+ cl->bcast(root,&tmp->num_msg ,1);
+ cl->bcast(root,&tmp->num_warmup_msg ,1);
+ cl->bcast(root,&tmp->len_msg ,1);
+ cl->bcast(root,&tmp->min_walltime ,1);
+ cl->bcast(root,&tmp->min_iterations ,1);
+ cl->bcast(root,&tmp->max_stest ,1);
+ cl->bcast(root,&tmp->do_use_gpus ,1);
+ cl->bcast(root,&tmp->do_bisection ,1);
+ cl->bcast(root,&tmp->do_group_processes_by_hostname,1);
+ cl->bcast(root,&tmp->use_multi_buf ,1);
+ cl->bcast(root,&tmp->num_multi_buf ,1);
+ cl->bcast(root,&tmp->randomize_buffers ,1);
+ cl->bcast(root,&tmp->buf_mt_seed ,1);
+ cl->bcast(root,&tmp->check_buffers ,1);
+ cl->bcast(root,&tmp->seed_randomize_steps ,1);
+ cl->bcast(root,&tmp->seed_randomize_tasks ,1);
/* Broadcast Output Filename */
std::uint64_t buf_size;
@@ -827,6 +850,7 @@ void print_cmdline_args(const struct linktest_args* args){
std::printf("Number of warm-up messages: %-" PRIu64 "\n" ,args->num_warmup_msg);
std::printf("Communication pattern: %-s\n" ,getPatternDescription());
std::printf("Memory-buffer type: %-s\n" ,getMemoryBufferType());
+ std::printf("Combine tasks on nodes? %-s\n" ,getYesNo(args->do_group_processes_by_hostname));
std::printf("Additionally test all-to-all? %-s\n" ,getYesNo(args->do_alltoall));
std::printf("Use GPU memory? %-s\n" ,getYesNo(args->do_use_gpus));
std::printf("Randomize test order? %-s\n" ,getYesNo(args->do_mix));
@@ -841,6 +865,10 @@ void print_cmdline_args(const struct linktest_args* args){
if(args->use_multi_buf){
std::printf("Number of multiple message buffers: %-" PRIu64 "\n",args->num_multi_buf);
}
+ std::printf("Randomize test order? %-s\n" ,getYesNo(args->do_mix));
+ if(args->do_mix){
+ std::printf("Seed for step randomization: %-" PRIu64 "\n" ,args->seed_randomize_steps);
+ }
std::printf("Number of communication schemes %-" PRIu64 "\n" ,args->num_randomize_tasks);
if(args->num_randomize_tasks){
std::printf("Seed for random comm. schemes: %-" PRIu64 "\n" ,args->seed_randomize_tasks);
diff --git a/benchmark/cmdline.h b/benchmark/cmdline.h
index 46644b1b5504ef9cdb8c41c0d1d2fa6d3037e392..1facf65035fd1c104b6560f3c952a70a176697cb 100644
--- a/benchmark/cmdline.h
+++ b/benchmark/cmdline.h
@@ -33,6 +33,7 @@ struct linktest_args {
std::int8_t do_use_gpus;
std::int8_t do_bisection;
std::int8_t use_multi_buf;
+ std::int8_t do_group_processes_by_hostname;
std::uint64_t num_msg;
std::uint64_t num_warmup_msg;
std::uint64_t len_msg;
@@ -40,6 +41,7 @@ struct linktest_args {
std::uint64_t num_multi_buf;
std::uint64_t num_randomize_tasks;
std::uint64_t seed_randomize_tasks;
+ std::uint64_t seed_randomize_steps;
std::uint64_t max_stest;
std::uint64_t min_iterations;
double min_walltime;
diff --git a/benchmark/output_sion.cc b/benchmark/output_sion.cc
index ac8c8d0fff94cd1e8bb92fab82b1fec8c8716855..f671e1f6ea3dea2c29102ae0399810e8620dc944 100644
--- a/benchmark/output_sion.cc
+++ b/benchmark/output_sion.cc
@@ -42,37 +42,39 @@ static Serializer getSerializer(VirtualCluster* cl, const struct linktest_args *
if(cl->rank() == 0) {
s.addDataSource(LINKTEST_ID_, LINKTEST_ID_, sizeof(LINKTEST_ID_)-1);
// Data Version / Code Version / Timestamp (if root)
- s.addDataSource("Version Major" ,&VERSION_MAJOR );
- s.addDataSource("Version Minor" ,&VERSION_MINOR );
- s.addDataSource("Version Patch" ,&VERSION_PATCH );
- s.addDataSource("Git-Hash" ,GIT_HASH_ ,GIT_HASH_LEN_ );
+ s.addDataSource("Version Major" ,&VERSION_MAJOR );
+ s.addDataSource("Version Minor" ,&VERSION_MINOR );
+ s.addDataSource("Version Patch" ,&VERSION_PATCH );
+ s.addDataSource("Git-Hash" , GIT_HASH_ ,GIT_HASH_LEN_ );
// Transport Layer
- s.addDataSource("Transport Layer Size" ,&cl->nameSizeRef() );
- s.addDataSource("Transport Layer" , cl->nameRef().c_str() ,cl->nameSizeRef() );
+ s.addDataSource("Transport Layer Size" ,&cl->nameSizeRef() );
+ s.addDataSource("Transport Layer" , cl->nameRef().c_str() ,cl->nameSizeRef() );
// int8_t options
- s.addDataSource("args->do_alltoall" ,&args->do_alltoall );
- s.addDataSource("args->do_bidir" ,&args->do_bidir );
- s.addDataSource("args->do_unidir" ,&args->do_unidir );
- s.addDataSource("args->do_bisection" ,&args->do_bisection );
- s.addDataSource("args->do_mix" ,&args->do_mix );
- s.addDataSource("args->do_serial" ,&args->do_serial );
- s.addDataSource("args->do_nosion" ,&args->do_nosion );
- s.addDataSource("args->do_sion_par" ,&args->do_sion_par );
- s.addDataSource("args->do_use_gpus" ,&args->do_use_gpus );
- s.addDataSource("args->use_multi_buf" ,&args->use_multi_buf );
- s.addDataSource("args->randomize_buffers" ,&args->randomize_buffers );
- s.addDataSource("args->check_buffers" ,&args->check_buffers );
- s.addDataSource("args->alloc_typ" ,&args->alloc_typ ); // enum aka int
+ s.addDataSource("args->do_alltoall" ,&args->do_alltoall );
+ s.addDataSource("args->do_bidir" ,&args->do_bidir );
+ s.addDataSource("args->do_unidir" ,&args->do_unidir );
+ s.addDataSource("args->do_bisection" ,&args->do_bisection );
+ s.addDataSource("args->do_mix" ,&args->do_mix );
+ s.addDataSource("args->do_serial" ,&args->do_serial );
+ s.addDataSource("args->do_nosion" ,&args->do_nosion );
+ s.addDataSource("args->do_sion_par" ,&args->do_sion_par );
+ s.addDataSource("args->do_use_gpus" ,&args->do_use_gpus );
+ s.addDataSource("args->use_multi_buf" ,&args->use_multi_buf );
+ s.addDataSource("args->randomize_buffers" ,&args->randomize_buffers );
+ s.addDataSource("args->check_buffers" ,&args->check_buffers );
+ s.addDataSource("args->do_group_processes_by_hostname",&args->do_group_processes_by_hostname );
+ s.addDataSource("args->alloc_typ" ,&args->alloc_typ ); // enum aka int
// uint64_t data
- s.addDataSource("args->num_msg" ,&args->num_msg );
- s.addDataSource("args->len_msg" ,&args->len_msg );
- s.addDataSource("args->num_warmup_msg" ,&args->num_warmup_msg );
- s.addDataSource("args->max_stest" ,&args->max_stest );
- s.addDataSource("args->num_multi_buf" ,&args->num_multi_buf );
- s.addDataSource("args->buf_mt_seed" ,&args->buf_mt_seed );
- s.addDataSource("args->num_randomize_tasks" ,&args->num_randomize_tasks );
- s.addDataSource("args->seed_randomize_tasks" ,&args->seed_randomize_tasks );
- s.addDataSource(END_HEADER_, END_HEADER_, sizeof(END_HEADER_)-1);
+ s.addDataSource("args->num_msg" ,&args->num_msg );
+ s.addDataSource("args->len_msg" ,&args->len_msg );
+ s.addDataSource("args->num_warmup_msg" ,&args->num_warmup_msg );
+ s.addDataSource("args->max_stest" ,&args->max_stest );
+ s.addDataSource("args->num_multi_buf" ,&args->num_multi_buf );
+ s.addDataSource("args->buf_mt_seed" ,&args->buf_mt_seed );
+ s.addDataSource("args->num_randomize_tasks" ,&args->num_randomize_tasks );
+ s.addDataSource("args->seed_randomize_tasks" ,&args->seed_randomize_tasks );
+ s.addDataSource("args->seed_randomize_steps" ,&args->seed_randomize_steps );
+ s.addDataSource(END_HEADER_ , END_HEADER_ ,sizeof(END_HEADER_)-1);
}
s.addDataSource("hostname_len" ,&cl->hostnameSizeRef() );
@@ -159,7 +161,8 @@ static int linktest_output_sion_funnelled_root(VirtualCluster* cl,
char *buffer,
long long sz,
double begin){
- int i, dummy;
+ int i;
+ long long dummy;
sion_int64 *chunksizes = NULL;
int *globalranks;
char filename[256];
@@ -179,7 +182,12 @@ static int linktest_output_sion_funnelled_root(VirtualCluster* cl,
static_assert(std::is_same<sion_int64, long long>::value);
EXEC_NOFAIL(cl->gather(0, chunksizes, &sz, 1));
-
+#ifdef DEBUG_OUTPUT
+ if(cl->rank()==0){
+ debug("[%3d] Chunksize: %d", 0, chunksizes[0]);
+ for(int i=1;i<cl->size();i++) debug("[%3d] Chunksize: %d", i, chunksizes[i]);
+ }
+#endif
/* The code implicitly assumes that the root process has the biggest chunksize
* which is true in the current code because we write some of the statistics data
* only on the root process. This assumption allows us to avoid allocating a
@@ -221,7 +229,6 @@ static int linktest_output_sion_funnelled_root(VirtualCluster* cl,
for (i = 1; i < cl->size(); ++i) {
EXEC_IFFAIL(cl->send(i, &dummy, 0), fatal("send failed."); return ERROR);
EXEC_IFFAIL(cl->recv(i, buffer, chunksizes[i]), fatal("recv failed."));
-
sion_seek_fp(outsid, i, SION_CURRENT_BLK, SION_CURRENT_POS, &outfp);
sion_fwrite(buffer, 1, chunksizes[i], outsid);
}
@@ -264,7 +271,7 @@ int linktest_output_sion_funnelled(VirtualCluster* cl,
long long dummy;
EXEC_IFFAIL(cl->gather(0, &dummy, &sz, 1), fatal("gather failed."));
EXEC_IFFAIL(cl->barrier(), fatal("barrier failed."));
- EXEC_IFFAIL(cl->recv(0, &dummy, 0), fatal("recv failed."));
+ EXEC_IFFAIL(cl->recv(0, &dummy, 0), fatal("recv failed."));
EXEC_IFFAIL(cl->send(0, buffer, sz), fatal("send failed."));
}
diff --git a/benchmark/vcluster.cc b/benchmark/vcluster.cc
index 7e435218a52b078d083063118ab110a02ec5f649..c4932410642310a94071c1ba905b1eaaf2c69383 100644
--- a/benchmark/vcluster.cc
+++ b/benchmark/vcluster.cc
@@ -35,7 +35,10 @@
#include "output_sion.h"
#include "stats.h"
#include <cstring>
+#include <iostream>
+#include <sstream>
#include <cstdlib>
+#include "utils.h"
VirtualCluster::VirtualCluster(const std::string name)
@@ -85,6 +88,26 @@ const int32_t& VirtualCluster::cpuIDRef(){
return cpuID_;
}
+int VirtualCluster::numHostRanks(){
+ return numHostRanks_;
+}
+
+int VirtualCluster::numLocalRanks(){
+ return numLocalRanks_;
+}
+
+int VirtualCluster::hostRank(){
+ return hostRank_;
+}
+
+int VirtualCluster::localRank(){
+ return localRank_;
+}
+
+std::shared_ptr<int[]> VirtualCluster::hostLocalRanks(){
+ return hostLocalRanks_;
+}
+
int VirtualCluster::barrier(){
return vcluster_helper_barrier(this);
}
@@ -482,7 +505,7 @@ const char* get_vcluster_impl_name(char** argv, const char* name)
}
}
// If this point is reached an unknown/unsupported implementation was encountered.
- error("Unkown/Unsupported command-line implementation encountered.");
+ error("Unknown/Unsupported command-line implementation encountered.");
return(NULL);
}
@@ -528,3 +551,111 @@ int VirtualCluster::write_funnelled(const linktest_args* args, const std::vector
{
return linktest_output_sion_funnelled(this, args, statsVec);
}
+
+void VirtualCluster::getHostAndLocalRank(){
+ std::int64_t buf64x4[4];
+ std::int32_t tmp32=hostnameSize();
+ /***************************************************/
+ /* Determine unique hostnames and associated ranks */
+ /***************************************************/
+ if(rank()==0){
+ /********************/
+ /* Gather Hostnames */
+ /********************/
+ std::vector<std::int32_t> hostnameLengths;
+ std::vector<std::string> hostnames;
+ hostnameLengths.resize(size());
+ gather(0, hostnameLengths.data(), &tmp32, 1); //Gather hostname lengths
+ hostnames.resize(size());
+ hostnames[0]=hostname();
+ for(int i=1;i<size();i++){
+ char buf[hostnameLengths[i]]; //We need an intermediate buffer because we cannot receive directly into the buffer of std::string
+ recv(i, buf, hostnameLengths[i]);
+ hostnames[i]=buf;
+ }
+
+ /***************************************************/
+ /* Determine unique hostnames and associated ranks */
+ /***************************************************/
+ std::vector<std::string> uhostnames;
+ uhostnames.push_back(hostnames[0]);
+ std::vector<std::vector<int>> ranks(1);
+ ranks[0].push_back(0);
+ for(int i=1;i<size();i++){
+ int j=uhostnames.size()-1;
+ for(;j>=0;j--){ //Iterate backwards as hostnames are likely to repeat
+ if(hostnames[i]!=uhostnames[j]) continue;
+ else{
+ ranks[j].push_back(i);
+ break;
+ }
+ }
+ if(j<0){
+ uhostnames.push_back(hostnames[i]);
+ ranks.resize(uhostnames.size());
+ ranks[uhostnames.size()-1].push_back(i);
+ }
+ }
+#ifdef DEBUG_Combine_Hosts
+ std::stringstream buf;
+ for(std::vector<std::string>::size_type i=0;i<uhostnames.size();i++){
+ buf << uhostnames[i].c_str() << ": " << ranks[i];
+ debug("[%3d] Ranks on %s", 0, buf.str().c_str());buf.str(std::string());
+ }
+#endif
+ /******************************************************/
+ /* Check that all hosts have the same number of ranks */
+ /******************************************************/
+ if(ranks.size()%2!=0) fatal("An even number of hosts is required!");
+ for(std::vector<std::vector<int>>::size_type i=0;i<ranks.size();i++) if(ranks[0].size()!=ranks[i].size()) fatal("Hosts have differing amounts of ranks!");
+
+ /**************************************/
+ /* Broadcast information to all ranks */
+ /**************************************/
+ // Buffer for communication
+ buf64x4[0]=(std::int64_t) ranks.size();
+ buf64x4[1]=0;
+ buf64x4[2]=(std::int64_t) ranks[0].size();
+ hostLocalRanks_ = std::make_unique<int[]>(buf64x4[0]*buf64x4[2]);
+ for(std::int64_t i=0;i<buf64x4[0];i++)for(std::int64_t j=0;j<buf64x4[2];j++) hostLocalRanks_[i*buf64x4[2]+j]=ranks[i][j];
+ // Set for rank 0
+ numHostRanks_ =buf64x4[0];
+ hostRank_ =buf64x4[1];
+ numLocalRanks_=buf64x4[2];
+ localRank_ =0;
+#ifdef DEBUG_Combine_Hosts
+ debug("[%3d] Host: %s Num. Host ranks: %d Host rank: %d Num. Local Ranks: %d Local Rank: %d",0,hostnames[0].c_str(),numHostRanks_,hostRank_,numLocalRanks_,localRank_);
+#endif
+ // Loop over other ranks on first host
+ for(std::vector<int>::size_type j=1;j<ranks[0].size();j++){
+ buf64x4[3]=(std::int64_t)j;
+ send(ranks[0][j], buf64x4, 4);
+#ifdef DEBUG_Combine_Hosts
+ debug("[%3d] Host: %s Num. Host ranks: %d Host rank: %d Num. Local Ranks: %d Local Rank: %d",ranks[0][j],uhostnames[0].c_str(),buf64x4[0],buf64x4[1],buf64x4[2],buf64x4[3]);
+#endif
+ }
+ // Loop over remaining ranks
+ for(std::vector<std::vector<int>>::size_type i=1;i<ranks.size();i++){
+ buf64x4[1]=(std::int64_t)i;
+ buf64x4[2]=ranks[i].size();
+ for(std::vector<int>::size_type j=0;j<ranks[i].size();j++){
+ buf64x4[3]=(std::int64_t)j;
+ send(ranks[i][j], buf64x4, 4);
+#ifdef DEBUG_Combine_Hosts
+ debug("[%3d] Host: %s Num. Host ranks: %d Host rank: %d Num. Local Ranks: %d Local Rank: %d",ranks[i][j],uhostnames[i].c_str(),buf64x4[0],buf64x4[1],buf64x4[2],buf64x4[3]);
+#endif
+ }
+ }
+ bcast(0, hostLocalRanks_.get(), buf64x4[0]*buf64x4[2]);
+ }else{
+ gather(0, &tmp32, &tmp32, 1); //Gather hostname lengths
+ send(0, hostname().c_str(), hostnameSize()); //Send hostname
+ recv(0, buf64x4, 4);
+ numHostRanks_ =buf64x4[0];
+ hostRank_ =buf64x4[1];
+ numLocalRanks_=buf64x4[2];
+ localRank_ =buf64x4[3];
+ hostLocalRanks_ = std::make_unique<int[]>(buf64x4[0]*buf64x4[2]);
+ bcast(0, hostLocalRanks_.get(), buf64x4[0]*buf64x4[2]);
+ }
+}
diff --git a/benchmark/vcluster.h b/benchmark/vcluster.h
index 9da08970212e617c301e5d86b9d2991fe796358e..751c4fe19e5098df05400adef89c08aeac6a8ce7 100644
--- a/benchmark/vcluster.h
+++ b/benchmark/vcluster.h
@@ -58,16 +58,21 @@ public:
virtual int size() = 0;
- std::string name() ;
- const std::string& nameRef() ;
- std::int32_t nameSize() ;
- const std::int32_t& nameSizeRef() ;
- std::string hostname() ;
- const std::string& hostnameRef() ;
- std::int32_t hostnameSize() ;
- const std::int32_t& hostnameSizeRef() ;
- std::int32_t cpuID() ;
- const std::int32_t& cpuIDRef() ;
+ std::string name() ;
+ const std::string& nameRef() ;
+ std::int32_t nameSize() ;
+ const std::int32_t& nameSizeRef() ;
+ std::string hostname() ;
+ const std::string& hostnameRef() ;
+ std::int32_t hostnameSize() ;
+ const std::int32_t& hostnameSizeRef();
+ std::int32_t cpuID() ;
+ const std::int32_t& cpuIDRef() ;
+ int numHostRanks() ;
+ int numLocalRanks() ;
+ int hostRank() ;
+ int localRank() ;
+ std::shared_ptr<int[]> hostLocalRanks() ;
/*! \brief send wrapped data in buf to rank dst (communication layer undefined)
*
@@ -190,9 +195,9 @@ public:
MemoryBuffer& buf1, MemoryBuffer& buf2,
int num_msg, double* timing);
- /*! \brief bidirectional kernel
+ /*! \brief unidirectional kernel
*
- * Sends a message without receiving confirmation.
+ * Sends multiple messages with only one confirmation at the end.
* from to
* |────►|
* | buf |
@@ -256,6 +261,9 @@ public:
const struct linktest_args* args,
double* time);
+ // Determines host ranks and host-local ranks
+ void getHostAndLocalRank();
+
/* I/O operations.
*
* - write_parallel: Write in parallel from all processing elements.
@@ -276,11 +284,16 @@ public:
static VirtualCluster* factory(const std::string& name);
private:
- std::string name_;
- const std::int32_t nameSize_;
- std::string hostname_;
- const std::int32_t hostnameSize_;
- const std::int32_t cpuID_;
+ std::string name_;
+ const std::int32_t nameSize_;
+ std::string hostname_;
+ const std::int32_t hostnameSize_;
+ const std::int32_t cpuID_;
+ int numHostRanks_;
+ int hostRank_;
+ int localRank_;
+ int numLocalRanks_;
+ std::shared_ptr<int[]> hostLocalRanks_;
};
/* Since a full implementation of send()/recv() logic on top of some transport layer is
@@ -296,10 +309,15 @@ class VirtualClusterWithHelper : public VirtualCluster {
public:
VirtualClusterWithHelper(const std::string& name);
- int rank() override;
- int size() override;
- int send(int dst, MemoryBuffer& buf) override;
- int recv(int src, MemoryBuffer& buf) override;
+ int rank() override;
+ int size() override;
+ int numHostRanks() ;
+ int numLocalRanks() ;
+ int hostRank() ;
+ int localRank() ;
+ std::shared_ptr<int[]> hostLocalRanks() ;
+ int send(int dst, MemoryBuffer& buf) override;
+ int recv(int src, MemoryBuffer& buf) override;
protected:
void set_helper_pointer(VirtualCluster* helper);
diff --git a/python/linktest/linktest.c b/python/linktest/linktest.c
index 838421166077b0c1f6320a21ff23f75c06487349..c5640e03533223baa1f489609e456a3c86328d7e 100644
--- a/python/linktest/linktest.c
+++ b/python/linktest/linktest.c
@@ -28,6 +28,8 @@
#include <linktest_2_1_15.h>
#include <linktest_2_1_16.h>
#include <linktest_2_1_17.h>
+#include <linktest_2_1_18.h>
+#include <linktest_2_1_19.h>
/* TODO: Properly handle all possible errors.
* Current plan is only to handle errors
@@ -164,6 +166,12 @@ static PyObject* readSION(PyObject *self, PyObject *args){
case 17:
err=readImport_2_1_17(&fh,swap,version,&pyLinkTestInfo,&pyLinkTestData);
break;
+ case 18:
+ err=readImport_2_1_18(&fh,swap,version,&pyLinkTestInfo,&pyLinkTestData);
+ break;
+ case 19:
+ err=readImport_2_1_19(&fh,swap,version,&pyLinkTestInfo,&pyLinkTestData);
+ break;
default:
PyErr_SetString(PyExc_RuntimeError, "Unsupported Version!");
err=1;
diff --git a/python/linktest/linktest_2_1_18.c b/python/linktest/linktest_2_1_18.c
new file mode 100644
index 0000000000000000000000000000000000000000..53a86fbae73478e861b86032b1f4903965af18c5
--- /dev/null
+++ b/python/linktest/linktest_2_1_18.c
@@ -0,0 +1,977 @@
+/****************************************************************************
+** LinkTest **
+*****************************************************************************
+** Copyright (c) 2008-2022 **
+** Forschungszentrum Juelich, Juelich Supercomputing Centre **
+** **
+** See the file COPYRIGHT in the package base directory for details **
+****************************************************************************/
+
+#include <linktest_DoNotIncludeInMain.h>
+#include <linktest.h>
+#include <linktest_helper.h>
+
+/*********************************/
+/* Constant File-Scope Variables */
+/*********************************/
+#define LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18 32
+#define GIT_HASH_SIZE_2_1_18 41
+#define LINKTEST_ID_STR_2_1_18 "LinkTest"
+#define END_HEADER_2_1_18 "END_HEADER"
+#define END_BLOCK_2_1_18 "END_BLOCK"
+
+/**************/
+/* Structures */
+/**************/
+struct LinkTestInfo_2_1_18{ //Description for LinkTest 2.1.18
+ char hash[GIT_HASH_SIZE_2_1_18]; // LinkTest Git Hash
+ char *mode; // Virtual-Cluster Implementation
+ int8_t options[13]; /* LinkTest Options
+ * Unused in Version [1.X.X & 2.0.0]
+ * options[ 0]: If non-zero all-to-all
+ * tests were performed at
+ * the beginning and end of
+ * each iteration.
+ * options[ 1]: If non-zero bidirectional
+ * tests were performed.
+ * options[ 2]: If non-zero unidirectional
+ * tests were performed.
+ * options[ 3]: If non-zero bisection
+ * tests were performed.
+ * options[ 4]: If non-zero steps were
+ * mixed before tests were
+ * performed.
+ * options[ 5]: If non-zero main testing
+ * was performed in serial.
+ * options[ 6]: If non-zero output was
+ * written to a SION file.
+ * Always non-zero since
+ * these options were
+ * written to a SION file.
+ * options[ 7]: If non-zero the SION file
+ * was generated using
+ * parallelized output.
+ * options[ 8]: If non-zero messages were
+ * sent from device memory,
+ * likely GPU memory.
+ * options[ 9]: If non-zero multiple
+ * buffers were used.
+ * options[10]: If non-zero the buffers
+ * were filled with pseudo-
+ * random numbers.
+ * options[11]: If non-zero buffers were
+ * checked after each kernel
+ * run.
+ * options[12]: Memory allocator type:
+ * 0: Invalid - ERROR
+ * 1: Default - ERROR
+ * 2: Memory-aligned Malloc
+ * 3: Pinned-Memory Map
+ * 4: POSIX Memory-aligned Malloc
+ * 5: CUDA Malloc
+ */
+ uint64_t iInfo[ 9]; /* Auxiliary Integer Information
+ * iInfo[ 0]: The number of messages
+ * timings were averaged
+ * over.
+ * iInfo[ 1]: The message size sent
+ * between tasks.
+ * iInfo[ 2]: The number of warm-up
+ * messages sent before
+ * beginning timing.
+ * iInfo[ 3]: Number of serial retests
+ * iInfo[ 4]: Number of memory buffers
+ * if multiple memory
+ * buffers were used.
+ * iInfo[ 5]: MT_19937 seed for buffer
+ * randomization..
+ * iInfo[ 6]: If non-zero indicates the
+ * number of different task
+ * permutations used for
+ * averaging communication
+ * times.
+ * iInfo[ 7]: MT_19937 seed for task
+ * randomization.
+ * iInfo[ 8]: MT_19937 seed for step
+ * randomization.
+ */
+};
+
+enum OPTIONS{
+ A2A________FLAG= 0,
+ BIDIR______FLAG= 1,
+ UNIDIR_____FLAG= 2,
+ BISECT_____FLAG= 3,
+ RAND_STEP__FLAG= 4,
+ SERIAL_____FLAG= 5,
+ SION_OUT___FLAG= 6,
+ SION_PAR___FLAG= 7,
+ DEVICE_MEM_FLAG= 8,
+ MULTI_BUF__FLAG= 9,
+ RAND_BUF___FLAG=10,
+ CHECK_BUF__FLAG=11,
+ MEM_ALLOC__TYPE=12
+};
+
+enum INFO{
+ NUM_MESSAGES=0,
+ MESSAGE_SIZE=1,
+ NUM_WARM__UP=2,
+ NUM___SERIAL=3,
+ NUM__MEM_BUF=4,
+ MEM_BUF_SEED=5,
+ NUM_PERMUTAT=6,
+ PERMUTA_SEED=7,
+ STEP____SEED=8
+};
+
+/*****************************/
+/* Read LinkTest Information */
+/*****************************/
+int readInfo_2_1_18(struct SionFile* const fh, const int swap, const Version version, struct LinkTestInfo_2_1_18* const linkTestInfo){
+ const size_t end_header_len=sizeof(END_HEADER_2_1_18)/sizeof(char);
+ char end_header[end_header_len];
+ size_t n;
+ int32_t strSize;
+
+ /* Git Hash */
+ n=fread(linkTestInfo->hash, sizeof(char), GIT_HASH_SIZE_2_1_18, fh->fp);
+ if (UNLIKELY(n != GIT_HASH_SIZE_2_1_18)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read git hash.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->hash, linkTestInfo->hash, sizeof(char), GIT_HASH_SIZE_2_1_18, swap);
+ if (UNLIKELY(linkTestInfo->hash[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: NULL git hashes are not supported.");
+ return -1;
+ }
+ linkTestInfo->hash[GIT_HASH_SIZE_2_1_18-1]='\0'; //Make sure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("git hash =%s\n",linkTestInfo->hash);fflush(stdout);
+ #endif
+
+ /* Read Virtual Cluster Implementation */
+ n=fread(&strSize, sizeof(int32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read string sizes for git hash, datetime and mode.");
+ return -1;
+ }
+ sion_swap(&strSize, &strSize, sizeof(int32_t), 1, swap);
+ linkTestInfo->mode=malloc(strSize*sizeof(char)); //TODO: Dangerous if sizeof(char) mismatches between writing and reading system.
+ n=fread(linkTestInfo->mode, sizeof(char), strSize, fh->fp);
+ if (UNLIKELY((int32_t)n != strSize)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read virtual-cluster implementation.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->mode, linkTestInfo->mode, sizeof(char), strSize, swap);
+ if (UNLIKELY(linkTestInfo->mode[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Virtual-cluster implementation NULL is not supported.");
+ return -1;
+ }
+ linkTestInfo->mode[strSize-1]='\0'; //Add null terminator to string
+ #ifdef DEBUG_OUTPUT
+ printf("mode =%s\n",linkTestInfo->mode);fflush(stdout);
+ #endif
+
+ /* Read 32-bit Integer Options */
+ n=fread(linkTestInfo->options, sizeof(int8_t), 13, fh->fp);
+ if (UNLIKELY(n != 13)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read 32-bit integer LinkTest options.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->options, linkTestInfo->options, sizeof(int8_t), 13, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("do_alltoall =%" PRId8 "\n",linkTestInfo->options[A2A________FLAG]);
+ printf("do_bidir =%" PRId8 "\n",linkTestInfo->options[BIDIR______FLAG]);
+ printf("do_unidir =%" PRId8 "\n",linkTestInfo->options[UNIDIR_____FLAG]);
+ printf("do_bisection =%" PRId8 "\n",linkTestInfo->options[BISECT_____FLAG]);
+ printf("do_mix =%" PRId8 "\n",linkTestInfo->options[RAND_STEP__FLAG]);
+ printf("do_serial =%" PRId8 "\n",linkTestInfo->options[SERIAL_____FLAG]);
+ printf("do_nosion =%" PRId8 "\n",linkTestInfo->options[SION_OUT___FLAG]);
+ printf("do_sion_par =%" PRId8 "\n",linkTestInfo->options[SION_PAR___FLAG]);
+ printf("do_use_gpus =%" PRId8 "\n",linkTestInfo->options[DEVICE_MEM_FLAG]);
+ printf("use_multi_buf =%" PRId8 "\n",linkTestInfo->options[MULTI_BUF__FLAG]);
+ printf("randomize_buffers =%" PRId8 "\n",linkTestInfo->options[RAND_BUF___FLAG]);
+ printf("check_buffers =%" PRId8 "\n",linkTestInfo->options[CHECK_BUF__FLAG]);
+ printf("mem alloc type =%" PRId8 "\n",linkTestInfo->options[MEM_ALLOC__TYPE]);
+ fflush(stdout);
+ #endif
+
+ /* Read 32-bit Integer Information */
+ n=fread(linkTestInfo->iInfo, sizeof(uint64_t), 9, fh->fp);
+ if (UNLIKELY(n != 9)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read 32-bit integer LinkTest information.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->iInfo, linkTestInfo->iInfo, sizeof(uint64_t), 9, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("num_msg =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_MESSAGES]);
+ printf("len_msg =%" PRIu64 "\n",linkTestInfo->iInfo[MESSAGE_SIZE]);
+ printf("num_warmup_msg =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_WARM__UP]);
+ printf("max_stest =%" PRIu64 "\n",linkTestInfo->iInfo[NUM___SERIAL]);
+ printf("num_multi_buf =%" PRIu64 "\n",linkTestInfo->iInfo[NUM__MEM_BUF]);
+ printf("buf_mt_seed =%" PRIu64 "\n",linkTestInfo->iInfo[MEM_BUF_SEED]);
+ printf("num_randomize_tasks =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_PERMUTAT]);
+ printf("seed_randomize_tasks =%" PRIu64 "\n",linkTestInfo->iInfo[PERMUTA_SEED]);
+ printf("seed_randomize_steps =%" PRIu64 "\n",linkTestInfo->iInfo[STEP____SEED]);
+ fflush(stdout);
+ #endif
+
+ /* Check If 0/1 Multiple Buffer Were Specified */
+ if(linkTestInfo->options[MULTI_BUF__FLAG] != 0){
+ if(linkTestInfo->iInfo[NUM__MEM_BUF] == 0){
+ PyErr_WarnEx(NULL,"Detected that multiple buffers were used, however, an unknown (0) number of buffers were used.\n ReadSION: The number of buffers used was likely equal to the number of messages. Interpret results with caution.",(Py_ssize_t )1);
+ }else if(linkTestInfo->iInfo[NUM__MEM_BUF] == 1){
+ PyErr_WarnEx(NULL,"Detected that multiple buffers were used, however, only one buffer was used.\n ReadSION: In this scenario multiple buffers should not have been used. Interpret results with caution.",(Py_ssize_t )1);
+ }
+ }
+
+ /* Check If Mersenne Twister Seed Is Zero */
+ if(linkTestInfo->options[RAND_BUF___FLAG] != 0 && linkTestInfo->iInfo[MEM_BUF_SEED] == 0){
+ PyErr_WarnEx(NULL,"Buffer content was to be randomized, however, the stored zero Mersenne Twister seed would have filled the buffers with zero. Interpret results with caution.",(Py_ssize_t )1);
+ }
+
+ /* Check END_HEADER String */
+ n=fread(end_header,sizeof(char),end_header_len-1,fh->fp);
+ if (UNLIKELY(n != end_header_len-1)){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read END_HEADER string.");
+ return -1;
+ }
+ end_header[end_header_len-1]='\0';
+ if(UNLIKELY(strcmp(end_header,END_HEADER_2_1_18))){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Alignment Error Detected. \"END_HEADER\" not found at the correct location.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/*******************************************/
+/* Import LinkTest Information Into Python */
+/*******************************************/
+int importInfo_2_1_18(const Version version, struct LinkTestInfo_2_1_18* const linkTestInfo, PyObject* const* const pyLinkTestInfo){
+ int err;
+
+ // Allocator Type
+ enum AllocatorType{
+ AllocatorInvalid =0, //Invalid Allocator - ERROR
+ AllocatorDefault =1, //Default Allocator - ERROR
+ AllocatorAlignedMalloc =2, //Memory-aligned malloc Allocator
+ AllocatorPinnedMmap =3, //Pinned--memory-map Allocator
+ AllocatorPOSIXAlignedMalloc=4, //POSIX Memory-aligned malloc Allocator
+ AllocatorCUDA =5 //CUDA malloc Allocator
+ };
+
+ /* Set LinkTest Version */
+ /* TODO: There is a potential bug here. %d is used for a digit type,
+ * however this may be insufficiently long for 32-bit version numbers.
+ * This can cause errors. Python, however, does not support the standard
+ * C way of using PRId32.
+ */
+ err=PyDict_SetItem(*pyLinkTestInfo,
+ PyUnicode_FromString("Version"),
+ PyUnicode_FromFormat("%d.%d.%d",version[0],version[1],version[2])
+ );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add Version to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set LinkTest Git Hash */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Git hash" ),PyUnicode_FromString(linkTestInfo->hash ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add hash to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set Virtual-Cluster Implementation */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Mode" ),PyUnicode_FromString(linkTestInfo->mode));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add mode to LinkTestInfo dictionary.");
+ return -1;
+ }
+ free(linkTestInfo->mode);
+
+ /* Set Options */
+ // NOTE: Casting to a long int from an int32_t to a long int should
+ // be safe since a long int on standard compliant compilers have a
+ // minimum size of 32 bytes.
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Test MPI all-to-all?" ),PyLong_FromLong((long int)linkTestInfo->options[A2A________FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doAlltoall to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do bidirectional tests?" ),PyLong_FromLong((long int)linkTestInfo->options[BIDIR______FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doBidir to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do unidirectional tests?" ),PyLong_FromLong((long int)linkTestInfo->options[UNIDIR_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doUnidir to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do a bisection test?" ),PyLong_FromLong((long int)linkTestInfo->options[BISECT_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doBisec to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Randomize step order?" ),PyLong_FromLong((long int)linkTestInfo->options[RAND_STEP__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doMix to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Test serially?" ),PyLong_FromLong((long int)linkTestInfo->options[SERIAL_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doSerial to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do not store results in a SION file?"),PyLong_FromLong((long int)linkTestInfo->options[SION_OUT___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doWrite to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Write SION output in parallel?" ),PyLong_FromLong((long int)linkTestInfo->options[SION_PAR___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add doParallelIO to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Store messages in GPU memory?" ),PyLong_FromLong((long int)linkTestInfo->options[DEVICE_MEM_FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add useGPUs to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Use multiple memory buffers?" ),PyLong_FromLong((long int)linkTestInfo->options[MULTI_BUF__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add use_multi_buf to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Randomize memory-buffer content?" ),PyLong_FromLong((long int)linkTestInfo->options[RAND_BUF___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add randomize_buffers to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Check memory-buffer content?" ),PyLong_FromLong((long int)linkTestInfo->options[CHECK_BUF__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add check_buffers to LinkTestInfo dictionary.");
+ return -1;
+ }
+ switch(linkTestInfo->options[MEM_ALLOC__TYPE]){
+ case AllocatorInvalid:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Invalid - ERROR" ));
+ break;
+ case AllocatorDefault:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Default - ERROR" ));
+ break;
+ case AllocatorAlignedMalloc:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Memory-aligned malloc" ));
+ break;
+ case AllocatorPinnedMmap:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Pinned memory-map" ));
+ break;
+ case AllocatorPOSIXAlignedMalloc:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("POSIX memory-aligned malloc"));
+ break;
+ case AllocatorCUDA:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("CUDA malloc" ));
+ break;
+ default:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("UNKNOWN" ));
+ break;
+ }
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add memory-buffer allocator to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set Integer Information */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of messages" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_MESSAGES]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add numberOfMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Message size" ),PyLong_FromLong((long int)linkTestInfo->iInfo[MESSAGE_SIZE]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add lengthOfMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of warm-up messages" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_WARM__UP]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add numberOfWarmUpMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of serial retests" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM___SERIAL]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add numberOfSerialRetests to LinkTestInfo dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->options[MULTI_BUF__FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of multiple buffers" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM__MEM_BUF]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add num_multi_buf to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[RAND_BUF___FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Buffer Mersenne Twister seed"),PyLong_FromLong((long int)linkTestInfo->iInfo[MEM_BUF_SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add buf_mt_seed to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of tests with randomized rank order"),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_PERMUTAT]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add num_randomize_tasks to LinkTestInfo dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->iInfo[NUM_PERMUTAT]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Seed for rank-order randomization" ),PyLong_FromLong((long int)linkTestInfo->iInfo[PERMUTA_SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add seed_randomize_tasks to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[RAND_STEP__FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Seed for step-order randomization" ),PyLong_FromLong((long int)linkTestInfo->iInfo[STEP____SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add seed_randomize_steps to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*******************************/
+/* Read & Import LinkTest Data */
+/*******************************/
+int readImportData_2_1_18(struct SionFile* const fh, const int swap, struct LinkTestInfo_2_1_18* const linkTestInfo, PyObject* const* const pyLinkTestInfo, PyObject** pyLinkTestData){
+ const size_t end_block_len=sizeof(END_BLOCK_2_1_18)/sizeof(char);
+ char end_block[end_block_len];
+ int err;
+ int32_t rank, hostname_len;
+ size_t n;
+ size_t i;
+ char* hostname;
+ npy_intp len[2];
+ void *ptr;
+
+ const size_t num_iterations =(linkTestInfo->iInfo[NUM_PERMUTAT]>1)?(size_t)linkTestInfo->iInfo[NUM_PERMUTAT]:1;
+ const size_t numMinMaxAvgTimings=(linkTestInfo->options[A2A________FLAG]!=0)?6:3;
+ double minMaxAvgTimings[num_iterations*numMinMaxAvgTimings];
+
+ //TODO: Handle Errors
+
+ /* Set Python Array Sizes */
+ len[0] = fh->size;
+ len[1] = fh->size;
+
+ /* Initialize Hostnames & Core IDs */
+ PyObject* const pyHosts = PyList_New((Py_ssize_t)fh->size);
+ PyObject* const pyCore = NewNumericPyArray(NPY_INT, 1, len);
+
+ /* Initialize Start & End Time Strings */
+ PyObject** const pyStartTime=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyStartTime[i]=PyDict_New();
+ PyObject** const pyEndTime =(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyEndTime[i] =PyDict_New();
+
+ /* Initialize Timings, Access Pattern & All-To-All Timings */
+ PyObject** const pyTimings =(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyTimings[i] =NewNumericPyArray(NPY_DOUBLE,2,len); //A python double-precision floating-point array of the LinkTest point-to-point timings
+ PyObject** const pyAccesspattern=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyAccesspattern[i]=NewNumericPyArray(NPY_UINT64,2,len); //A python integer array indicating the access order
+ PyObject** const pyA2ATimings =(linkTestInfo->options[A2A________FLAG]!=0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL; //Silences maybe-used-uninitialized warning
+ if(linkTestInfo->options[A2A________FLAG]!=0) for(i=0;i<num_iterations;i++) pyA2ATimings[i] =NewNumericPyArray(NPY_DOUBLE, 1, len); //A python double-precision floating-point array of the linkTest all-to-all imings
+
+ /* Initialize Serial Retest Timings */
+ len[0] = linkTestInfo->iInfo[NUM___SERIAL];
+ PyObject** const pySerialTimingsNew=(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTimingsNew[i] = NewNumericPyArray(NPY_DOUBLE, 1, len);
+ PyObject** const pySerialTimingsOld=(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTimingsOld[i] = NewNumericPyArray(NPY_DOUBLE, 1, len);
+ PyObject** const pySerialFrom =(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialFrom[i] = NewNumericPyArray(NPY_UINT64, 1, len);
+ PyObject** const pySerialTo =(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTo[i] = NewNumericPyArray(NPY_UINT64, 1, len);
+
+ for (rank = 0; rank < fh->size; ++rank) { //Loop over the ranks, each one was saved to its own chunk
+ /* Read Host Name (Multiple Tasks May Have The Same Host Name */
+ n=fread(&hostname_len, sizeof(uint32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import hostname length.");
+ return -1;
+ }
+ sion_swap(&hostname_len, &hostname_len, sizeof(uint32_t), 1, swap);
+ hostname=(char*)malloc(hostname_len*sizeof(char));
+ n=fread(hostname, sizeof(char), hostname_len, fh->fp);
+ if (UNLIKELY((int32_t)n != hostname_len)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import Hostname.");
+ return -1;
+ }
+ sion_swap(&hostname, &hostname, sizeof(char), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("Hostname[%d] =%s\n",rank,hostname);fflush(stdout);
+ #endif
+ PyList_SET_ITEM(pyHosts,(Py_ssize_t)rank,PyBytes_FromStringAndSize(hostname,(Py_ssize_t)hostname_len));
+ /* Read Pinned-Core ID */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pyCore, rank);
+ n=fread(ptr, sizeof(int32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import Pinned Core ID.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(int32_t), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("cpuID[%d] =%d\n",rank,*(int32_t*)ptr);fflush(stdout);
+ #endif
+
+ for(i=0;i<num_iterations;i++){
+ if(rank==0){
+ /* Read Start Time */
+ char datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18+1]; //TODO: Dangerous if sizeof(char) mismatches between writing and reading system.
+ n=fread(datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18, fh->fp);
+ if (UNLIKELY(n != LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read startDatetime.");
+ return -1;
+ }
+ sion_swap(datetime_string, datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18, swap);
+ if (UNLIKELY(datetime_string[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: startDatetime NULL is not supported.");
+ return -1;
+ }
+ datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18]='\0'; //Ensure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("startDatetime[0,%zu] =%s",i,datetime_string);
+ fflush(stdout);
+ #endif
+ struct Datetime startDatetime;
+ n=sscanf(datetime_string,
+ "%2"SCNu8"/%2"SCNu8"/%"SCNu64" %2"SCNu8":%2"SCNu8":%2"SCNu8" UTC+0",
+ &startDatetime.day,
+ &startDatetime.month,
+ &startDatetime.year,
+ &startDatetime.hour,
+ &startDatetime.minute,
+ &startDatetime.second
+ );
+ if (UNLIKELY(n!=6)){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: startDatetime is not a correctly formatted string.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("startDatetime.year[0,%zu] =%" PRIu64 "\n",i,startDatetime.year );
+ printf("startDatetime.month[0,%zu] =%" PRIu8 "\n",i,startDatetime.month );
+ printf("startDatetime.day[0,%zu] =%" PRIu8 "\n",i,startDatetime.day );
+ printf("startDatetime.hour[0,%zu] =%" PRIu8 "\n",i,startDatetime.hour );
+ printf("startDatetime.minute[0,%zu] =%" PRIu8 "\n",i,startDatetime.minute);
+ printf("startDatetime.second[0,%zu] =%" PRIu8 "\n",i,startDatetime.second);
+ printf("startDatetime.timezone[0,%zu]=UTC+0\n" ,i );
+ fflush(stdout);
+ #endif
+ /* Import Start Time */
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Year" ),PyLong_FromLong((long int)startDatetime.year ));
+ if(err){//TODO: Add iteration index to error message
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_year to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Month" ),PyLong_FromLong((long int)startDatetime.month ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_month to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Day" ),PyLong_FromLong((long int)startDatetime.day ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_day to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Hour" ),PyLong_FromLong((long int)startDatetime.hour ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_hour to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Minute" ),PyLong_FromLong((long int)startDatetime.minute));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_minute to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Second" ),PyLong_FromLong((long int)startDatetime.second));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_second to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Timezone"),PyUnicode_FromFormat("UTC+0"));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_timezone to pyStartTime dictionary.");
+ return -1;
+ }
+ /* Read Minimum, Maximum & Average LinkTest Timings */
+ n=fread(&minMaxAvgTimings[i*numMinMaxAvgTimings], sizeof(double), numMinMaxAvgTimings, fh->fp);
+ if (UNLIKELY(n != numMinMaxAvgTimings)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read minimum, maximum & average LinkTest timings.");
+ return -1;
+ }
+ sion_swap(minMaxAvgTimings, minMaxAvgTimings, sizeof(double), numMinMaxAvgTimings, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("min_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+0]);
+ printf("max_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+1]);
+ printf("avg_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+2]);
+ if(numMinMaxAvgTimings==6){
+ printf("min_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+3]);
+ printf("max_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+4]);
+ printf("avg_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+5]);
+ }
+ fflush(stdout);
+ #endif
+ /* Read Serial Retest Data */
+ if (linkTestInfo->iInfo[NUM___SERIAL]>0){
+ /* Read New Serially Retested Timings */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTimingsNew[i], 0);
+ n=fread(ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import retested serial timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsNew[0,%zu] =%f",i,((double*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%f",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read Old Serially Retested Timings */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTimingsOld[i], 0);
+ n=fread(ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import retest-needed timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsOld[0,%zu] =%f",i,((double*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%f",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read Origin Partner For Serial Retests */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialFrom[i] , 0);
+ n=fread(ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import serial retest origin.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsFrom[0,%zu] =%d",i,((uint64_t*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%"PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ sion_swap(ptr, ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ /* Read Destination Partner For Serial Retests */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTo[i] , 0);
+ n=fread(ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import serial retest destination.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsTo[0,%zu] =%d",i,((uint64_t*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%"PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ }
+ /* Read End Time */
+ n=fread(datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18, fh->fp);
+ if (UNLIKELY(n != LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: Failed to read endDatetime.");
+ return -1;
+ }
+ sion_swap(datetime_string, datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18, swap);
+ if (UNLIKELY(datetime_string[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: endDatetime NULL is not supported.");
+ return -1;
+ }
+ datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_18]='\0'; //Ensure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("endDatetime[0,%zu] =%s",i,datetime_string);
+ fflush(stdout);
+ #endif
+ struct Datetime endDatetime;
+ n=sscanf(datetime_string,
+ "%2"SCNu8"/%2"SCNu8"/%"SCNu64" %2"SCNu8":%2"SCNu8":%2"SCNu8" UTC+0",
+ &endDatetime.day,
+ &endDatetime.month,
+ &endDatetime.year,
+ &endDatetime.hour,
+ &endDatetime.minute,
+ &endDatetime.second
+ );
+ if (UNLIKELY(n!=6)){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_18: endDatetime is not a correctly formatted string.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("endDatetime.year[0,%zu] =%" PRIu64 "\n",i,endDatetime.year );
+ printf("endDatetime.month[0,%zu] =%" PRIu8 "\n",i,endDatetime.month );
+ printf("endDatetime.day[0,%zu] =%" PRIu8 "\n",i,endDatetime.day );
+ printf("endDatetime.hour[0,%zu] =%" PRIu8 "\n",i,endDatetime.hour );
+ printf("endDatetime.minute[0,%zu] =%" PRIu8 "\n",i,endDatetime.minute);
+ printf("endDatetime.second[0,%zu] =%" PRIu8 "\n",i,endDatetime.second);
+ printf("endDatetime.timezone[0,%zu] =UTC+0\n" ,i );
+ fflush(stdout);
+ #endif
+ /* Import End Time */
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Year" ),PyLong_FromLong((long int)endDatetime.year ));
+ if(err){//TODO: Add iteration index to error message
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_year to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Month" ),PyLong_FromLong((long int)endDatetime.month ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_month to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Day" ),PyLong_FromLong((long int)endDatetime.day ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_day to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Hour" ),PyLong_FromLong((long int)endDatetime.hour ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_hour to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Minute" ),PyLong_FromLong((long int)endDatetime.minute));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_minute to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Second" ),PyLong_FromLong((long int)endDatetime.second));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_second to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Timezone"),PyUnicode_FromFormat("UTC+0"));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add DateTime_timezone to pyEndTime dictionary.");
+ return -1;
+ }
+ }
+ /* Read LinkTest Timings */
+ ptr=PyArray_GETPTR2((PyArrayObject *)pyTimings[i] , rank, 0);
+ n=fread(ptr, sizeof(double), fh->size, fh->fp);
+ if (UNLIKELY((int32_t)n != fh->size)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import LinkTest timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double), fh->size, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("timings[%d,%zu] =%e",rank,i,((double*)ptr)[0]);
+ for(n=1;(int32_t)n<fh->size;n++) printf(",%e",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read LinkTest Access Pattern */
+ ptr=PyArray_GETPTR2((PyArrayObject *)pyAccesspattern[i], rank, 0);
+ n=fread(ptr, sizeof(int64_t), fh->size, fh->fp);
+ if (UNLIKELY((int64_t)n != fh->size)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import LinkTest access pattern.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(uint64_t), fh->size, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("accesspattern[%d,%zu] =%" PRIu64,rank,i,((uint64_t*)ptr)[0]);
+ for(n=1;(int32_t)n<fh->size;n++) printf(",%" PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read LinkTest All-To-All Timings */
+ if (linkTestInfo->options[A2A________FLAG]){
+ ptr=PyArray_GETPTR1((PyArrayObject *)pyA2ATimings[i], rank);
+ n=fread(ptr, sizeof(double), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read/import all-to-all timing.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("all-to-all time[%d,%zu] =%f\n",rank,i,((double*)ptr)[0]);fflush(stdout);
+ #endif
+ }
+ }
+
+ /* Check Alignment by Testing for END_BLOCK */
+ n=fread(end_block,sizeof(char),end_block_len-1,fh->fp);
+ if (UNLIKELY(n != end_block_len-1)){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Failed to read END_BLOCK string.");
+ return -1;
+ }
+ end_block[end_block_len-1]='\0';
+ if(UNLIKELY(strcmp(end_block,END_BLOCK_2_1_18))){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_18: Alignment Error Detected. \"END_BLOCK\" not found at the correct location.");
+ return -1;
+ }
+ }
+
+ /* Close SION File */
+ sion_close(fh->sid);
+
+ /* Add Hostname To LinkTestInfo Dictionary */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Hosts"),pyHosts);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add Hosts to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Add Core ID To LinkTestInfo Dictionary */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Core ID"),pyCore);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to add Core ID to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Create Data Dictionaries */
+ PyObject** const pyLinkTestDataDicts=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyLinkTestDataDicts[i] = PyDict_New();
+
+ /* Fill Dictionaries */
+ for(i=0;i<num_iterations;i++){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Start Time" ), pyStartTime[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add start time to LinkTestData dictionary.");
+ return -1;
+ }
+ /* Set Double-Precision Floating-Point Information */
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Minimum time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+0]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add minTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Maximum time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+1]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add maxTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Average time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+2]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add avgTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ if(numMinMaxAvgTimings==6){ //Only execute if all-to-all testing was performed
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Minimum all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+3]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aMinTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Maximum all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+4]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aMaxTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Average all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+5]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aAvgTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Timings" ), pyTimings[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add timings to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Access Pattern"), pyAccesspattern[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add accessPatterns to LinkTestData dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->iInfo[NUM___SERIAL]>0){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("New serially retested timings" ), pySerialTimingsNew[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add serialTimingsNew to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Old serially retested timings" ), pySerialTimingsOld[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add serialTimingsOld to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Serial-retest origin hosts" ), pySerialFrom[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add serialFrom to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Serial-retest destination hosts"), pySerialTo[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add serialTo to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[A2A________FLAG]){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("All-to-all timings"), pyA2ATimings[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add a2aTimings to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("End Time" ), pyEndTime[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_18: Failed to add end time to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+
+ /* Create Python List If Necessary */
+ if(num_iterations>1){
+ *pyLinkTestData=PyList_New(num_iterations);
+ for(i=0;i<num_iterations;i++)PyList_SET_ITEM(*pyLinkTestData,i,pyLinkTestDataDicts[i]);
+ }else{
+ *pyLinkTestData=pyLinkTestDataDicts[0];
+ }
+
+ /* Free Memory */
+ free(pyTimings );
+ free(pyAccesspattern);
+ if(pyA2ATimings!=NULL) free(pyA2ATimings);
+ if(linkTestInfo->iInfo[NUM___SERIAL]>0){
+ free(pySerialTimingsNew);
+ free(pySerialTimingsOld);
+ free(pySerialFrom );
+ free(pySerialTo );
+ }
+
+ return 0;
+}
+
+/**********************/
+/* Read & Import Data */
+/**********************/
+int readImport_2_1_18(struct SionFile* const fh, const int swap, const Version version, PyObject** const pyLinkTestInfo, PyObject** const pyLinkTestData){
+ struct LinkTestInfo_2_1_18 linkTestInfo;
+ int err;
+ /* Read LinkTest Information */
+ err = readInfo_2_1_18(fh, swap, version, &linkTestInfo);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to read LinkTest information.");
+ return err;
+ }
+ /* Import LinkTest Information Into Python */
+ *pyLinkTestInfo = PyDict_New(); //Create LinkTest information dictionary
+ err = importInfo_2_1_18(version, &linkTestInfo, pyLinkTestInfo);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to populate the LinkTestInfo dictionary.");
+ //TODO: Safely Deallocate Python Dictionary
+ return err;
+ }
+ /* Read & Import LinkTest Data Into Python */
+ err = readImportData_2_1_18(fh, swap, &linkTestInfo, pyLinkTestInfo, pyLinkTestData);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_18: Failed to read LinkTest data or to populate the LinkTestData dictionary.");
+ //TODO: Safely Deallocate Python Dictionary
+ return err;
+ }
+
+ return(0);
+}
diff --git a/python/linktest/linktest_2_1_18.h b/python/linktest/linktest_2_1_18.h
new file mode 100644
index 0000000000000000000000000000000000000000..fa46d78d793f74f2b2b2611d69032075662f73c1
--- /dev/null
+++ b/python/linktest/linktest_2_1_18.h
@@ -0,0 +1,13 @@
+/****************************************************************************
+** LinkTest **
+*****************************************************************************
+** Copyright (c) 2008-2022 **
+** Forschungszentrum Juelich, Juelich Supercomputing Centre **
+** **
+** See the file COPYRIGHT in the package base directory for details **
+****************************************************************************/
+
+/**********************/
+/* Read & Import Data */
+/**********************/
+int readImport_2_1_18(struct SionFile* const fh, const int swap, const Version version, PyObject** const pyLinkTestInfo, PyObject** const pyLinkTestData);
diff --git a/python/linktest/linktest_2_1_19.c b/python/linktest/linktest_2_1_19.c
new file mode 100644
index 0000000000000000000000000000000000000000..e4ba8e2f40d9349a446e67f8c59a574af3e7264d
--- /dev/null
+++ b/python/linktest/linktest_2_1_19.c
@@ -0,0 +1,988 @@
+/****************************************************************************
+** LinkTest **
+*****************************************************************************
+** Copyright (c) 2008-2022 **
+** Forschungszentrum Juelich, Juelich Supercomputing Centre **
+** **
+** See the file COPYRIGHT in the package base directory for details **
+****************************************************************************/
+
+#include <linktest_DoNotIncludeInMain.h>
+#include <linktest.h>
+#include <linktest_helper.h>
+
+/*********************************/
+/* Constant File-Scope Variables */
+/*********************************/
+#define LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19 32
+#define GIT_HASH_SIZE_2_1_19 41
+#define LINKTEST_ID_STR_2_1_19 "LinkTest"
+#define END_HEADER_2_1_19 "END_HEADER"
+#define END_BLOCK_2_1_19 "END_BLOCK"
+
+/**************/
+/* Structures */
+/**************/
+struct LinkTestInfo_2_1_19{ //Description for LinkTest 2.1.18
+ char hash[GIT_HASH_SIZE_2_1_19]; // LinkTest Git Hash
+ char *mode; // Virtual-Cluster Implementation
+ int8_t options[14]; /* LinkTest Options
+ * Unused in Version [1.X.X & 2.0.0]
+ * options[ 0]: If non-zero all-to-all
+ * tests were performed at
+ * the beginning and end of
+ * each iteration.
+ * options[ 1]: If non-zero bidirectional
+ * tests were performed.
+ * options[ 2]: If non-zero unidirectional
+ * tests were performed.
+ * options[ 3]: If non-zero bisection
+ * tests were performed.
+ * options[ 4]: If non-zero steps were
+ * mixed before tests were
+ * performed.
+ * options[ 5]: If non-zero main testing
+ * was performed in serial.
+ * options[ 6]: If non-zero output was
+ * written to a SION file.
+ * Always non-zero since
+ * these options were
+ * written to a SION file.
+ * options[ 7]: If non-zero the SION file
+ * was generated using
+ * parallelized output.
+ * options[ 8]: If non-zero messages were
+ * sent from device memory,
+ * likely GPU memory.
+ * options[ 9]: If non-zero multiple
+ * buffers were used.
+ * options[10]: If non-zero the buffers
+ * were filled with pseudo-
+ * random numbers.
+ * options[11]: If non-zero buffers were
+ * checked after each kernel
+ * run.
+ * options[12]: If non-zero processes were
+ * grouped according to hostname
+ * and testing only occurred
+ * between groups.
+ * options[13]: Memory allocator type:
+ * 0: Invalid - ERROR
+ * 1: Default - ERROR
+ * 2: Memory-aligned Malloc
+ * 3: Pinned-Memory Map
+ * 4: POSIX Memory-aligned Malloc
+ * 5: CUDA Malloc
+ */
+ uint64_t iInfo[ 9]; /* Auxiliary Integer Information
+ * iInfo[ 0]: The number of messages
+ * timings were averaged
+ * over.
+ * iInfo[ 1]: The message size sent
+ * between tasks.
+ * iInfo[ 2]: The number of warm-up
+ * messages sent before
+ * beginning timing.
+ * iInfo[ 3]: Number of serial retests
+ * iInfo[ 4]: Number of memory buffers
+ * if multiple memory
+ * buffers were used.
+ * iInfo[ 5]: MT_19937 seed for buffer
+ * randomization..
+ * iInfo[ 6]: If non-zero indicates the
+ * number of different task
+ * permutations used for
+ * averaging communication
+ * times.
+ * iInfo[ 7]: MT_19937 seed for task
+ * randomization.
+ * iInfo[ 8]: MT_19937 seed for step
+ * randomization.
+ */
+};
+
+enum OPTIONS{
+ A2A________FLAG= 0,
+ BIDIR______FLAG= 1,
+ UNIDIR_____FLAG= 2,
+ BISECT_____FLAG= 3,
+ RAND_STEP__FLAG= 4,
+ SERIAL_____FLAG= 5,
+ SION_OUT___FLAG= 6,
+ SION_PAR___FLAG= 7,
+ DEVICE_MEM_FLAG= 8,
+ MULTI_BUF__FLAG= 9,
+ RAND_BUF___FLAG=10,
+ CHECK_BUF__FLAG=11,
+ GROUP_HOST_FLAG=12,
+ MEM_ALLOC__TYPE=13
+};
+
+enum INFO{
+ NUM_MESSAGES=0,
+ MESSAGE_SIZE=1,
+ NUM_WARM__UP=2,
+ NUM___SERIAL=3,
+ NUM__MEM_BUF=4,
+ MEM_BUF_SEED=5,
+ NUM_PERMUTAT=6,
+ PERMUTA_SEED=7,
+ STEP____SEED=8
+};
+
+/*****************************/
+/* Read LinkTest Information */
+/*****************************/
+int readInfo_2_1_19(struct SionFile* const fh, const int swap, const Version version, struct LinkTestInfo_2_1_19* const linkTestInfo){
+ const size_t end_header_len=sizeof(END_HEADER_2_1_19)/sizeof(char);
+ char end_header[end_header_len];
+ size_t n;
+ int32_t strSize;
+
+ /* Git Hash */
+ n=fread(linkTestInfo->hash, sizeof(char), GIT_HASH_SIZE_2_1_19, fh->fp);
+ if (UNLIKELY(n != GIT_HASH_SIZE_2_1_19)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read git hash.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->hash, linkTestInfo->hash, sizeof(char), GIT_HASH_SIZE_2_1_19, swap);
+ if (UNLIKELY(linkTestInfo->hash[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: NULL git hashes are not supported.");
+ return -1;
+ }
+ linkTestInfo->hash[GIT_HASH_SIZE_2_1_19-1]='\0'; //Make sure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("git hash =%s\n",linkTestInfo->hash);fflush(stdout);
+ #endif
+
+ /* Read Virtual Cluster Implementation */
+ n=fread(&strSize, sizeof(int32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read string sizes for git hash, datetime and mode.");
+ return -1;
+ }
+ sion_swap(&strSize, &strSize, sizeof(int32_t), 1, swap);
+ linkTestInfo->mode=malloc(strSize*sizeof(char)); //TODO: Dangerous if sizeof(char) mismatches between writing and reading system.
+ n=fread(linkTestInfo->mode, sizeof(char), strSize, fh->fp);
+ if (UNLIKELY((int32_t)n != strSize)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read virtual-cluster implementation.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->mode, linkTestInfo->mode, sizeof(char), strSize, swap);
+ if (UNLIKELY(linkTestInfo->mode[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Virtual-cluster implementation NULL is not supported.");
+ return -1;
+ }
+ linkTestInfo->mode[strSize-1]='\0'; //Add null terminator to string
+ #ifdef DEBUG_OUTPUT
+ printf("mode =%s\n",linkTestInfo->mode);fflush(stdout);
+ #endif
+
+ /* Read 32-bit Integer Options */
+ n=fread(linkTestInfo->options, sizeof(int8_t), 14, fh->fp);
+ if (UNLIKELY(n != 14)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read 32-bit integer LinkTest options.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->options, linkTestInfo->options, sizeof(int8_t), 14, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("do_alltoall =%" PRId8 "\n",linkTestInfo->options[A2A________FLAG]);
+ printf("do_bidir =%" PRId8 "\n",linkTestInfo->options[BIDIR______FLAG]);
+ printf("do_unidir =%" PRId8 "\n",linkTestInfo->options[UNIDIR_____FLAG]);
+ printf("do_bisection =%" PRId8 "\n",linkTestInfo->options[BISECT_____FLAG]);
+ printf("do_mix =%" PRId8 "\n",linkTestInfo->options[RAND_STEP__FLAG]);
+ printf("do_serial =%" PRId8 "\n",linkTestInfo->options[SERIAL_____FLAG]);
+ printf("do_nosion =%" PRId8 "\n",linkTestInfo->options[SION_OUT___FLAG]);
+ printf("do_sion_par =%" PRId8 "\n",linkTestInfo->options[SION_PAR___FLAG]);
+ printf("do_use_gpus =%" PRId8 "\n",linkTestInfo->options[DEVICE_MEM_FLAG]);
+ printf("use_multi_buf =%" PRId8 "\n",linkTestInfo->options[MULTI_BUF__FLAG]);
+ printf("randomize_buffers =%" PRId8 "\n",linkTestInfo->options[RAND_BUF___FLAG]);
+ printf("check_buffers =%" PRId8 "\n",linkTestInfo->options[CHECK_BUF__FLAG]);
+ printf("group_processes_by_hostname=%" PRId8 "\n",linkTestInfo->options[GROUP_HOST_FLAG]);
+ printf("memory allocator type =%" PRId8 "\n",linkTestInfo->options[MEM_ALLOC__TYPE]);
+ fflush(stdout);
+ #endif
+
+ /* Read 32-bit Integer Information */
+ n=fread(linkTestInfo->iInfo, sizeof(uint64_t), 9, fh->fp);
+ if (UNLIKELY(n != 9)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read 32-bit integer LinkTest information.");
+ return -1;
+ }
+ sion_swap(linkTestInfo->iInfo, linkTestInfo->iInfo, sizeof(uint64_t), 9, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("num_msg =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_MESSAGES]);
+ printf("len_msg =%" PRIu64 "\n",linkTestInfo->iInfo[MESSAGE_SIZE]);
+ printf("num_warmup_msg =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_WARM__UP]);
+ printf("max_stest =%" PRIu64 "\n",linkTestInfo->iInfo[NUM___SERIAL]);
+ printf("num_multi_buf =%" PRIu64 "\n",linkTestInfo->iInfo[NUM__MEM_BUF]);
+ printf("buf_mt_seed =%" PRIu64 "\n",linkTestInfo->iInfo[MEM_BUF_SEED]);
+ printf("num_randomize_tasks =%" PRIu64 "\n",linkTestInfo->iInfo[NUM_PERMUTAT]);
+ printf("seed_randomize_tasks =%" PRIu64 "\n",linkTestInfo->iInfo[PERMUTA_SEED]);
+ printf("seed_randomize_steps =%" PRIu64 "\n",linkTestInfo->iInfo[STEP____SEED]);
+ fflush(stdout);
+ #endif
+
+ /* Check If 0/1 Multiple Buffer Were Specified */
+ if(linkTestInfo->options[MULTI_BUF__FLAG] != 0){
+ if(linkTestInfo->iInfo[NUM__MEM_BUF] == 0){
+ PyErr_WarnEx(NULL,"Detected that multiple buffers were used, however, an unknown (0) number of buffers were used.\n ReadSION: The number of buffers used was likely equal to the number of messages. Interpret results with caution.",(Py_ssize_t )1);
+ }else if(linkTestInfo->iInfo[NUM__MEM_BUF] == 1){
+ PyErr_WarnEx(NULL,"Detected that multiple buffers were used, however, only one buffer was used.\n ReadSION: In this scenario multiple buffers should not have been used. Interpret results with caution.",(Py_ssize_t )1);
+ }
+ }
+
+ /* Check If Mersenne Twister Seed Is Zero */
+ if(linkTestInfo->options[RAND_BUF___FLAG] != 0 && linkTestInfo->iInfo[MEM_BUF_SEED] == 0){
+ PyErr_WarnEx(NULL,"Buffer content was to be randomized, however, the stored zero Mersenne Twister seed would have filled the buffers with zero. Interpret results with caution.",(Py_ssize_t )1);
+ }
+
+ /* Check END_HEADER String */
+ n=fread(end_header,sizeof(char),end_header_len-1,fh->fp);
+ if (UNLIKELY(n != end_header_len-1)){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read END_HEADER string.");
+ return -1;
+ }
+ end_header[end_header_len-1]='\0';
+ if(UNLIKELY(strcmp(end_header,END_HEADER_2_1_19))){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Alignment Error Detected. \"END_HEADER\" not found at the correct location.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/*******************************************/
+/* Import LinkTest Information Into Python */
+/*******************************************/
+int importInfo_2_1_19(const Version version, struct LinkTestInfo_2_1_19* const linkTestInfo, PyObject* const* const pyLinkTestInfo){
+ int err;
+
+ // Allocator Type
+ enum AllocatorType{
+ AllocatorInvalid =0, //Invalid Allocator - ERROR
+ AllocatorDefault =1, //Default Allocator - ERROR
+ AllocatorAlignedMalloc =2, //Memory-aligned malloc Allocator
+ AllocatorPinnedMmap =3, //Pinned--memory-map Allocator
+ AllocatorPOSIXAlignedMalloc=4, //POSIX Memory-aligned malloc Allocator
+ AllocatorCUDA =5 //CUDA malloc Allocator
+ };
+
+ /* Set LinkTest Version */
+ /* TODO: There is a potential bug here. %d is used for a digit type,
+ * however this may be insufficiently long for 32-bit version numbers.
+ * This can cause errors. Python, however, does not support the standard
+ * C way of using PRId32.
+ */
+ err=PyDict_SetItem(*pyLinkTestInfo,
+ PyUnicode_FromString("Version"),
+ PyUnicode_FromFormat("%d.%d.%d",version[0],version[1],version[2])
+ );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add Version to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set LinkTest Git Hash */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Git hash" ),PyUnicode_FromString(linkTestInfo->hash ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add hash to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set Virtual-Cluster Implementation */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Mode" ),PyUnicode_FromString(linkTestInfo->mode));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add mode to LinkTestInfo dictionary.");
+ return -1;
+ }
+ free(linkTestInfo->mode);
+
+ /* Set Options */
+ // NOTE: Casting to a long int from an int32_t to a long int should
+ // be safe since a long int on standard compliant compilers have a
+ // minimum size of 32 bytes.
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Test MPI all-to-all?" ),PyLong_FromLong((long int)linkTestInfo->options[A2A________FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doAlltoall to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do bidirectional tests?" ),PyLong_FromLong((long int)linkTestInfo->options[BIDIR______FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doBidir to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do unidirectional tests?" ),PyLong_FromLong((long int)linkTestInfo->options[UNIDIR_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doUnidir to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do a bisection test?" ),PyLong_FromLong((long int)linkTestInfo->options[BISECT_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doBisec to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Randomize step order?" ),PyLong_FromLong((long int)linkTestInfo->options[RAND_STEP__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doMix to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Test serially?" ),PyLong_FromLong((long int)linkTestInfo->options[SERIAL_____FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doSerial to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Do not store results in a SION file?" ),PyLong_FromLong((long int)linkTestInfo->options[SION_OUT___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doWrite to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Write SION output in parallel?" ),PyLong_FromLong((long int)linkTestInfo->options[SION_PAR___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add doParallelIO to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Store messages in GPU memory?" ),PyLong_FromLong((long int)linkTestInfo->options[DEVICE_MEM_FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add useGPUs to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Use multiple memory buffers?" ),PyLong_FromLong((long int)linkTestInfo->options[MULTI_BUF__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add use_multi_buf to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Randomize memory-buffer content?" ),PyLong_FromLong((long int)linkTestInfo->options[RAND_BUF___FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add randomize_buffers to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Check memory-buffer content?" ),PyLong_FromLong((long int)linkTestInfo->options[CHECK_BUF__FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add check_buffers to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Group processes according to hostname?" ),PyLong_FromLong((long int)linkTestInfo->options[GROUP_HOST_FLAG]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add check_buffers to LinkTestInfo dictionary.");
+ return -1;
+ }
+ switch(linkTestInfo->options[MEM_ALLOC__TYPE]){
+ case AllocatorInvalid:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Invalid - ERROR" ));
+ break;
+ case AllocatorDefault:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Default - ERROR" ));
+ break;
+ case AllocatorAlignedMalloc:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Memory-aligned malloc" ));
+ break;
+ case AllocatorPinnedMmap:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("Pinned memory-map" ));
+ break;
+ case AllocatorPOSIXAlignedMalloc:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("POSIX memory-aligned malloc"));
+ break;
+ case AllocatorCUDA:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("CUDA malloc" ));
+ break;
+ default:
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Allocator type"),PyUnicode_FromString("UNKNOWN" ));
+ break;
+ }
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add memory-buffer allocator to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Set Integer Information */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of messages" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_MESSAGES]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add numberOfMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Message size" ),PyLong_FromLong((long int)linkTestInfo->iInfo[MESSAGE_SIZE]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add lengthOfMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of warm-up messages" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_WARM__UP]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add numberOfWarmUpMessages to LinkTestInfo dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of serial retests" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM___SERIAL]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add numberOfSerialRetests to LinkTestInfo dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->options[MULTI_BUF__FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of multiple buffers" ),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM__MEM_BUF]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add num_multi_buf to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[RAND_BUF___FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Buffer Mersenne Twister seed" ),PyLong_FromLong((long int)linkTestInfo->iInfo[MEM_BUF_SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add buf_mt_seed to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Number of tests with randomized rank order"),PyLong_FromLong((long int)linkTestInfo->iInfo[NUM_PERMUTAT]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add num_randomize_tasks to LinkTestInfo dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->iInfo[NUM_PERMUTAT]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Seed for rank-order randomization"),PyLong_FromLong((long int)linkTestInfo->iInfo[PERMUTA_SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add seed_randomize_tasks to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[RAND_STEP__FLAG]){
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Seed for step-order randomization"),PyLong_FromLong((long int)linkTestInfo->iInfo[STEP____SEED]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add seed_randomize_steps to LinkTestInfo dictionary.");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*******************************/
+/* Read & Import LinkTest Data */
+/*******************************/
+int readImportData_2_1_19(struct SionFile* const fh, const int swap, struct LinkTestInfo_2_1_19* const linkTestInfo, PyObject* const* const pyLinkTestInfo, PyObject** pyLinkTestData){
+ const size_t end_block_len=sizeof(END_BLOCK_2_1_19)/sizeof(char);
+ char end_block[end_block_len];
+ int err;
+ int32_t rank, hostname_len;
+ size_t n;
+ size_t i;
+ char* hostname;
+ npy_intp len[2];
+ void *ptr;
+
+ const size_t num_iterations =(linkTestInfo->iInfo[NUM_PERMUTAT]>1)?(size_t)linkTestInfo->iInfo[NUM_PERMUTAT]:1;
+ const size_t numMinMaxAvgTimings=(linkTestInfo->options[A2A________FLAG]!=0)?6:3;
+ double minMaxAvgTimings[num_iterations*numMinMaxAvgTimings];
+
+ //TODO: Handle Errors
+
+ /* Set Python Array Sizes */
+ len[0] = fh->size;
+ len[1] = fh->size;
+
+ /* Initialize Hostnames & Core IDs */
+ PyObject* const pyHosts = PyList_New((Py_ssize_t)fh->size);
+ PyObject* const pyCore = NewNumericPyArray(NPY_INT, 1, len);
+
+ /* Initialize Start & End Time Strings */
+ PyObject** const pyStartTime=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyStartTime[i]=PyDict_New();
+ PyObject** const pyEndTime =(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyEndTime[i] =PyDict_New();
+
+ /* Initialize Timings, Access Pattern & All-To-All Timings */
+ PyObject** const pyTimings =(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyTimings[i] =NewNumericPyArray(NPY_DOUBLE,2,len); //A python double-precision floating-point array of the LinkTest point-to-point timings
+ PyObject** const pyAccesspattern=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyAccesspattern[i]=NewNumericPyArray(NPY_UINT64,2,len); //A python integer array indicating the access order
+ PyObject** const pyA2ATimings =(linkTestInfo->options[A2A________FLAG]!=0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL; //Silences maybe-used-uninitialized warning
+ if(linkTestInfo->options[A2A________FLAG]!=0) for(i=0;i<num_iterations;i++) pyA2ATimings[i] =NewNumericPyArray(NPY_DOUBLE, 1, len); //A python double-precision floating-point array of the linkTest all-to-all imings
+
+ /* Initialize Serial Retest Timings */
+ len[0] = linkTestInfo->iInfo[NUM___SERIAL];
+ PyObject** const pySerialTimingsNew=(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTimingsNew[i] = NewNumericPyArray(NPY_DOUBLE, 1, len);
+ PyObject** const pySerialTimingsOld=(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTimingsOld[i] = NewNumericPyArray(NPY_DOUBLE, 1, len);
+ PyObject** const pySerialFrom =(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialFrom[i] = NewNumericPyArray(NPY_UINT64, 1, len);
+ PyObject** const pySerialTo =(len[0]>0)?(PyObject**)malloc(num_iterations*sizeof(PyObject)):NULL;
+ if(len[0]>0) for(i=0;i<num_iterations;i++) pySerialTo[i] = NewNumericPyArray(NPY_UINT64, 1, len);
+
+ for (rank = 0; rank < fh->size; ++rank) { //Loop over the ranks, each one was saved to its own chunk
+ /* Read Host Name (Multiple Tasks May Have The Same Host Name */
+ n=fread(&hostname_len, sizeof(uint32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import hostname length.");
+ return -1;
+ }
+ sion_swap(&hostname_len, &hostname_len, sizeof(uint32_t), 1, swap);
+ hostname=(char*)malloc(hostname_len*sizeof(char));
+ n=fread(hostname, sizeof(char), hostname_len, fh->fp);
+ if (UNLIKELY((int32_t)n != hostname_len)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import Hostname.");
+ return -1;
+ }
+ sion_swap(&hostname, &hostname, sizeof(char), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("Hostname[%d] =%s\n",rank,hostname);fflush(stdout);
+ #endif
+ PyList_SET_ITEM(pyHosts,(Py_ssize_t)rank,PyBytes_FromStringAndSize(hostname,(Py_ssize_t)hostname_len));
+ /* Read Pinned-Core ID */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pyCore, rank);
+ n=fread(ptr, sizeof(int32_t), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import Pinned Core ID.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(int32_t), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("cpuID[%d] =%d\n",rank,*(int32_t*)ptr);fflush(stdout);
+ #endif
+
+ for(i=0;i<num_iterations;i++){
+ if(rank==0){
+ /* Read Start Time */
+ char datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19+1]; //TODO: Dangerous if sizeof(char) mismatches between writing and reading system.
+ n=fread(datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19, fh->fp);
+ if (UNLIKELY(n != LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read startDatetime.");
+ return -1;
+ }
+ sion_swap(datetime_string, datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19, swap);
+ if (UNLIKELY(datetime_string[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: startDatetime NULL is not supported.");
+ return -1;
+ }
+ datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19]='\0'; //Ensure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("startDatetime[0,%zu] =%s",i,datetime_string);
+ fflush(stdout);
+ #endif
+ struct Datetime startDatetime;
+ n=sscanf(datetime_string,
+ "%2"SCNu8"/%2"SCNu8"/%"SCNu64" %2"SCNu8":%2"SCNu8":%2"SCNu8" UTC+0",
+ &startDatetime.day,
+ &startDatetime.month,
+ &startDatetime.year,
+ &startDatetime.hour,
+ &startDatetime.minute,
+ &startDatetime.second
+ );
+ if (UNLIKELY(n!=6)){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: startDatetime is not a correctly formatted string.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("startDatetime.year[0,%zu] =%" PRIu64 "\n",i,startDatetime.year );
+ printf("startDatetime.month[0,%zu] =%" PRIu8 "\n",i,startDatetime.month );
+ printf("startDatetime.day[0,%zu] =%" PRIu8 "\n",i,startDatetime.day );
+ printf("startDatetime.hour[0,%zu] =%" PRIu8 "\n",i,startDatetime.hour );
+ printf("startDatetime.minute[0,%zu] =%" PRIu8 "\n",i,startDatetime.minute);
+ printf("startDatetime.second[0,%zu] =%" PRIu8 "\n",i,startDatetime.second);
+ printf("startDatetime.timezone[0,%zu]=UTC+0\n" ,i );
+ fflush(stdout);
+ #endif
+ /* Import Start Time */
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Year" ),PyLong_FromLong((long int)startDatetime.year ));
+ if(err){//TODO: Add iteration index to error message
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_year to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Month" ),PyLong_FromLong((long int)startDatetime.month ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_month to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Day" ),PyLong_FromLong((long int)startDatetime.day ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_day to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Hour" ),PyLong_FromLong((long int)startDatetime.hour ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_hour to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Minute" ),PyLong_FromLong((long int)startDatetime.minute));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_minute to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Second" ),PyLong_FromLong((long int)startDatetime.second));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_second to pyStartTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyStartTime[i],PyUnicode_FromString("Timezone"),PyUnicode_FromFormat("UTC+0"));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_timezone to pyStartTime dictionary.");
+ return -1;
+ }
+ /* Read Minimum, Maximum & Average LinkTest Timings */
+ n=fread(&minMaxAvgTimings[i*numMinMaxAvgTimings], sizeof(double), numMinMaxAvgTimings, fh->fp);
+ if (UNLIKELY(n != numMinMaxAvgTimings)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read minimum, maximum & average LinkTest timings.");
+ return -1;
+ }
+ sion_swap(minMaxAvgTimings, minMaxAvgTimings, sizeof(double), numMinMaxAvgTimings, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("min_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+0]);
+ printf("max_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+1]);
+ printf("avg_time[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+2]);
+ if(numMinMaxAvgTimings==6){
+ printf("min_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+3]);
+ printf("max_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+4]);
+ printf("avg_time_a2a[0,%zu] =%e\n",i,minMaxAvgTimings[i*numMinMaxAvgTimings+5]);
+ }
+ fflush(stdout);
+ #endif
+ /* Read Serial Retest Data */
+ if (linkTestInfo->iInfo[NUM___SERIAL]>0){
+ /* Read New Serially Retested Timings */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTimingsNew[i], 0);
+ n=fread(ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import retested serial timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsNew[0,%zu] =%f",i,((double*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%f",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read Old Serially Retested Timings */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTimingsOld[i], 0);
+ n=fread(ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import retest-needed timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double ), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsOld[0,%zu] =%f",i,((double*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%f",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read Origin Partner For Serial Retests */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialFrom[i] , 0);
+ n=fread(ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import serial retest origin.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsFrom[0,%zu] =%d",i,((uint64_t*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%"PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ sion_swap(ptr, ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ /* Read Destination Partner For Serial Retests */
+ ptr=PyArray_GETPTR1((PyArrayObject *)pySerialTo[i] , 0);
+ n=fread(ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], fh->fp);
+ if (UNLIKELY((uint64_t)n != linkTestInfo->iInfo[NUM___SERIAL])) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import serial retest destination.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(uint64_t), linkTestInfo->iInfo[NUM___SERIAL], swap);
+ #ifdef DEBUG_OUTPUT
+ printf("serialTimingsTo[0,%zu] =%d",i,((uint64_t*)ptr)[0]);
+ for(n=1;(uint64_t)n<linkTestInfo->iInfo[NUM___SERIAL];n++) printf(",%"PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ }
+ /* Read End Time */
+ n=fread(datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19, fh->fp);
+ if (UNLIKELY(n != LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19)) {
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: Failed to read endDatetime.");
+ return -1;
+ }
+ sion_swap(datetime_string, datetime_string, sizeof(char), LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19, swap);
+ if (UNLIKELY(datetime_string[0] == '\0')){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: endDatetime NULL is not supported.");
+ return -1;
+ }
+ datetime_string[LINKTEST_OUTPUT_DATETIME_LENGTH_2_1_19]='\0'; //Ensure string is null terminated
+ #ifdef DEBUG_OUTPUT
+ printf("endDatetime[0,%zu] =%s",i,datetime_string);
+ fflush(stdout);
+ #endif
+ struct Datetime endDatetime;
+ n=sscanf(datetime_string,
+ "%2"SCNu8"/%2"SCNu8"/%"SCNu64" %2"SCNu8":%2"SCNu8":%2"SCNu8" UTC+0",
+ &endDatetime.day,
+ &endDatetime.month,
+ &endDatetime.year,
+ &endDatetime.hour,
+ &endDatetime.minute,
+ &endDatetime.second
+ );
+ if (UNLIKELY(n!=6)){
+ PyErr_SetString(PyExc_RuntimeError, "readInfo_2_1_19: endDatetime is not a correctly formatted string.");
+ return -1;
+ }
+ #ifdef DEBUG_OUTPUT
+ printf("endDatetime.year[0,%zu] =%" PRIu64 "\n",i,endDatetime.year );
+ printf("endDatetime.month[0,%zu] =%" PRIu8 "\n",i,endDatetime.month );
+ printf("endDatetime.day[0,%zu] =%" PRIu8 "\n",i,endDatetime.day );
+ printf("endDatetime.hour[0,%zu] =%" PRIu8 "\n",i,endDatetime.hour );
+ printf("endDatetime.minute[0,%zu] =%" PRIu8 "\n",i,endDatetime.minute);
+ printf("endDatetime.second[0,%zu] =%" PRIu8 "\n",i,endDatetime.second);
+ printf("endDatetime.timezone[0,%zu] =UTC+0\n" ,i );
+ fflush(stdout);
+ #endif
+ /* Import End Time */
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Year" ),PyLong_FromLong((long int)endDatetime.year ));
+ if(err){//TODO: Add iteration index to error message
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_year to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Month" ),PyLong_FromLong((long int)endDatetime.month ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_month to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Day" ),PyLong_FromLong((long int)endDatetime.day ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_day to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Hour" ),PyLong_FromLong((long int)endDatetime.hour ));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_hour to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Minute" ),PyLong_FromLong((long int)endDatetime.minute));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_minute to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Second" ),PyLong_FromLong((long int)endDatetime.second));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_second to pyEndTime dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyEndTime[i],PyUnicode_FromString("Timezone"),PyUnicode_FromFormat("UTC+0"));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add DateTime_timezone to pyEndTime dictionary.");
+ return -1;
+ }
+ }
+ /* Read LinkTest Timings */
+ ptr=PyArray_GETPTR2((PyArrayObject *)pyTimings[i] , rank, 0);
+ n=fread(ptr, sizeof(double), fh->size, fh->fp);
+ if (UNLIKELY((int32_t)n != fh->size)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import LinkTest timings.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double), fh->size, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("timings[%d,%zu] =%e",rank,i,((double*)ptr)[0]);
+ for(n=1;(int32_t)n<fh->size;n++) printf(",%e",((double*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read LinkTest Access Pattern */
+ ptr=PyArray_GETPTR2((PyArrayObject *)pyAccesspattern[i], rank, 0);
+ n=fread(ptr, sizeof(int64_t), fh->size, fh->fp);
+ if (UNLIKELY((int64_t)n != fh->size)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import LinkTest access pattern.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(uint64_t), fh->size, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("accesspattern[%d,%zu] =%" PRIu64,rank,i,((uint64_t*)ptr)[0]);
+ for(n=1;(int32_t)n<fh->size;n++) printf(",%" PRIu64,((uint64_t*)ptr)[n]);
+ printf("\n");
+ fflush(stdout);
+ #endif
+ /* Read LinkTest All-To-All Timings */
+ if (linkTestInfo->options[A2A________FLAG]){
+ ptr=PyArray_GETPTR1((PyArrayObject *)pyA2ATimings[i], rank);
+ n=fread(ptr, sizeof(double), 1, fh->fp);
+ if (UNLIKELY(n != 1)) {
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read/import all-to-all timing.");
+ return -1;
+ }
+ sion_swap(ptr, ptr, sizeof(double), 1, swap);
+ #ifdef DEBUG_OUTPUT
+ printf("all-to-all time[%d,%zu] =%f\n",rank,i,((double*)ptr)[0]);fflush(stdout);
+ #endif
+ }
+ }
+
+ /* Check Alignment by Testing for END_BLOCK */
+ n=fread(end_block,sizeof(char),end_block_len-1,fh->fp);
+ if (UNLIKELY(n != end_block_len-1)){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Failed to read END_BLOCK string.");
+ return -1;
+ }
+ end_block[end_block_len-1]='\0';
+ if(UNLIKELY(strcmp(end_block,END_BLOCK_2_1_19))){
+ PyErr_SetString(PyExc_RuntimeError, "readImportData_2_1_19: Alignment Error Detected. \"END_BLOCK\" not found at the correct location.");
+ return -1;
+ }
+ }
+
+ /* Close SION File */
+ sion_close(fh->sid);
+
+ /* Add Hostname To LinkTestInfo Dictionary */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Hosts"),pyHosts);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add Hosts to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Add Core ID To LinkTestInfo Dictionary */
+ err=PyDict_SetItem(*pyLinkTestInfo,PyUnicode_FromString("Core ID"),pyCore);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to add Core ID to LinkTestInfo dictionary.");
+ return -1;
+ }
+
+ /* Create Data Dictionaries */
+ PyObject** const pyLinkTestDataDicts=(PyObject**)malloc(num_iterations*sizeof(PyObject));
+ for(i=0;i<num_iterations;i++) pyLinkTestDataDicts[i] = PyDict_New();
+
+ /* Fill Dictionaries */
+ for(i=0;i<num_iterations;i++){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Start Time" ), pyStartTime[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add start time to LinkTestData dictionary.");
+ return -1;
+ }
+ /* Set Double-Precision Floating-Point Information */
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Minimum time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+0]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add minTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Maximum time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+1]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add maxTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Average time" ),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+2]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add avgTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ if(numMinMaxAvgTimings==6){ //Only execute if all-to-all testing was performed
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Minimum all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+3]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aMinTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Maximum all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+4]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aMaxTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i],PyUnicode_FromString("Average all-to-all time"),PyFloat_FromDouble(minMaxAvgTimings[i*numMinMaxAvgTimings+5]));
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_2: Failed to add a2aAvgTimeAll to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Timings" ), pyTimings[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add timings to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Access Pattern"), pyAccesspattern[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add accessPatterns to LinkTestData dictionary.");
+ return -1;
+ }
+ if(linkTestInfo->iInfo[NUM___SERIAL]>0){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("New serially retested timings" ), pySerialTimingsNew[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add serialTimingsNew to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Old serially retested timings" ), pySerialTimingsOld[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add serialTimingsOld to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Serial-retest origin hosts" ), pySerialFrom[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add serialFrom to LinkTestData dictionary.");
+ return -1;
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("Serial-retest destination hosts"), pySerialTo[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add serialTo to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+ if(linkTestInfo->options[A2A________FLAG]){
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("All-to-all timings"), pyA2ATimings[i]);
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add a2aTimings to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+ err=PyDict_SetItem(pyLinkTestDataDicts[i], PyUnicode_FromString("End Time" ), pyEndTime[i] );
+ if(err){
+ PyErr_SetString(PyExc_RuntimeError,"readImportData_2_1_19: Failed to add end time to LinkTestData dictionary.");
+ return -1;
+ }
+ }
+
+ /* Create Python List If Necessary */
+ if(num_iterations>1){
+ *pyLinkTestData=PyList_New(num_iterations);
+ for(i=0;i<num_iterations;i++)PyList_SET_ITEM(*pyLinkTestData,i,pyLinkTestDataDicts[i]);
+ }else{
+ *pyLinkTestData=pyLinkTestDataDicts[0];
+ }
+
+ /* Free Memory */
+ free(pyTimings );
+ free(pyAccesspattern);
+ if(pyA2ATimings!=NULL) free(pyA2ATimings);
+ if(linkTestInfo->iInfo[NUM___SERIAL]>0){
+ free(pySerialTimingsNew);
+ free(pySerialTimingsOld);
+ free(pySerialFrom );
+ free(pySerialTo );
+ }
+
+ return 0;
+}
+
+/**********************/
+/* Read & Import Data */
+/**********************/
+int readImport_2_1_19(struct SionFile* const fh, const int swap, const Version version, PyObject** const pyLinkTestInfo, PyObject** const pyLinkTestData){
+ struct LinkTestInfo_2_1_19 linkTestInfo;
+ int err;
+ /* Read LinkTest Information */
+ err = readInfo_2_1_19(fh, swap, version, &linkTestInfo);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to read LinkTest information.");
+ return err;
+ }
+ /* Import LinkTest Information Into Python */
+ *pyLinkTestInfo = PyDict_New(); //Create LinkTest information dictionary
+ err = importInfo_2_1_19(version, &linkTestInfo, pyLinkTestInfo);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to populate the LinkTestInfo dictionary.");
+ //TODO: Safely Deallocate Python Dictionary
+ return err;
+ }
+ /* Read & Import LinkTest Data Into Python */
+ err = readImportData_2_1_19(fh, swap, &linkTestInfo, pyLinkTestInfo, pyLinkTestData);
+ if(err){
+// PyErr_SetString(PyExc_RuntimeError,"importInfo_2_1_19: Failed to read LinkTest data or to populate the LinkTestData dictionary.");
+ //TODO: Safely Deallocate Python Dictionary
+ return err;
+ }
+
+ return(0);
+}
diff --git a/python/linktest/linktest_2_1_19.h b/python/linktest/linktest_2_1_19.h
new file mode 100644
index 0000000000000000000000000000000000000000..1d607c32cefb5cc341bf867110156ee64eb72dd8
--- /dev/null
+++ b/python/linktest/linktest_2_1_19.h
@@ -0,0 +1,13 @@
+/****************************************************************************
+** LinkTest **
+*****************************************************************************
+** Copyright (c) 2008-2022 **
+** Forschungszentrum Juelich, Juelich Supercomputing Centre **
+** **
+** See the file COPYRIGHT in the package base directory for details **
+****************************************************************************/
+
+/**********************/
+/* Read & Import Data */
+/**********************/
+int readImport_2_1_19(struct SionFile* const fh, const int swap, const Version version, PyObject** const pyLinkTestInfo, PyObject** const pyLinkTestData);
diff --git a/python/linktest/report.py b/python/linktest/report.py
index e8cd0d5fde600973ed3d7e2db7380e2711df7d8a..e340599484de3be3796a8e15da20c2ddbdf8800c 100644
--- a/python/linktest/report.py
+++ b/python/linktest/report.py
@@ -27,9 +27,9 @@ from sys import exit
from argparse import ArgumentParser
from importlib.metadata import metadata,version
from operator import add
-from numpy import amax,amin,append,arange,argmax,array,ceil,concatenate,empty,floor,linspace,log2,log10,nan,nansum,nanmin,nanmax,unique
+from numpy import amax,amin,append,arange,argmax,argsort,argwhere,array,ceil,concatenate,empty,floor,linspace,log2,log10,nan,nansum,nanmin,nanmax,unique,zeros
from time import time
-import datetime
+from datetime import datetime
from matplotlib.cm import get_cmap
from matplotlib.colors import Normalize
from matplotlib.figure import Figure
@@ -133,7 +133,7 @@ def max_bbox_extend_1pt(strings,fast):
return([max_text_width_cm,max_text_height_cm])
def auto_fit_fontsize(width_cm_1p,height_cm_1pt,width_cm,height_cm):
- '''Estimates elligible font size based on 1pt text box extend
+ '''Estimates eligible font size based on 1pt text box extend
Inputs:
width_cm_1p (float): 1pt textbox width in cm
@@ -157,7 +157,16 @@ def auto_fit_fontsize(width_cm_1p,height_cm_1pt,width_cm,height_cm):
font_size=1.0
return(font_size)
-def SI_Prefix(num,suffixes=["y","z","a","f","p","n","µ","m"," ","k","M","G","T","P","E","Z","Y"]):
+def SI_Prefix(num):
+ '''Formats a number into a string such that after the number there is a space followed the relevant SI prefix.
+
+ Inputs:
+ num: The number to be formatted into a string
+
+ Output:
+ str: A string containing the number followed by a white space and the corresponding SI prefix
+ '''
+ suffixes=["y","z","a","f","p","n","µ","m"," ","k","M","G","T","P","E","Z","Y"]
if num==0:
return "0.0000 "
m=floor(log10(num)/3)
@@ -165,27 +174,35 @@ def SI_Prefix(num,suffixes=["y","z","a","f","p","n","µ","m"," ","k","M","G","T
m=(m+8).astype(int)
if m<0:m=0
if num<10: #TODO: Is there not a more elegant solution for this?
- return "{num:.4f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.4f} {suffixes[m]}"
elif num<100:
- return "{num:.3f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.3f} {suffixes[m]}"
elif num<1000:
- return "{num:.2f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.2f} {suffixes[m]}"
else:
- return "{num:.1f}. {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.1f} {suffixes[m]}"
def IEC_Binary_Prefix(num,suffixes=[" ","ki","Mi","Gi","Ti","Pi","Ei","Zi","Yi"]):
+ '''Formats a number into a string such that after the number there is a space followed the relevant IEC binary prefix.
+
+ Inputs:
+ num: The number to be formatted into a string
+
+ Output:
+ str: A string containing the number followed by a white space and the corresponding IEC binary prefix
+ '''
if num==0:
return "0.0000 "
m=sum([abs(num/1024.0**x)>=1 for x in range(1,len(suffixes))])
num/=1024.0**m
if num<10: #TODO: Is there not a more elegant solution for this?
- return "{num:.4f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.4f} {suffixes[m]}"
elif num<100:
- return "{num:.3f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.3f} {suffixes[m]}"
elif num<1000:
- return "{num:.2f} {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.2f} {suffixes[m]}"
else:
- return "{num:.1f}. {suffix}".format(num=num,suffix=suffixes[m])
+ return f"{num:.1f} {suffixes[m]}"
def add_coordinates(coordinate1,coordinate2):
return(list(map(add,coordinate1,coordinate2)))
@@ -317,6 +334,41 @@ def determine_Cutoffs(data,cutoffs,clip_range):
timings_clipped=True
return cutoffs,minMaxTimings,minMaxPlotTimings,timings_clipped
+def aggregateAccordingToHostname(metaData, timingData, aggregate_bandwidth_according_to_hostname):
+ # TODO: Store the idx from one call to the next as for
+ # a given SION file they will not change.
+ Hosts=array([x.decode('UTF-8')[:-1] for x in metaData["Hosts"]])
+ tmp, ind=unique(Hosts,return_index=True)
+ ind=argsort(ind)
+ uHosts=tmp[ind]
+ del(tmp)
+ metaData["Hosts"]=array([(x+'!').encode('UTF-8') for x in uHosts]) #Add ! to end of string so that it can be removed by other functions later
+ del(ind)
+ idx=[argwhere(Hosts==u) for u in uHosts]
+ del(Hosts)
+ timings=zeros([len(idx),len(idx)])
+ if aggregate_bandwidth_according_to_hostname:
+ for i in range(len(idx)):
+ for j in range(len(idx)):
+ if(i==j):
+ timings[i][j]=nan
+ continue;
+ for k in range(len(idx[i])):
+ for l in range(len(idx[j])):
+ timings[i][j]+=1/timingData["Timings"][idx[i][k],idx[j][l]]
+ timings[i][j]=len(idx[i])/timings[i][j]
+ else:
+ for i in range(len(idx)):
+ for j in range(len(idx)):
+ if(i==j):
+ timings[i][j]=nan
+ continue;
+ for k in range(len(idx[i])):
+ for l in range(len(idx[j])):
+ timings[i][j]+=timingData["Timings"][idx[i][k],idx[j][l]]
+ timings[i][j]/=len(idx[i])*len(idx[j])
+ timingData["Timings"]=timings #Bandwidth data still encoded as timing
+
def include_Page_Number(pageNumberInfo,figure):
page_number_position_cm=[10.5, 0.7, 0.0, 0.0] #Anchor: Baseline Center
page_number_font_size =7
@@ -342,7 +394,9 @@ def timing___report(pdf,
replace_slow,
correct_image_dimensions,
pageNumberInfo,
- aggregated=False
+ aggregated=False,
+ average_according_to_hostname=False,
+ aggregate_bandwidth_according_to_hostname=False
):
# Position Data
@@ -368,22 +422,28 @@ def timing___report(pdf,
else:
timings_replaced=False;
+ # Aggregate According to Hostname
+ if average_according_to_hostname or aggregate_bandwidth_according_to_hostname:
+ mData=metaData
+ metaData=metaData.copy() #We modify the metadata so we need to make a copy of it to reconstruct it later. Python can only pass by reference!
+ aggregateAccordingToHostname(metaData,timingData,aggregate_bandwidth_according_to_hostname)
+
# Precompute Bisection Bandwidth
# This needs to be precomputed in case timings are clipped
numHosts=len(metaData["Hosts"])
if "Do a bisection test?" in metaData and metaData["Do a bisection test?"] != 0:
bisection_bandwidth=metaData["Message size"]*nansum(1.0/timingData["Timings"])/numHosts
- # Determine Data Range & Apply Cutoffs
+ # Determine Data Range & Apply Cut-Offs
cutoffs,minMaxTimings,minMaxPlotTimings,timings_clipped=determine_Cutoffs(timingData["Timings"],cutoffs,plot_range)
if verbose:
- time2=time();print(f"Replacing slowest connections, precomputing bisection bandwidth and determining cutoffs took {time2-time1:.3f} s.");time1=time2;
+ time2=time();print(f"Replacing slowest connections, aggregating according to hostname, precomputing bisection bandwidth and determining cutoffs took {time2-time1:.3f} s.");time1=time2;
# Determine Hosts
numDigitsLastHostRank=floor(log10(numHosts)).astype(int)+1
hosts=[f'{host[:-1].decode("UTF-8")}{"." if numHosts!=0 else ""}{rank:0{numDigitsLastHostRank}d}' for rank,host in enumerate(metaData["Hosts"],start=0)]
- if domain: #Empty strings are falsy
+ if domain: #Empty strings are false
hosts=[host.replace(domain,"") for host in hosts]
# Determine String Y-Label Maximum String Length
@@ -425,10 +485,13 @@ def timing___report(pdf,
txt=f"{metaData['Write timestamp: Hour']:02d}:{metaData['Write timestamp: Minute']:02d}:{metaData['Write timestamp: Second']:02d} {metaData[ 'Write timestamp: Day']:02d}/{metaData['Write timestamp: Month']:02d}/{metaData['Write timestamp: Year']:0d} {metaData['Write timestamp: Timezone']}"
else:
txt=f"{metaData['Write timestamp: Hour']:02d}:{metaData['Write timestamp: Minute']:02d}:{metaData['Write timestamp: Second']:02d} {metaData[ 'Write timestamp: Day']:02d}/{metaData['Write timestamp: Month']:02d}/{metaData['Write timestamp: Year']:0d} UTC+?"
- if "Start Time" in timingData: #Present from version 2.1.16 onwards
- txt=f"{timingData['Start Time']['Hour']:02d}:{timingData['Start Time']['Minute']:02d}:{timingData['Start Time']['Second']:02d} {timingData['Start Time']['Day']:02d}/{timingData['Start Time']['Month']:02d}/{timingData['Start Time']['Year']:0d} \u2014 {timingData['End Time']['Hour']:02d}:{timingData['End Time']['Minute']:02d}:{timingData['End Time']['Second']:02d} {timingData['End Time']['Day']:02d}/{timingData['End Time']['Month']:02d}/{timingData['End Time']['Year']:0d} {timingData['End Time']['Timezone']}" #\u2014 is the unicode char for the emdash
+ else:
+ if "Start Time" in timingData: #Present from version 2.1.16 onwards
+ txt=f"{timingData['Start Time']['Hour']:02d}:{timingData['Start Time']['Minute']:02d}:{timingData['Start Time']['Second']:02d} {timingData['Start Time']['Day']:02d}/{timingData['Start Time']['Month']:02d}/{timingData['Start Time']['Year']:0d} \u2014 {timingData['End Time']['Hour']:02d}:{timingData['End Time']['Minute']:02d}:{timingData['End Time']['Second']:02d} {timingData['End Time']['Day']:02d}/{timingData['End Time']['Month']:02d}/{timingData['End Time']['Year']:0d} {timingData['End Time']['Timezone']}" #\u2014 is the unicode char for the emdash
if txt!="":
figure.text(x=date_position_norm[0] ,y=date_position_norm[1] ,s=txt,fontsize=meta_font_size,horizontalalignment="right",verticalalignment="top")
+
+ # Draw Clipping/Replacement Info
if timings_clipped:
if timings_replaced:
txt="WARNING: Replaced & Clipped"
@@ -461,12 +524,19 @@ def timing___report(pdf,
if correct_image_dimensions:
fixedPixelImage_shiftLabels(figure=figure,axis=TTT_axis)
cticks=linspace(minMaxPlotTimings[0],minMaxPlotTimings[1],num=25)
- clabels=[f"{SI_Prefix(tick)}s\n{IEC_Binary_Prefix(metaData['Message size']/tick)}B/s" for tick in cticks]
+ if aggregate_bandwidth_according_to_hostname:
+ clabels=[f"{IEC_Binary_Prefix(metaData['Message size']/tick)}B/s" for tick in cticks]
+ else:
+ clabels=[f"{SI_Prefix(tick)}s\n{IEC_Binary_Prefix(metaData['Message size']/tick)}B/s" for tick in cticks]
colorbar_axis=figure.add_axes([TTT_axis.get_position().x1+0.01,TTT_axis.get_position().y0,0.02,TTT_axis.get_position().height])
colorbar_axis.tick_params(labelsize=6)
figure.colorbar(TTT_Image,cax=colorbar_axis,ticks=cticks) #BUG: Vertical extent too large
colorbar_axis.set_yticklabels(clabels)
colorbar_axis.invert_yaxis()
+ if average_according_to_hostname:
+ colorbar_axis.set_ylabel("Averaged times for grouping according to hostname")
+ if aggregate_bandwidth_according_to_hostname:
+ colorbar_axis.set_ylabel("Aggregated bandwidths for grouping according to hostname")
if verbose:
time2=time();print(f"Drawing indexed-image took {time2-time1:.3f} s.");time1=time2;
@@ -498,7 +568,10 @@ def timing___report(pdf,
patch.set_facecolor(histogram_cmap(norm))
# Determine X-Tick Labels
histogram_x_ticks=linspace(bins[0],bins[-1],num=11)
- histogram_x_tick_labels=[f'{SI_Prefix(tick)}s\n{IEC_Binary_Prefix(metaData["Message size"]/tick)}B/s' for tick in histogram_x_ticks]
+ if aggregate_bandwidth_according_to_hostname:
+ histogram_x_tick_labels=[f'{IEC_Binary_Prefix(metaData["Message size"]/tick)}B/s' for tick in histogram_x_ticks]
+ else:
+ histogram_x_tick_labels=[f'{SI_Prefix(tick)}s\n{IEC_Binary_Prefix(metaData["Message size"]/tick)}B/s' for tick in histogram_x_ticks]
histogram_axis.set_xlim(bins[0],bins[-1])
histogram_axis.set_xticks(histogram_x_ticks)
histogram_axis.set_xticklabels(histogram_x_tick_labels,fontsize=6)
@@ -677,8 +750,8 @@ def timing___report(pdf,
figure.text(x=text_position_norm[0],y=text_position_norm[1],s="Step PRNG Seed:",fontsize=meta_font_size)
if "Randomize step order?" in metaData:
if metaData["Randomize step order?"]!=0:
- if "Step Mersenne Twister seed" in metaData:
- txt=f"{metaData['Step Mersenne Twister seed']}"
+ if "Seed for step-order randomization" in metaData:
+ txt=f"{metaData['Seed for step-order randomization']}"
else:
txt="???"
else:
@@ -756,7 +829,19 @@ def timing___report(pdf,
figure.text(x=text_position_norm[0],y=text_position_norm[1],s=f"{IEC_Binary_Prefix(metaData['Message size']*n*(n-1)/timingData['Maximum all-to-all time'])}B/s",fontsize=meta_font_size,horizontalalignment="right")
nslow-=2
ypos-=0.80
- if "Do a bisection test?" in metaData and metaData["Do a bisection test?"] != 0:
+ if "Group processes according to hostname?" in metaData and metaData["Group processes according to hostname?"] != 0:
+ text_position_norm=normalize_position(add_coordinates(metadata_position_cm,[ 0.00,ypos,0,0]),figure_size_cm)
+ figure.text(x=text_position_norm[0],y=text_position_norm[1],s="Grouping:",fontsize=meta_font_size)
+ text_position_norm=[text1.get_window_extent(renderer=renderer).x1/(figure.dpi*figure_size_in[0]),(metadata_position_cm[1]+ypos)/figure_size_cm[1]]
+ figure.text(x=text_position_norm[0],y=text_position_norm[1],s="Hostname",fontsize=meta_font_size,horizontalalignment="right")
+ if "Do a bisection test?" in metaData and metaData["Do a bisection test?"] != 0:
+ text_position_norm=normalize_position(add_coordinates(metadata_position_cm,[ 5.90,ypos,0,0]),figure_size_cm)
+ figure.text(x=text_position_norm[0],y=text_position_norm[1],s="Bisection Bandwidth:",fontsize=meta_font_size)
+ text_position_norm=[text2.get_window_extent(renderer=renderer).x1/(figure.dpi*figure_size_in[0]),(metadata_position_cm[1]+ypos)/figure_size_cm[1]]
+ figure.text(x=text_position_norm[0],y=text_position_norm[1],s=f"{IEC_Binary_Prefix(bisection_bandwidth)}B/s",fontsize=meta_font_size,horizontalalignment="right")
+ nslow-=1
+ ypos-=0.40
+ elif "Do a bisection test?" in metaData and metaData["Do a bisection test?"] != 0:
text_position_norm=normalize_position(add_coordinates(metadata_position_cm,[ 0.00,ypos,0,0]),figure_size_cm)
figure.text(x=text_position_norm[0],y=text_position_norm[1],s="Bisection Bandwidth:",fontsize=meta_font_size)
text_position_norm=[text1.get_window_extent(renderer=renderer).x1/(figure.dpi*figure_size_in[0]),(metadata_position_cm[1]+ypos)/figure_size_cm[1]]
@@ -809,10 +894,13 @@ def timing___report(pdf,
time2=time();print(f"Drawing footer took {time2-time1:.3f} s.");time1=time2;
# Save Report
- pdf.savefig(figure)
+ pdf.savefig(figure,transparent=True)
if verbose:
time2=time();print(f"Saving timing report took {time2-time1:.3f} s.");
+ if average_according_to_hostname:
+ metaData=mData #Reset metaData
+
def pattern__report(pdf,
metaData,
timingData,
@@ -844,16 +932,13 @@ def pattern__report(pdf,
time1=time()
# Determine Data Range & Apply Cut-Offs
- minMaxTimings=[nanmin(timingData["Timings"]),nanmax(timingData["Timings"])]
-
- # Determine Data Range & Apply Cutoffs
cutoffs,minMaxTimings,minMaxPlotTimings,timings_clipped=determine_Cutoffs(timingData["Timings"],cutoffs,plot_range)
# Determine Hosts
numHosts=len(metaData["Hosts"])
numDigitsLastHostRank=floor(log10(numHosts)).astype(int)+1
hosts=[f'{host[:-1].decode("UTF-8")}{"." if len(host)!=0 else ""}{rank:0{numDigitsLastHostRank}d}' for rank,host in enumerate(metaData["Hosts"],start=0)]
- if domain: #Empty strings are falsy
+ if domain: #Empty strings are false
hosts=[host.replace(domain,"") for host in hosts]
# Determine String Y-Label Maximum String Length
@@ -934,11 +1019,15 @@ def pattern__report(pdf,
if correct_image_dimensions:
fixedPixelImage_shiftLabels(figure=figure,axis=TTS_axis)
# Add Colourbar
- cticks=list(range(1,numSteps,floor(numSteps/51).astype(int)))
- if (cticks[-1]-cticks[-2])/(cticks[-2]-cticks[-3])>0.1:
- cticks.append(numSteps)
+ num_cticks=floor(numSteps/51).astype(int)
+ cticks=list(range(1,numSteps,num_cticks if num_cticks>0 else 1))
+ if (numSteps>51):
+ if (cticks[-1]-cticks[-2])/(cticks[-2]-cticks[-3])>0.1:
+ cticks.append(numSteps)
+ else:
+ cticks[-1]=numSteps;
else:
- cticks[-1]=numSteps;
+ cticks.append(numSteps);
colorbar_axis=figure.add_axes([TTS_axis.get_position().x1+0.01,TTS_axis.get_position().y0,0.02,TTS_axis.get_position().height])
colorbar_axis.tick_params(labelsize=6)
colorbar_axis.invert_yaxis()
@@ -952,8 +1041,8 @@ def pattern__report(pdf,
# Draw TTS Title
figure.text(x=TST_title_position_norm[0],y=TST_title_position_norm[1],s="Task-Step-Timings Plot",fontsize=title_font_size,verticalalignment="top")
# Sort Data
- TST=empty([numHosts,numSteps]);
- TST[:]=nan;
+ TST=empty([numHosts,numSteps])
+ TST[:]=nan
for i in range(numHosts):
for j in range(numHosts):
if timingData["Access Pattern"][i,j].astype(int)==0:
@@ -996,7 +1085,7 @@ def pattern__report(pdf,
include_Page_Number(pageNumberInfo,figure)
# Save Report
- pdf.savefig(figure)
+ pdf.savefig(figure,transparent=True)
if verbose:
time2=time();print(f"Saving pattern report took {time2-time1:.3f} s.");
@@ -1010,7 +1099,7 @@ def empty____report(pdf,pageNumberInfo):
# Page Number
include_Page_Number(pageNumberInfo,figure)
# Save Report
- pdf.savefig(figure)
+ pdf.savefig(figure,transparent=True)
def linktest_report(metaData,
timingData,
@@ -1025,15 +1114,17 @@ def linktest_report(metaData,
histogram_colormap=["gist_rainbow_r"],
correct_image_dimensions=False,
replace_slow=False,
- two_page=False
+ two_page=False,
+ average_according_to_hostname=False,
+ aggregate_bandwidth_according_to_hostname=False
):
if verbose:
time1=time();
# Check version
- if metaData["Version"]!="2.0.0" and metaData["Version"]!="2.1.1" and metaData["Version"]!="2.1.2" and metaData["Version"]!="2.1.4" and metaData["Version"]!="2.1.5" and metaData["Version"]!="2.1.6" and metaData["Version"]!="2.1.7" and metaData["Version"]!="2.1.8" and metaData["Version"]!="2.1.9" and metaData["Version"]!="2.1.10" and metaData["Version"]!="2.1.11" and metaData["Version"]!="2.1.12" and metaData["Version"]!="2.1.13" and metaData["Version"]!="2.1.14" and metaData["Version"]!="2.1.15" and metaData["Version"]!="2.1.16" and metaData["Version"]!="2.1.17":
- print(f"Potentially unsupported LinkTest version output detected.\nThe Supported LinkTest versions are 2.0.0 and 2.1.[1,2,4-16] while this file comes from {metaData['Version']}")
+ if metaData["Version"]!="2.0.0" and metaData["Version"]!="2.1.1" and metaData["Version"]!="2.1.2" and metaData["Version"]!="2.1.4" and metaData["Version"]!="2.1.5" and metaData["Version"]!="2.1.6" and metaData["Version"]!="2.1.7" and metaData["Version"]!="2.1.8" and metaData["Version"]!="2.1.9" and metaData["Version"]!="2.1.10" and metaData["Version"]!="2.1.11" and metaData["Version"]!="2.1.12" and metaData["Version"]!="2.1.13" and metaData["Version"]!="2.1.14" and metaData["Version"]!="2.1.15" and metaData["Version"]!="2.1.16" and metaData["Version"]!="2.1.17" and metaData["Version"]!="2.1.18" and metaData["Version"]!="2.1.19":
+ print(f"Potentially unsupported LinkTest version output detected.\nThe Supported LinkTest versions are 2.0.0 and 2.1.[1,2,4-19] while this file comes from {metaData['Version']}")
if "Do a bisection test?" in metaData and metaData["Do a bisection test?"]!=0 and (metaData["Version"]=="2.0.0" or metaData["Version"]=="2.1.1" or metaData["Version"]=="2.1.2" or metaData["Version"]=="2.1.4" or metaData["Version"]=="2.1.5" or metaData["Version"]=="2.1.6" or metaData["Version"]=="2.1.7" or metaData["Version"]=="2.1.8" or metaData["Version"]=="2.1.9"):
print("Bisection runs were not properly executed prior to LinkTest version 2.1.10. The Report may be faulty.");
@@ -1041,7 +1132,7 @@ def linktest_report(metaData,
with PdfPages(output_report_filename) as pdf:
if isinstance(timingData, list): #If multiple iterations TimingData is a list
# Generate a one/double-sided page report summarizing the results
- tData=timingData[0].copy();
+ tData=timingData[0].copy()
tData['Timings']=sum(data['Timings'] for data in timingData)/len(timingData)
tData['Minimum time']=min(data['Minimum time'] for data in timingData)
tData['Average time']=sum(data['Average time'] for data in timingData)/len(timingData)
@@ -1068,7 +1159,9 @@ def linktest_report(metaData,
False,
correct_image_dimensions,
pageNumberInfo,
- aggregated=True
+ aggregated=True,
+ average_according_to_hostname=average_according_to_hostname,
+ aggregate_bandwidth_according_to_hostname=aggregate_bandwidth_according_to_hostname
);
pageNumberInfo[0]+=1;
if two_page:
@@ -1093,7 +1186,10 @@ def linktest_report(metaData,
histogram_colormap,
replace_slow,
correct_image_dimensions,
- pageNumberInfo
+ pageNumberInfo,
+ aggregated=False,
+ average_according_to_hostname=average_according_to_hostname,
+ aggregate_bandwidth_according_to_hostname=aggregate_bandwidth_according_to_hostname
);
pageNumberInfo[0]+=1;
if two_page:
@@ -1137,7 +1233,10 @@ def linktest_report(metaData,
histogram_colormap,
replace_slow,
correct_image_dimensions,
- pageNumberInfo
+ pageNumberInfo,
+ aggregated=False,
+ average_according_to_hostname=average_according_to_hostname,
+ aggregate_bandwidth_according_to_hostname=aggregate_bandwidth_according_to_hostname
);
if two_page:
pageNumberInfo[0]+=1;
@@ -1158,18 +1257,18 @@ def linktest_report(metaData,
);
if verbose:
print(f"Generating report took {time()-time0:.3f} s.");
- # Set Metainfo
+ # Set Metadata Information
d=pdf.infodict()
d['Title' ] = title_string
d['Author' ] = f"LinkTest Python Report Tool - {__version__}, Forschungszentrum Jülich GmbH"
d['Subject' ] = "LinkTest Report"
d['Keywords' ] = "LinkTest"
- d['CreationDate'] = datetime.datetime.today()
- d['ModDate' ] = datetime.datetime.today()
+ d['CreationDate'] = datetime.today()
+ d['ModDate' ] = datetime.today()
def main():
parser=ArgumentParser(prog="linktest_report",
- description="LinkTest Python Report Tool Version {version}.\n\nThis function generates a MatPlotLib based report on A4 paper which includes an indexed-colour image of the LinkTest timing data, a histogram of said data, a textual overview of the metadata and the serially retested timings for at most the five worst connections.\n\nThis script has been tested to work with Python version 3.8.5. Other versions may work as well.".format(version=__version__),
+ description=f"LinkTest Python Report Tool Version {__version__}.\n\nThis function generates a MatPlotLib based report on A4 paper which includes an indexed-colour image of the LinkTest timing data, a histogram of said data, a textual overview of the metadata and the serially retested timings for at most the five worst connections.\n\nThis script has been tested to work with Python version 3.8.5. Other versions may work as well.",
epilog="Provided as is by the Forschungszentrum Jülich GmbH. Copyright 2008-2022.\nPlease see the license under which this software is provided in the file LICENSE.\n This file was enclosed with the distribution of this software."
)
parser.add_argument("-i",
@@ -1268,6 +1367,18 @@ def main():
default=False,
help="If set generates a second page showing in which step certain times were measured and the times aggregated according to the step."
)
+ parser.add_argument("--average-according-to-hostname",
+ required=False,
+ action='store_true',
+ default=False,
+ help="If set causes timings to be averaged on a per-hostname basis. The resultant timing information is removed from the plots."
+ )
+ parser.add_argument("--aggregate-bandwidth-according-to-hostname",
+ required=False,
+ action='store_true',
+ default=False,
+ help="If set causes bandwidths to be aggregated on a per-hostname basis. The resultant timing information is removed from the plots."
+ )
# Parse Input Arguments
args=parser.parse_args()
@@ -1307,6 +1418,10 @@ def main():
if args.verbose:
print(f"Loading data took {time()-time0:.3f} s.");
+ if args.average_according_to_hostname and args.aggregate_bandwidth_according_to_hostname:
+ print(f"ERROR: --average-according-to-hostname and --aggregate-bandwidth-according-to-hostname cannot be used together")
+ exit(-1)
+
# Create Report
linktest_report(metaData=metaData,
timingData=timingData,
@@ -1321,7 +1436,9 @@ def main():
histogram_colormap=args.colormap_histogram[0],
correct_image_dimensions=args.correct_image_dimensions,
replace_slow=args.replace_slowest,
- two_page=args.two_page
+ two_page=args.two_page,
+ average_according_to_hostname=args.average_according_to_hostname,
+ aggregate_bandwidth_according_to_hostname=args.aggregate_bandwidth_according_to_hostname
)
if args.verbose:
print(f"DONE! Total time required: {time()-time0:.3f} s.");
diff --git a/python/setup.py b/python/setup.py
index 86c9dd677c188b6199a1e92792f1dc524c366f82..b201a691585b9dddb9a8209de0ba718e1f0a6885 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -12,7 +12,7 @@ import codecs
from setuptools import setup, Extension
# Hard-coded LinkTest version for older pip installers
-link_ver="2.1.17"
+link_ver="2.1.19"
# Hard-coded LinkTest git hash for older pip installers
link_hash="UNKNOWN"
link_hash_short="UNKNOWN"
@@ -83,7 +83,9 @@ ext_modules=[
"linktest/linktest_2_1_14.c",
"linktest/linktest_2_1_15.c",
"linktest/linktest_2_1_16.c",
- "linktest/linktest_2_1_17.c",],
+ "linktest/linktest_2_1_17.c",
+ "linktest/linktest_2_1_18.c",
+ "linktest/linktest_2_1_19.c"],
include_dirs=['linktest'],
extra_compile_args = sionlib_compile_args, #Add `+['-DDEBUG_OUTPUT']` to turn on debug output
extra_link_args = sionlib_link_args