Added files from master without history

.gitignore

+*.o
+*.cubin
+*.dat
+*.sion
+*.egg-info
+install/
+**.ipynb_checkpoints/

COPYRIGHT

+                  LinkTest - A multi-API parallel pingpong test 
+
+   Copyright (C) 2022, Dr. Wolfgang Frings, Dr. Dorian Krause, Yannik Müller
+   and Dr. Max Enno Holicki, Forschungszentrum Juelich GmbH, Federal Republic of
+   Germany. All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are met:
+
+     - Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+     - Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+     - Any publications that result from the use of this software shall
+       reasonably refer to the Research Centre's development.
+
+     - All advertising materials mentioning features or use of this software
+       must display the following acknowledgement:
+
+           This product includes software developed by Forschungszentrum
+           Juelich GmbH, Federal Republic of Germany.
+
+     - Forschungszentrum Juelich GmbH is not obligated to provide the user with
+       any support, consulting, training or assistance of any kind with regard
+       to the use, operation and performance of this software or to provide
+       the user with any updates, revisions or new versions.
+
+
+   THIS SOFTWARE IS PROVIDED BY FORSCHUNGSZENTRUM JUELICH GMBH "AS IS" AND ANY
+   EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+   DISCLAIMED. IN NO EVENT SHALL FORSCHUNGSZENTRUM JUELICH GMBH BE LIABLE FOR
+   ANY SPECIAL, DIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+   RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+   CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+   CONNECTION WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.

ImHex_Patterns/LinkTest_2-1-16_SIONlib_1-7-6.hexpat

+#include <std/sys.pat>
+#include <std/io.pat>
+#include <std/cint.pat>
+
+// Valid for LinkTest Version 2.1.16
+// Valid for SIONlib  Version 1.7.6
+//           SIONlib Bug: 4-byte endian integer not set
+//                        If you need to change the endianness search for "ce=false; //See SIONlib bug above" and set it to true
+
+// ImHex Bugs:
+//     1: Attributes applied to an array do not inherit to its elements.
+//        This makes conversion of the displayed values in case of endinaness changes impossible.
+
+// TODO: Currently only works if there is only one SION block in total and one SION chunk per task.
+
+std::size_t offset;
+std::size_t numIter;
+bool ce; //If true endianness needs to be converted
+
+fn Change_Endianness_u32(u32 val){
+	return ((val>>24)&0xff)|((val<<8)&0xff0000)|((val>>8)&0xff00)|((val<<24)&0xff000000);
+};
+
+fn Change_Endianness_u64(u64 val){
+	return (((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32);
+};
+
+fn Change_Endianness_double(double val){
+	return double((((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32));
+};
+
+fn Format_Change_Endianness_u32(u32 val){
+	return std::format("{} ({:#04x})",((val>>24)&0xff)|((val<<8)&0xff0000)|((val>>8)&0xff00)|((val<<24)&0xff000000),((val>>24)&0xff)|((val<<8)&0xff0000)|((val>>8)&0xff00)|((val<<24)&0xff000000));
+};
+
+fn Format_Change_Endianness_u64(u64 val){
+	return std::format("{} ({:#04x})",(((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32),(((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((val << 8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((val<<8)&0xFF00FF00FF00FF00)|((val>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32));
+};
+
+fn Format_Change_Endianness_double(double val){
+	return std::format("{} ({:#04x})",double((((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32)),double((((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)<<32)|(((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)<<16)&0xFFFF0000FFFF0000)|((((u64(val)<<8)&0xFF00FF00FF00FF00)|((u64(val)>>8)&0x00FF00FF00FF00FF)>>16)&0x0000FFFF0000FFFF)>>32)));
+};
+
+struct SION_Header_{
+	if(ce){
+		u32  Version [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  Patchlevel [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  File_version [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  Block_Size [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  Num_Tasks;
+		u32  Num_Files [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  File_Num [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+	}else{
+		u32  Version;
+		u32  Patchlevel;
+		u32  File_version;
+		u32  Block_Size;
+		u32  Num_Tasks;
+		u32  Num_Files;
+		u32  File_Num;
+	}
+	padding[16];
+	char Filename[1024];
+	if(ce){
+		u64  GlobalRank[Num_Tasks] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Chunk_Size[Num_Tasks] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u32  Max_Chunks [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u64  Var_Start [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+	}else{
+		u64  GlobalRank[Num_Tasks];
+		u64  Chunk_Size[Num_Tasks];
+		u32  Max_Chunks;
+		u64  Var_Start;
+	}
+	if(Block_Size>1){
+		padding[Block_Size-$];
+	}
+};
+
+struct LinkTest_Header_{
+	if(ce){
+		u32  Major_Version [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  Minor_Version [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+		u32  Patchlevel [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+	}else{
+		u32  Major_Version;
+		u32  Minor_Version;
+		u32  Patchlevel;
+	}
+	char Git_Hash[41];
+	if(ce){
+		u32  Mode_Length [[hidden,transform("Change_Endianness_u32")]];
+	}else{
+		u32  Mode_Length [[hidden]];
+	}
+	char Mode[Mode_Length];
+	u8   A2A_Flag;
+	u8   Bidirectional_Flag;
+	u8   Unidirectional_Flag;
+	u8   Bisection_Flag;
+	u8   Step_Randomization_Flag;
+	u8   Serialize_Testing_Flag;
+	u8   No_SION_File_Flag;
+	u8   Parallel_SION_File_Flag;
+	u8   GPU_Memory_Flag;
+	u8   Multi_Buffer_Flag;
+	u8   Randomized_Buffer_Flag;
+	u8   Check_Buffer_Flag;
+	u8   Memory_Allocator_Type;
+	if(ce){
+		u64  Num_Messages [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Size_Messages [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Num_WarmUp_Messages [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+	}else{
+		u64  Num_Messages;
+		u64  Size_Messages;
+		u64  Num_WarmUp_Messages;
+	}
+	padding[8];
+	if(ce){
+		u64  Num_Serial_Retests [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Num_Multi_Buffer [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Buffer_Randomization_Seed [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Num_Randomized_Task [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		u64  Task_Randomization_Seed [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+	}else{
+		u64  Num_Serial_Retests;
+		u64  Num_Multi_Buffer;
+		u64  Buffer_Randomization_Seed;
+		u64  Num_Randomized_Task;
+		u64  Task_Randomization_Seed;
+	}
+};
+
+struct Host_{
+	if(rank!=0){
+		char id[5] [[hidden]];
+		std::assert_warn(id=="LKTST","Failed to match LKTST!");
+	}
+	if(ce){
+		u32  Hostname_Length [[hidden,transform("Change_Endianness_u32")]];
+	}else{
+		u32  Hostname_Length [[hidden]];
+	}
+	char Hostname[Hostname_Length];
+	if(ce){
+		u32  Core_ID [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+	}else{
+		u32  Core_ID;
+	}
+};
+
+struct IterationData_{
+	if(rank==0){
+		char   Start_Time[32];
+		if(ce){
+			double Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			double Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			double Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			if(LinkTest_Header.A2A_Flag){
+				double A2A_Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+				double A2A_Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+				double A2A_Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			}
+		}else{
+			double Minimum_Time;
+			double Average_Time;
+			double Maximum_Time;
+			if(LinkTest_Header.A2A_Flag){
+				double A2A_Minimum_Time;
+				double A2A_Average_Time;
+				double A2A_Maximum_Time;
+			}
+		}
+	}
+	if(ce){
+		double Timings[SION_Header.Num_Tasks] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+		u64    AccessPattern[SION_Header.Num_Tasks] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		if(LinkTest_Header.A2A_Flag){
+			double A2A_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+		}
+	}else{
+		double Timings[SION_Header.Num_Tasks];
+		u64    AccessPattern[SION_Header.Num_Tasks];
+		if(LinkTest_Header.A2A_Flag){
+			double A2A_Time;
+		}
+	}
+	if(rank==0){
+		if(ce){
+			double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			u64    Retests_From[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+			u64    Retests_To[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+			char   End_Time[32];
+		}else{
+			double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+			double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+			u64    Retests_From[LinkTest_Header.Num_Serial_Retests];
+			u64    Retests_To[LinkTest_Header.Num_Serial_Retests];
+			char   End_Time[32];
+		}
+	}
+};
+
+struct Data_{
+	Host_          Host;
+	IterationData_ Data[numIter];
+	char           END_BLOCK[10] [[hidden]];
+	rank=rank+1;
+};
+	
+char SION_String[4] @ 0x0 [[hidden]];
+std::assert(SION_String=="sion","File does not start with sion!");
+u32  Endian_Test_1  @ $ [[hidden]];
+offset=$;
+//std::assert(Endian_Test_1==u32(1),"File does not start with sion!");
+if(Endian_Test_1==1){
+	little=false;
+}else if(Endian_Test_1==16777216){
+	little=true;
+}else{
+	//TODO: Find a more elegant solution to
+	//      generate an error.
+//	std::assert(Endian_Test_1==u32(1),"File neither little- nor big-endian!");
+}
+ce=false; //See SIONlib bug above
+$=offset;
+SION_Header_     SION_Header     @ $ [[comment("SIONlib File Header"), name("SIONlib Header")]];
+offset=$;
+std::assert(SION_Header.Version==1007,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Patchlevel==6,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Num_Files==1,"No support for multi-file SIONlib data!");
+std::assert(SION_Header.Num_Files==1,"No support for multiple SION chunks!");
+$=offset;
+char             ID_String[5]    @ $ [[hidden]];
+offset=$;
+std::assert_warn(ID_String=="LKTST","Failed to match LKTST!");
+$=offset;
+LinkTest_Header_ LinkTest_Header @ $ [[comment("LinkTest File Header"), name("LinkTest Header")]];
+offset=$;
+std::assert_warn(LinkTest_Header.No_SION_File_Flag==0,"No-SION-File Flag set!");
+if(LinkTest_Header.Num_Randomized_Task!=0){
+	numIter=LinkTest_Header.Num_Randomized_Task;
+}else{
+	numIter=1;
+}
+u64 rank;
+$=offset;
+Data_ Data[SION_Header.Num_Tasks] @ $;
+

ImHex_Patterns/LinkTest_2-1-17_SIONlib_1-7-6.hexpat

+#include <std/sys.pat>
+#include <std/io.pat>
+#include <std/cint.pat>
+
+// Valid for LinkTest Version 2.1.17
+// Valid for SIONlib  Version 1.7.6 or 1.7.7
+//           SIONlib bug: 1.7.7 forgot to increment the patch level saved to the file
+
+// ImHex Bugs:
+//     1: Attributes applied to an array do not inherit to its elements.
+//        This makes conversion of the displayed values in case of endinaness changes impossible.
+
+// TODO: Currently only works if there is only one SION block in total and one SION chunk per task.
+
+std::size_t offset;
+std::size_t numIter;
+bool bes; //If true SION headers are big endian
+bool bel; //If true LinkTest data is big-endian
+
+struct SION_Header_{
+	if(bes){
+		be u32 Version;
+		be u32 Patchlevel;
+		be u32 File_version;
+		be u32 Block_Size;
+		be u32 Num_Tasks;
+		be u32 Num_Files;
+		be u32 File_Num;
+	}else{
+		le u32 Version;
+		le u32 Patchlevel;
+		le u32 File_version;
+		le u32 Block_Size;
+		le u32 Num_Tasks;
+		le u32 Num_Files;
+		le u32 File_Num;
+	}
+	padding[16];
+	char Filename[1024];
+	if(bes){
+		be u64 GlobalRank[Num_Tasks];
+		be u64 Chunk_Size[Num_Tasks];
+		be u32 Max_Chunks;
+		be u64 Var_Start;
+	}else{
+		le u64 GlobalRank[Num_Tasks];
+		le u64 Chunk_Size[Num_Tasks];
+		le u32 Max_Chunks;
+		le u64 Var_Start;
+	}
+	if(Block_Size>1){
+		padding[Block_Size-$];
+	}
+};
+
+struct LinkTest_Header_{
+	if(bel){
+		be u32 Major_Version;
+		be u32 Minor_Version;
+		be u32 Patchlevel;
+	}else{
+		le u32 Major_Version;
+		le u32 Minor_Version;
+		le u32 Patchlevel;
+	}
+	char Git_Hash[41];
+	if(bel){
+		be u32 Mode_Length [[hidden]];
+	}else{
+		le u32 Mode_Length [[hidden]];
+	}
+	char Mode[Mode_Length];
+	u8   A2A_Flag;
+	u8   Bidirectional_Flag;
+	u8   Unidirectional_Flag;
+	u8   Bisection_Flag;
+	u8   Step_Randomization_Flag;
+	u8   Serialize_Testing_Flag;
+	u8   No_SION_File_Flag;
+	u8   Parallel_SION_File_Flag;
+	u8   GPU_Memory_Flag;
+	u8   Multi_Buffer_Flag;
+	u8   Randomized_Buffer_Flag;
+	u8   Check_Buffer_Flag;
+	u8   Memory_Allocator_Type;
+	if(bel){
+		be u64 Num_Messages;
+		be u64 Size_Messages;
+		be u64 Num_WarmUp_Messages;
+		be u64 Num_Serial_Retests;
+		be u64 Num_Multi_Buffer;
+		be u64 Buffer_Randomization_Seed;
+		be u64 Num_Randomized_Task;
+		be u64 Task_Randomization_Seed;
+	}else{
+		le u64 Num_Messages;
+		le u64 Size_Messages;
+		le u64 Num_WarmUp_Messages;
+		le u64 Num_Serial_Retests;
+		le u64 Num_Multi_Buffer;
+		le u64 Buffer_Randomization_Seed;
+		le u64 Num_Randomized_Task;
+		le u64 Task_Randomization_Seed;
+	}
+	char END_HEADER[10] [[hidden]];
+	std::assert(END_HEADER=="END_HEADER","LinkTest header did not end with END_HEADER");
+};
+
+struct Host_{
+	if(bel){
+		be u32 Hostname_Length [[hidden,transform("Change_Endianness_u32")]];
+	}else{
+		le u32 Hostname_Length [[hidden]];
+	}
+	char Hostname[Hostname_Length];
+	if(bel){
+		be u32 Core_ID [[format("Format_Change_Endianness_u32"),transform("Change_Endianness_u32")]];
+	}else{
+		le u32 Core_ID;
+	}
+};
+
+struct IterationData_{
+	if(rank==0){
+		char Start_Time[32];
+		if(bel){
+			be double Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			be double Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			be double Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			if(LinkTest_Header.A2A_Flag){
+				be double A2A_Minimum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+				be double A2A_Average_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+				be double A2A_Maximum_Time [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			}
+			be double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			be double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_double"),transform("Change_Endianness_double")]];
+			be u64    Retests_From[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+			be u64    Retests_To[LinkTest_Header.Num_Serial_Retests] [[format("Format_Change_Endianness_u64"),transform("Change_Endianness_u64")]];
+		}else{
+			le double Minimum_Time;
+			le double Average_Time;
+			le double Maximum_Time;
+			if(LinkTest_Header.A2A_Flag){
+				le double A2A_Minimum_Time;
+				le double A2A_Average_Time;
+				le double A2A_Maximum_Time;
+			}
+			le double New_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+			le double Old_Retested_Timings[LinkTest_Header.Num_Serial_Retests];
+			le u64    Retests_From[LinkTest_Header.Num_Serial_Retests];
+			le u64    Retests_To[LinkTest_Header.Num_Serial_Retests];
+		}
+		char End_Time[32];
+	}
+	if(bel){
+		be double Timings[SION_Header.Num_Tasks];
+		be u64    AccessPattern[SION_Header.Num_Tasks];
+		if(LinkTest_Header.A2A_Flag){
+			be double A2A_Time;
+		}
+	}else{
+		le double Timings[SION_Header.Num_Tasks];
+		le u64    AccessPattern[SION_Header.Num_Tasks];
+		if(LinkTest_Header.A2A_Flag){
+			le double A2A_Time;
+		}
+	}
+};
+
+struct Data_{
+	Host_          Host;
+	IterationData_ Data[numIter];
+	char           END_BLOCK[9] [[hidden]];
+	std::assert(END_BLOCK=="END_BLOCK","LinkTest per-rank data did not end with END_BLOCK");
+	rank=rank+1;
+};
+	
+char SION_String[4] @ 0x0 [[hidden]];
+std::assert(SION_String=="sion","File does not start with sion!");
+u8 Endianness_Flags[4] @ $ [[hidden]];
+offset=$;
+std::assert(Endianness_Flags[0]==Endianness_Flags[3],"Endianness Flag not for SIONlib data not symmetric");
+if(Endianness_Flags[0]==1){
+	bes=true;
+}else{
+	bes=false;
+}
+std::assert(Endianness_Flags[1]==Endianness_Flags[2],"Endianness Flag not for LinkTest data not symmetric");
+if(Endianness_Flags[1]==1){
+	bel=true;
+}else{
+	bel=false;
+}
+$=offset;
+SION_Header_     SION_Header     @ $ [[comment("SIONlib File Header"), name("SIONlib Header")]];
+offset=$;
+std::assert(SION_Header.Version==1007,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Patchlevel==6,"SION file not from SIONlib version 1.7.6 or 1.7.7!");
+std::assert(SION_Header.Num_Files==1,"No support for multi-file SIONlib data!");
+std::assert(SION_Header.Num_Files==1,"No support for multiple SION chunks!");
+$=offset;
+char             ID_String[8]    @ $ [[hidden]];
+offset=$;
+std::assert_warn(ID_String=="LinkTest","Failed to match LinkTest!");
+$=offset;
+LinkTest_Header_ LinkTest_Header @ $ [[comment("LinkTest File Header"), name("LinkTest Header")]];
+offset=$;
+std::assert_warn(LinkTest_Header.No_SION_File_Flag==0,"No-SION-File Flag set!");
+if(LinkTest_Header.Num_Randomized_Task!=0){
+	numIter=LinkTest_Header.Num_Randomized_Task;
+}else{
+	numIter=1;
+}
+u64 rank;
+$=offset;
+Data_ Data[SION_Header.Num_Tasks] @ $;
+

README.md

+# LinkTest
+LinkTest performs parallel end-to-end bandwidth tests.
+Additionally to MPI it supports multiple other transport layers.
+Single- and multi-node tests can be performed, depending on the transport layer.
+Results are written through SIONlib, a highly scalable IO library, thus a large number of processes can be tested.
+
+## Wiki
+Our wiki contains further information including details of how to build, run, and analyze with LinkTest  
+https://gitlab.jsc.fz-juelich.de/cstao-public/linktest/-/wikis/home
+
+## Code Structure
+The LinkTest repository consists of 4 parts
+1. benchmark/
+	- Performs a parallel all-to-all performance test and produces results in form of stdout and an optional binary file in sion format
+	- Makefile produces the `linktest` executable
+2. python/
+	- Python-based analysis tool. Visualizes data from the SION files as pdf
+	- python -m pip installs the `linktest-report` package
+3. example/
+	- Python notebook showcasing how to write a custom (advanced) analysis based on the python API
+
+## Copyright
+Copyright (c) 2008-2022
+Dr. Wolfgang Frings, Dr. Dorian Krause, Yannik Müller & Dr. Max Holicki
+Forschungszentrum Juelich GmbH - Juelich Supercomputing Centre
+
+Read the copyright-terms in COPYRIGHT before usage or distribution.
+
+## Quickstart
+For the quickstart Examples, you will need: 
+* A C++ compiler
+* An MPI Installation
+* SIONlib: http://fz-juelich.de/ias/jsc/sionlib
+* Python 3.8.5
+    * Including the NumPy and MatPlotLib packages
+
+### Build procedure
+In general, you have to do the following steps
+1. Install Dependencies
+2. Set-Up Environment, Paths etc.
+3. Install `linktest` with `make`, enable transport layers other than mpi
+4. [Optional] Install `linktest-report` via pip (Update PYTHONPATH or use virtual environments)
+
+Quick-Start Example installing LinkTest with only MPI support. (Might not work on your system)
+```bash
+mkdir -p install
+cd benchmark
+ml GCC ParaStationMPI SIONlib SciPy-Stack                     # 1 + 2 
+make PREFIX=../install clean install                          # 3
+python -m pip install ../python/setup.py --prefix=../install  # 4
+```
+
+### Full Example
+The example scripts are written for the JUWELS System.
+They assume MPI, TCP, Infiniband and UCX are available.
+They also uses specific slurm accounts in order to work out of the box.
+You likely want to change those.
+To work through the example do the following:  
+
+1. Build `linktest` executable and `linktest-report`: `./exampleBuild.sh`.
+2. Execute LinkTest in parallel with an even number of processes: `./exampleRun.sh`.
+
+A file named `pingpong_results_bin.sion` will be produced, and a text based report will be printed on stdout. 
+
+3. Analyse the produced SION file with `linktest-report`: `./exampleAnalysis.sh`.
+
+A file named `report.pdf` should be produced.
+This report shows the communication matrix (ntasks x ntasks), a histogram of the connection bandwidth, the parameters that were active during the run, and the results of the retest of the slowest connections.

benchmark/.gitignore

+cuda_kernels.cc

benchmark/Makefile

+#/****************************************************************************
+#**  fzjlinktest                                                            **
+#*****************************************************************************
+#**  Copyright (c) 2008-2022                                                **
+#**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+#**                                                                         **
+#**  See the file COPYRIGHT in the package base directory for details       **
+#****************************************************************************/
+PREFIX    = /usr/local/bin
+
+USE_POSIX                 = 1
+
+HAVE_SION                 = 1
+HAVE_MPI                  = 1
+HAVE_MINIPMI              = 0
+HAVE_TCP                  = 1
+HAVE_IBVERBS              = 0
+HAVE_PSM2                 = 0
+HAVE_CUDA                 = 0
+HAVE_UCP                  = 0
+
+FSANITIZE = address
+
+ifeq (1, $(HAVE_IBVERBS))
+	HAVE_MINIPMI = 1
+	HAVE_TCP     = 1
+endif
+ifeq (1, $(HAVE_PSM2))
+	HAVE_MINIPMI = 1
+	HAVE_TCP     = 1
+endif
+ifeq (1, $(HAVE_CUDA))
+	HAVE_MINIPMI = 1
+	HAVE_TCP     = 1
+endif
+ifeq (1, $(HAVE_UCP))
+	HAVE_MINIPMI = 1
+	HAVE_TCP     = 1
+endif
+
+ifdef V
+$(info USE_POSIX    = $(USE_POSIX))
+$(info HAVE_SION    = $(HAVE_SION))
+$(info HAVE_MPI     = $(HAVE_MPI))
+$(info HAVE_MINIPMI = $(HAVE_MINIPMI))
+$(info HAVE_TCP     = $(HAVE_TCP))
+$(info HAVE_IBVERBS = $(HAVE_IBVERBS))
+$(info HAVE_PSM2    = $(HAVE_PSM2))
+$(info HAVE_CUDA    = $(HAVE_CUDA))
+$(info HAVE_UCP     = $(HAVE_UCP))
+endif
+
+SYSTEM   = generic
+GIT_HASH = $(shell git rev-parse --verify HEAD)
+GIT_HASH_SHORT= $(shell git rev-parse --verify --short HEAD)
+CC       = mpicxx
+CFLAGS   = -std=c++17 -Wall
+CPPFLAGS = -D_GNU_SOURCE -DLINKTEST_LINUX=1 -DLINKTEST_SYSTEM="\"$(SYSTEM)\"" -DGIT_HASH=\"$(GIT_HASH)\" -DGIT_HASH_SHORT=\"$(GIT_HASH_SHORT)\"
+LD       = $(CC)
+LDFLAGS  = 
+LIBS     =
+
+# Use POSIX
+ifeq (1, ${USE_POSIX})
+	CPPFLAGS += -D__USE_POSIX
+endif
+
+# SIONlib Options
+ifeq (1, $(HAVE_SION))
+#	CFLAGS   +=
+	CPPFLAGS += -D_FILE_OFFSET_BITS=64 -DUSE_SION=1 $(shell sionconfig --64 --gcc --cflags --mpi)
+#	LDFLAGS  +=
+	LIBS     += $(shell sionconfig --64 --gcc --libs --mpi)
+endif
+
+# MINIPMI Options
+ifeq (1, $(HAVE_MINIPMI))
+#	CFLAGS   +=
+	CPPFLAGS += -Iminipmi -DHAVE_MINIPMI=1
+	LDFLAGS  += -Lminipmi
+	LIBS     += -lminipmi
+endif
+
+# UCP Options
+ifeq (1, $(HAVE_UCP))
+#	CFLAGS   +=
+#	CPPFLAGS +=
+#	LDFLAGS  +=
+#	LIBS     +=
+endif
+
+# CUDA Options
+ifeq (1, $(HAVE_CUDA))
+	CU       = nvcc
+	CUARCH   = 
+	ifeq (, $(CUARCH))
+$(error CUARCH is not set)
+	endif
+	CUFLAGS  = --gpu-architecture $(CUARCH)
+#	CFLAGS   +=
+	CPPFLAGS += -I$(CUDA)/include -DHAVE_CUDA=1
+	LDFLAGS  += -L$(CUDA)/lib
+	LIBS     += -lcuda -lcudart
+endif
+
+linktest-versions = 
+ifeq (1, $(HAVE_MPI))
+	linktest-versions += linktest.mpi
+endif
+ifeq (1, $(HAVE_TCP))
+	linktest-versions += linktest.tcp
+endif
+ifeq (1, $(HAVE_MINIPMI))
+	ifeq (1, $(HAVE_IBVERBS))
+		linktest-versions += linktest.ibverbs
+	endif
+	ifeq (1, $(HAVE_PSM2))
+		linktest-versions += linktest.psm2
+	endif
+	ifeq (1, $(HAVE_UCP))
+		linktest-versions += linktest.ucp
+	endif
+	ifeq (1, $(HAVE_CUDA))
+		linktest-versions += linktest.cuda
+	endif
+endif
+
+ifdef V
+$(info linktest-versions = $(linktest-versions))
+endif
+
+linktest-obj = linktest.o \
+               system.o \
+               benchmark.o \
+               cmdline.o \
+               environ.o \
+               error.o \
+               memusage.o \
+               output_sion.o \
+               serializer.o \
+               slow_pairs.o \
+               stats.o \
+               stopwatch.o \
+               timing.o \
+               utils.o \
+               memory.o \
+               memory_multi.o \
+               pmi.o \
+               vcluster.o \
+               vcluster_helper.o \
+               format_units.o
+
+ifeq (1, $(HAVE_MPI))
+	linktest-obj += vcluster_mpi.o
+	CFLAGS       += -DHAVE_VCLUSTER_MPI=1
+endif
+ifeq (1, $(HAVE_TCP))
+	linktest-obj += vcluster_tcp.o
+	CFLAGS       += -DHAVE_VCLUSTER_TCP=1
+endif
+ifeq (1, $(HAVE_MINIPMI))
+	ifeq (1, $(HAVE_IBVERBS))
+		linktest-obj += vcluster_ibverbs.o \
+		                ibverbs_mr.o \
+		                ibverbs_qp.o \
+		                ibverbs_cq.o \
+		                ibverbs_pd.o \
+		                ibverbs_ctx.o
+		CFLAGS       += -DHAVE_VCLUSTER_IBVERBS=1 -DIBVERBS_SEND_INLINE=1
+		LIBS         += -libverbs
+	endif
+	ifeq (1, $(HAVE_PSM2))
+		linktest-obj += vcluster_psm2.o
+		CFLAGS       += -DHAVE_VCLUSTER_PSM2=1
+		LIBS         += -lpsm2
+	endif
+	ifeq (1, $(HAVE_UCP))
+		linktest-obj += vcluster_ucp.o
+		CFLAGS       += -DHAVE_VCLUSTER_UCP=1
+		LIBS         += -lucp
+	endif
+	ifeq (1, $(HAVE_CUDA))
+		linktest-obj += vcluster_cuda.o \
+		                cuda_kernels.o \
+		                gpu_nvidia.o \
+		                memory_cuda.o
+		CFLAGS       += -DHAVE_VCLUSTER_CUDA=1
+	endif
+endif
+
+ifeq (1, $(HAVE_SION))
+	linktest-obj += vcluster_sion_generic_adapter.o
+endif
+
+ifdef V
+	Q =
+else
+	Q = @
+endif
+
+link = $(Q)ln -s linktest linktest.$(1)
+
+SYMB_EXE := $(shell find . -type l -iname "linktest.*")
+
+.PHONY: all
+all: optimized
+
+.PHONY: compile
+compile: linktest $(linktest-versions)
+
+.PHONY: optimized
+optimized: CFLAGS += -O3
+optimized: compile
+
+.PHONY: debug
+debug: CFLAGS += -O0 -g
+debug: compile
+
+.PHONY: sanitized
+sanitized: debug
+sanitized: CFLAGS   += -fsanitize=$(FSANITIZE) -static-libasan -fno-omit-frame-pointer
+sanitized: LDFLAGS  += -fsanitize=$(FSANITIZE) -static-libasan
+sanitized: compile
+
+memory_cuda.cc: cuda_kernels.cc
+
+%.o: %.cc
+	@echo " "CC $@
+	$(Q)$(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@	
+
+%.cubin: %.cu
+	@echo " "CU $@
+	$(Q)$(CU) $(CUFLAGS) --cubin $< -o $@
+
+%.cc: %.cubin convert.py
+	@echo " "CONVERT $@
+	$(Q)python3 convert.py $< $@ $(basename $@)
+
+linktest: $(linktest-obj)
+	@echo " "LD $@
+	$(Q)$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+linktest.tcp: linktest
+	@echo " "LN $@
+	$(call link,tcp)
+
+linktest.mpi: linktest
+	@echo " "LN $@
+	$(call link,mpi)
+
+linktest.ibverbs: linktest
+	@echo " "LN $@
+	$(call link,ibverbs)
+
+linktest.psm2: linktest
+	@echo " "LN $@
+	$(call link,psm2)
+
+linktest.ucp: linktest
+	@echo " "LN $@
+	$(call link,ucp)
+
+linktest.cuda: linktest
+	@echo " "LN $@
+	$(call link,cuda)
+
+.PHONY: clean
+clean:
+	rm -f *.o linktest $(SYMB_EXE)
+
+install: linktest $(linktest-versions)
+	@echo " "INSTALL $^
+	mkdir -p $(PREFIX)
+	for f in $^ ; do              \
+		cp -d $$f $(PREFIX)/$$f ; \
+	done

benchmark/benchmark.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_BENCHMARK_H
+#define LINKTEST_BENCHMARK_H
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+#include <random>
+#include "stats.h"
+#include "slow_pairs.h"
+
+#if HAVE_CUDA == 1
+    #include "gpu_nvidia.h"
+#endif
+
+#ifndef _POSIX_HOST_NAME_MAX
+    #define _POSIX_HOST_NAME_MAX 4096 //Hopefully enough
+#endif
+
+class VirtualCluster;
+class Allocator;
+class MemoryBuffer;
+class MemoryBufferMulti;
+
+struct linktest_args;
+
+namespace linktest{
+    // Working set for the LinkTest
+    class Benchmark{
+        public:
+            Benchmark() = default;
+            Benchmark(const Benchmark&) = delete;
+            Benchmark(Benchmark&&) = delete;
+            int  main_cmdline();
+            int  benchmark(); // Run the main benchmark
+            [[nodiscard]] int  rank() const;
+            [[nodiscard]] int  size() const;
+            void barrier() const;
+            const struct linktest_args* args;
+            std::unique_ptr<VirtualCluster> cl;
+            #if HAVE_CUDA == 1
+                std::unique_ptr<cuda::GpuDevice>  gpudev;
+                std::unique_ptr<cuda::GpuContext> gpuctx; // Declaration order important! MemoryBuffer~ needs to be called before before GpuContext~
+            #endif
+            std::unique_ptr<Allocator>         alloc;
+            std::unique_ptr<MemoryBuffer>      buf1;
+            std::unique_ptr<MemoryBuffer>      buf2;        // Second buffer only for bidirectional bandwidth tests
+            std::unique_ptr<MemoryBuffer>      buf_all2all; //Much larger buffer for all-to-all testing
+            std::unique_ptr<MemoryBufferMulti> buf_multi;   //Pointer to array of pointers to buffers
+        private:
+            std::mt19937 randomNumberEngine;
+            LinktestStats* stats = nullptr;
+            std::vector<LinktestStats> newStatsVec;
+            std::vector<int>           stepPermutation; // permutation of the step numbers
+            std::vector<int>           getRankFromStep; // maps step to communication partner rank
+            std::vector<int>           getStepFromRank; // maps communication partner rank to step
+            [[nodiscard]] unsigned int getFrom(int i, int j, int step) const; // defines direction of communication
+            [[nodiscard]] unsigned int getTo(int i, int j, int step) const;   // defines direction of communication
+            [[nodiscard]] unsigned int getNumSteps() const;
+            int init(); // Initialize and validate settings
+            void prepareBuffers(); // Resize and fill all data Buffers that will be used.
+            void computeRankStepMappings();   // compute stepPermutation, getRankFromStep and getStepFromRank
+            std::vector<int> oneFactorAlgorithm(const int size, const int rank);
+            void mapWith1FactorAlgorithm(bool randomizeTasks);
+            void mapWithBisection();
+            int work_alltoall();
+            /**
+            * Executes semi-, bi- or unidirectional tests
+            */
+            int kernel(const int from, const int to, double* const avg_time_per_msg, const bool doBarrier); //Switch between different kernels
+            int work_pingpong_parallel(const int partner,const int sign,double* const avg_time_per_msg);
+            int work_pingpong_serial(const int partner,double* const avg_time_per_msg);
+            /**
+            * @param min_time minimum of all communication pairs of average times measured in the kernel
+            * @param max_time minimum of all communication pairs of average times measured in the kernel
+            * @param avg_time average over all communication pairs of average times measured in the kernel
+            * @param bandwidth bandwidth of all communication pairs together (equals bisection bandwidth if the cut according to ranks is a minimal cut)
+            */
+            int work_pingpong(const int step, double* const min_time, double* const max_time,double* const avg_time,double* const bandwidth);
+            int work(const int step);
+            int run_iteration(const std::size_t iter, const std::size_t numRemainingIterationsIterations, double* const twork, double* const tstep);
+            void run_Stresstest();
+            void run_Measuringtest();
+            void finalize_results();
+            [[nodiscard]] bool isLastIteration(const double time, const double tstep, const uint64_t finishedIterations) const;
+            [[nodiscard]] bool isStressTest() const;
+            [[nodiscard]] bool isParallelTest() const;
+            [[nodiscard]] bool randomizeTasks() const;
+            int handle_slow_pairs(const int iter);
+            int retest_slow_pairs(slow_pair* const sp,const int n,const int iter);
+            int retest_one_slow_pair(const int from,const int to, double* const time);
+            int gather_slow_pairs(struct slow_pair *sp,const int n);
+            int mapping_string(char* buf, int szofbuf);
+            int printStepResults(const std::size_t numStepsProcessed, const std::size_t numRemainingIterations, const double tWork,
+                                 const double minTimeForStep, const double avgTimeForStep, const double maxTimeForStep,
+                                 const double sumBandwidth, const double runningSumBandwidth, const unsigned int strLength);
+            int printIterationResults(const int iter);
+            int write_to_file();
+    };
+} // namespace linktest
+
+#endif

benchmark/cconfig.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_CCONFIG_H
+#define LINKTEST_CCONFIG_H
+
+// C Header needed since this config file is used for linktest-report 
+#include <string.h>
+#include <stdint.h>
+
+/* for nanosleep() */
+#ifndef _POSIX_C_SOURCE
+    #define _POSIX_C_SOURCE 199309L
+#endif
+
+/* for gethostname() */
+#ifndef _DEFAULT_SOURCE
+    #define _DEFAULT_SOURCE 1
+#endif
+
+#ifndef _POSIX_HOST_NAME_MAX
+    #define _POSIX_HOST_NAME_MAX 4096 //Hopefully enough
+#endif
+
+#define LINKTEST_VERSION 2
+#define LINKTEST_VERSION_SUB 1
+#define LINKTEST_VERSION_PATCHLEVEL 17
+#define GIT_HASH_LEN 41
+#ifndef GIT_HASH
+    #define GIT_HASH "UNKNOWN"
+#endif
+#define GIT_HASH_SHORT_LEN 8
+#ifndef GIT_HASH_SHORT
+    #define GIT_HASH_SHORT "UNKNOWN"
+#endif
+
+/* Common prefix for all LinkTest environment variables */
+#define LINKTEST_ENVIRON_PREFIX "LINKTEST"
+
+
+/* Linktest identification string for output files (offset: 0x00 for LinkTest data) */
+#define LINKTEST_ID "LinkTest"
+
+/* Linktest alignment string for output files. Marks the end of the Binary Header */
+#define END_HEADER "END_HEADER"
+
+/* Linktest alignment string for output files. Marks end of thread output */
+#define END_BLOCK "END_BLOCK"
+
+/* Seven dimensions is the maximum dimensionality as of this
+ * writing (2014-05-17). The K computer uses a 6-D torus,
+ * BG/Q uses a 5-D torus. Considering the core affinity/thread
+ * affinity as an additional dimension we end up with 7 or
+ * 6 dimensions, respectively.
+ * Note that 1 + 15 == 16 so there will be no additional padding
+ * in mapping_info (not that it actually matters, though).
+ */
+#define LINKTEST_MAPPING_MAX_DIM 15
+/*  Maximal length of the location string. */
+#define LINKTEST_MAPPING_MAX_LOCATION_LEN 256
+/* Upper limit for the buffer size that is sent inline with verbs */
+#define LINKTEST_IBVERBS_MAX_INLINE_SZ 64
+/* Upper limit for the buffer size for the date and time when writing
+ * started by rank 0. 16 characters are required to store day, month,
+ * hour, minute and second. At least 4 for the year are needed if we
+ * continue to use the Gregorian calendar or a similar calendar with
+ * the same epoch. 32 charcter as such should be sufficient.
+ */
+#define LINKTEST_OUTPUT_DATETIME_LENGTH 32
+
+#endif

benchmark/cmdline.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_CMDLINE_H
+#define LINKTEST_CMDLINE_H
+
+#include <string>
+#include "vcluster.h"
+
+
+/* Command line arguments. Default values are specified
+ * in the .c file.
+ *
+ * linktest_args is a singleton. Use parse_cmdline_args to
+ * get a pointer to the one instance of linktest_args.
+ */
+struct linktest_args {
+                 bool          special_args;
+                 std::int8_t   randomize_buffers;
+                 std::int8_t   check_buffers;
+                 std::int8_t   do_alltoall;
+                 std::int8_t   do_bidir;
+                 std::int8_t   do_unidir;
+                 std::int8_t   do_mix;
+                 std::int8_t   do_serial;
+                 std::int8_t   do_nosion;
+                 std::int8_t   do_sion_par;
+                 std::int8_t   do_use_gpus;
+                 std::int8_t   do_bisection;
+                 std::int8_t   use_multi_buf;
+                 std::uint64_t num_msg;
+                 std::uint64_t num_warmup_msg;
+                 std::uint64_t len_msg;
+                 std::uint64_t buf_mt_seed;
+                 std::uint64_t num_multi_buf;
+                 std::uint64_t num_randomize_tasks;
+                 std::uint64_t seed_randomize_tasks;
+                 std::uint64_t max_stest;
+                 std::uint64_t min_iterations;
+                 double        min_walltime;
+    static const std::size_t   VClusterImplmaxCharacters=16;
+                 AllocatorType alloc_typ;
+                 std::string   virtual_cluster_implementation;
+                 std::string   memory_buffer_allocator;
+                 std::string   output;
+};
+
+/* Parse the command line arguments. This is usually only done by
+ * the root process. If the spawner forwards the command line arguments
+ * to all processing elements this function may also be executed
+ * everywhere.
+ *
+ * An earlier design of the command line parsing functionality provided
+ * options to pass command line arguments to the vcluster implementation.
+ * This however is only feasible if all processes have a copy of the
+ * argv argument which is not even guaranteed by the MPI standard.
+ * Instead it is better to provide options to the spawner which can use
+ * environment variables to pass the information to the processes.
+ */
+
+const struct linktest_args* parse_cmdline_args(int argc, char **argv);
+const struct linktest_args* fixup_cmdline_args(std::uint64_t size);
+
+
+/* Broadcast the command line arguments
+ */
+const struct linktest_args *bcast_cmdline_args(VirtualCluster* cl,
+                                               int root, const struct
+                                               linktest_args* args);
+
+/* Print the command line arguments. The function takes into account the
+ * standard/global arguments (see struct linktest_args) as well as the
+ * vcluster implementation specific arguments.
+ */
+void print_cmdline_args(const struct linktest_args *args);
+
+#endif

benchmark/compiler.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_COMPILER_H
+#define LINKTEST_COMPILER_H
+
+#include <cstdint>
+#include "error.h"
+
+#if defined(__GNUC__) || defined(__xlc__)
+# define likely(x)      __builtin_expect(!!(x), ERROR)
+# define unlikely(x)    __builtin_expect(!!(x), SUCCESS)
+// EXEC x, if this fails exec y
+# define EXEC_IFFAIL(x,y) do { \
+  int retval = x; \
+  if (unlikely(retval)) { \
+    y; \
+  } \
+} while (0)
+// EXEC x, if this fails return ERROR
+# define EXEC_NOFAIL(x) EXEC_IFFAIL(x,return ERROR)
+
+#ifdef __cplusplus
+#define restrict        __restrict
+#endif
+#endif
+
+#endif
\ No newline at end of file

benchmark/config.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_CONFIG_H
+#define LINKTEST_CONFIG_H
+
+#include <cstdint>
+
+// C Header needed since this config file is used for pingponganalysis and linktest-report
+#include "cconfig.h"
+
+constexpr std::int32_t VERSION_MAJOR                    = LINKTEST_VERSION;
+constexpr std::int32_t VERSION_MINOR                    = LINKTEST_VERSION_SUB;
+constexpr std::int32_t VERSION_PATCH                    = LINKTEST_VERSION_PATCHLEVEL;
+constexpr char         GIT_HASH_[]                      = GIT_HASH;
+constexpr std::int32_t GIT_HASH_LEN_                    = GIT_HASH_LEN; // sizeof(GIT_HASH_);
+constexpr char         GIT_HASH_SHORT_[]                = GIT_HASH;
+constexpr std::int32_t GIT_HASH_SHORT_LEN_              = GIT_HASH_SHORT_LEN; // sizeof(GIT_HASH_SHORT_);
+constexpr char         LINKTEST_ENVIRON_PREFIX_[]       = LINKTEST_ENVIRON_PREFIX;
+constexpr char         LINKTEST_ID_[]                   = LINKTEST_ID;
+constexpr char         END_HEADER_[]                    = END_HEADER;
+constexpr char         END_BLOCK_[]                     = END_BLOCK;
+constexpr std::size_t  LINKTEST_IBVERBS_MAX_INLINE_SZ_  = LINKTEST_IBVERBS_MAX_INLINE_SZ;
+constexpr std::size_t  LINKTEST_OUTPUT_DATETIME_LENGTH_ = LINKTEST_OUTPUT_DATETIME_LENGTH;
+
+#endif

benchmark/convert.py

+
+import sys
+
+
+def _file_as_char_list(fn):
+    """
+    Read in a file and return the characters in hexadecimal
+    string form.
+    """
+    with open(fn, "rb") as f:
+        chars = ["0x%02x" % c for c in f.read()]
+    chars += ["0x0"]
+
+    return chars
+
+def _chars_in_groups_of_eight(chars):
+    """
+    From groups of 8 to print out a long array in multiple lines
+    to valid C code.
+    """
+    sep = ", "
+    lines = []
+
+    i = 0
+    while i < len(chars):
+        line = []
+        for j in range(8):
+            if (i + j) < len(chars):
+                line += [chars[i + j]]
+
+        i = i + 8
+
+        lines += ["\t" + sep.join(line)]
+
+    return lines
+
+def main(i_fn, o_fn, var_name):
+    chars = _file_as_char_list(i_fn)
+    lines = _chars_in_groups_of_eight(chars)
+
+    with open(o_fn, "w", encoding="utf-8") as f:
+        f.write("\n")
+        f.write("const unsigned char %s[] = {\n" % var_name)
+        f.write(",\n".join(lines) + "\n")
+        f.write("};\n")
+        f.write("\n")
+
+if "__main__" == __name__:
+    main(i_fn=sys.argv[1], o_fn=sys.argv[2], var_name=sys.argv[3])

benchmark/cuda_kernels.cu

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+__global__ void fill_cuda_kernel_char(char* start, const char* end, const char val)
+{
+
+    start += blockIdx.x*blockDim.x + threadIdx.x;
+    for(; start < end; start += (gridDim.x*blockDim.x))
+        *start = val;
+}

benchmark/environ.cc

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#include "environ.h"
+#include "config.h"
+#include "compiler.h"
+
+#include <cstdlib>
+
+int read_environ_int(const char* name, int *x)
+{
+    const char* str;
+    int tmp;
+    char* tailptr;
+
+    str = getenv(name);
+    if (unlikely(!str))
+        return ERROR;
+
+    tmp = (int )strtol(str, &tailptr, 0);
+    if (unlikely(0 != *tailptr)) {
+        return ERROR;
+    }
+
+    *x = tmp;
+    return SUCCESS;
+}
+
+int read_environ_long(const char* name, long *x)
+{
+    const char* str;
+    long tmp;
+    char* tailptr;
+
+    str = getenv(name);
+    if (unlikely(!str))
+        return ERROR;
+
+    tmp = (long )strtol(str, &tailptr, 0);
+    if (unlikely(0 != *tailptr)) {
+        return ERROR;
+    }
+
+    *x = tmp;
+    return SUCCESS;
+}
+
+int read_environ_float(const char* name, float *x)
+{
+    const char* str;
+    float tmp;
+    char* tailptr;
+
+    str = getenv(name);
+    if (unlikely(!str))
+        return ERROR;
+
+    tmp = strtof(str, &tailptr);
+    if (unlikely(0 != *tailptr)) {
+        return ERROR;
+    }
+
+    *x = tmp;
+    return SUCCESS;
+}
+
+int read_environ_double(const char* name, double *x)
+{
+    const char* str;
+    double tmp;
+    char* tailptr;
+
+    str = getenv(name);
+    if (unlikely(!str))
+        return ERROR;
+
+    tmp = strtod(str, &tailptr);
+    if (unlikely(0 != *tailptr)) {
+        return ERROR;
+    }
+
+    *x = tmp;
+    return SUCCESS;
+}
+
+int read_environ_str(const char* name, const char **x)
+{
+    const char* str;
+
+    str = getenv(name);
+    if (unlikely(!str))
+        return ERROR;
+
+    *x = str;
+    return SUCCESS;
+}

benchmark/environ.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_ENVIRON_H
+#define LINKTEST_ENVIRON_H
+
+/*
+ * Read environment variables and convert variables.
+ */
+int read_environ_int(   const char* name, int         *x);
+int read_environ_long(  const char* name, long        *x);
+int read_environ_float( const char* name, float       *x);
+int read_environ_double(const char* name, double      *x);
+int read_environ_str(   const char* name, const char **x);
+
+#endif

benchmark/error.cc

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#include "error.h"
+#include "config.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstdarg>
+#include <exception>
+
+static void report(const char* prefix, const char* file,
+           const char* func, long line, const char* fmt,
+           va_list vl)
+{
+    static char msg[4096];
+
+    std::vsnprintf(msg, sizeof(msg), fmt, vl);
+    std::fprintf(stderr, " [%s in %s:%ld] %s%s\n", file, func,
+        line, prefix, msg );
+}
+
+
+void linktest_fatal(const char* file, const char* func, long line, const char* fmt, ...)
+{
+    va_list vl;
+
+    va_start(vl, fmt);
+    report("fatal: ", file, func, line, fmt, vl);
+    va_end(vl);
+
+    std::fflush(NULL);
+    std::terminate();
+}
+
+void linktest_error(const char* file, const char* func, long line, const char* fmt, ...)
+{
+    va_list vl;
+
+    va_start(vl, fmt);
+    report("error: ", file, func, line, fmt, vl);
+    va_end(vl);
+
+    std::fflush(NULL);
+}
+
+void linktest_warn(const char* file, const char* func, long line, const char* fmt, ...)
+{
+    va_list vl;
+
+    va_start(vl, fmt);
+    report("warning: ", file, func, line, fmt, vl);
+    va_end(vl);
+
+    std::fflush(NULL);
+}
+
+void linktest_info(const char* file, const char* func, long line, const char* fmt, ...)
+{
+    va_list vl;
+
+    va_start(vl, fmt);
+    report("info: ", file, func, line, fmt, vl);
+    va_end(vl);
+}
+
+void linktest_debug(const char* file, const char* func, long line, const char* fmt, ...)
+{
+    va_list vl;
+
+    va_start(vl, fmt);
+    report("debug: ", file, func, line, fmt, vl);
+    va_end(vl);
+
+    std::fflush(NULL);
+}
\ No newline at end of file

benchmark/error.h

+/****************************************************************************
+**  LinkTest                                                               **
+*****************************************************************************
+**  Copyright (c) 2008-2022                                                **
+**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
+**                                                                         **
+**  See the file COPYRIGHT in the package base directory for details       **
+****************************************************************************/
+#ifndef LINKTEST_ERROR_H
+#define LINKTEST_ERROR_H
+
+constexpr int SUCCESS = 0;
+constexpr int ERROR   = 1;
+
+void linktest_fatal(const char* file, const char* func, long line, const char* fmt, ...);
+void linktest_error(const char* file, const char* func, long line, const char* fmt, ...);
+void linktest_warn(const char* file, const char* func, long line, const char* fmt, ...);
+void linktest_info(const char* file, const char* func, long line, const char* fmt, ...);
+void linktest_debug(const char* file, const char* func, long line, const char* fmt, ...);
+
+/* Error handling macros. fatal prints an error message and the terminates the
+ * execution.
+ *
+ * The names are pretty generic so we have to be careful to avoid naming conflicts
+ * that result in hard to understand compiler errors.
+ */
+#define fatal(fmt, ...) linktest_fatal(__FILE__, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define error(fmt, ...) linktest_error(__FILE__, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define warn(fmt, ...) linktest_warn(__FILE__, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define info(fmt, ...) linktest_info(__FILE__, __func__, __LINE__, fmt, ## __VA_ARGS__)
+#define debug(fmt, ...) linktest_debug(__FILE__, __func__, __LINE__, fmt, ## __VA_ARGS__)
+
+#endif
\ No newline at end of file