diff --git a/benchmark/benchmark.cc b/benchmark/benchmark.cc
index 6717b033c9d9f06868839f4b775dea46772416d2..f4bc04cdafb1ac723cfef0271de693aee04ecd4f 100644
--- a/benchmark/benchmark.cc
+++ b/benchmark/benchmark.cc
@@ -19,6 +19,7 @@
#include "environ.h"
#include "format_units.h"
#include "format_print.h"
+#include "output_sion.h"
#include <cstdlib>
#include <cstdio>
#include <cstring>
@@ -107,10 +108,8 @@ inline unsigned int Benchmark::getNumSteps() const {
const int INVALID_RANK = -8; // Random negative Number
const int INVALID_STEP = -9; // Random negative Number
-void Benchmark::computeRankStepMappings() {
+void Benchmark::permuteSteps(){
stepPermutation.resize(getNumSteps());
- getRankFromStep.resize(getNumSteps());
- getStepFromRank.resize(size());
std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
@@ -128,7 +127,27 @@ void Benchmark::computeRankStepMappings() {
}
rootWatch->stop();
printTimingIfRoot(cl->rank(), "[randvec]", rootWatch->getDuration());
+}
+
+void Benchmark::printCommunicationScheme(){
+ #ifdef DEBUG_PRINT_COMMUNICATION_SCHEME
+ for(std::int i = 0; i < size(); i++) {
+ barrier();
+ if( rank() == i) {
+ std::cout << rank() << ": getRankFromStep: " << getRankFromStep << "\n";
+ std::cout << rank() << ": getStepFromRank: " << getStepFromRank << "\n";
+ std::cout << rank() << ": stepPermutation: " << stepPermutation << "\n";
+ }
+ }
+ #endif
+}
+
+void Benchmark::computeRankStepMappings() {
+ getRankFromStep.resize(getNumSteps());
+ getStepFromRank.resize(size());
+
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
rootWatch->start();
std::fill(getRankFromStep.begin(), getRankFromStep.end(), INVALID_RANK);
std::fill(getStepFromRank.begin(), getStepFromRank.end(), INVALID_STEP);
@@ -137,20 +156,229 @@ void Benchmark::computeRankStepMappings() {
} else {
mapWith1FactorAlgorithm(args->num_randomize_tasks > 0);
}
+
+ //Permute Steps
+ stepPermutation.resize(getNumSteps());
+ permuteSteps();
rootWatch->stop();
printTimingIfRoot(cl->rank(), "[getpart]", rootWatch->getDuration());
+ //DEBUG: Print Communication Scheme
#ifdef DEBUG_PRINT_COMMUNICATION_SCHEME
- for(int i = 0; i < size(); i++) {
- barrier();
- if( rank() == i) {
- std::cout << rank() << ": getRankFromStep: " << getRankFromStep << "\n";
- std::cout << rank() << ": getStepFromRank: " << getStepFromRank << "\n";
- std::cout << rank() << ": stepPermutation: " << stepPermutation << "\n";
+ printCommunicationScheme();
+ #endif
+}
+
+int Benchmark::checkRankStepMappings(std::vector<uint64_t> com){
+ const std::size_t num_steps=com.size();
+ int err=0;
+ std::size_t i, j, k;
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
+
+ /* Bad Connection Vector */
+ std::vector<std::unique_ptr<std::uint64_t[]>> bc;
+
+ /* Check Bounds */
+ rootWatch->start();
+ k=0;
+ for(i=0;i<num_steps;i++){
+ if((int)(com[i])>=size()){ //TODO: Currently only using 1st
+ std::unique_ptr<std::uint64_t[]> tmp(new std::uint64_t[3]{(std::uint64_t)rank(),com[i],(std::uint64_t)i}); //TODO: Currently only using 1st
+ bc.push_back(std::move(tmp));
+ k++;
}
}
- barrier();
- #endif
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[checkcombounds]", rootWatch->getDuration());
+
+ /* Print Out-Of-Bounds Connections */
+ if(rank()==0){
+ if(k>0){
+ fprintf(stderr,"Task %d identified the following %zu out-of-bounds connections:",rank(),k);
+ for(j=0;j<k;j++){
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",bc[j][0],bc[j][1],bc[j][2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ for(i=1;i<(size_t)size();i++){
+ cl->recv(i,&k,1);
+ if(k>0){
+ fprintf(stderr,"Task %zu identified the following %zu out-of-bounds connections:",i,k);
+ for(j=0;j<k;j++){
+ std::uint64_t tmp[3];
+ cl->recv(i,tmp,3);
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",tmp[0],tmp[1],tmp[2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ }
+ }else{
+ cl->send(0,&k,1);
+ for(j=0;j<k;j++){
+ cl->send(0,bc[j].get(),3);
+ }
+ }
+
+ /* Bin Communication Matrix According To Partner */
+ rootWatch->start();
+ std::vector<std::uint64_t>* con=new std::vector<std::uint64_t>[size()];
+ for(i=0;i<(size_t)size();i++){
+ for(j=0;j<(num_steps);j++){
+ if((size_t)(com[j])==i) con[i].push_back(j+1); //TODO: Currently only using 1st
+ }
+ }
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[bincom]", rootWatch->getDuration());
+
+ /* Check For Duplicate Connections */
+ bc.resize(0);
+ rootWatch->start();
+ k=0;
+ for(i=0;i<(size_t)size();i++){
+ if(con[i].size()>1){
+ for(j=1;j<con[i].size();j++){
+ std::unique_ptr<std::uint64_t[]> tmp(new std::uint64_t[3]{(std::uint64_t)rank(),i,con[i][j]});
+ bc.push_back(std::move(tmp));
+ k++;
+ }
+ }
+ }
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[checkcomduplicates]", rootWatch->getDuration());
+
+ /* Print Duplicate Connections */
+ if(rank()==0){
+ if(k>0){
+ fprintf(stderr,"Task %d identified the following %zu duplicate connections:",rank(),k);
+ for(j=0;j<k;j++){
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",bc[j][0],bc[j][1],bc[j][2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ for(i=1;i<(std::size_t)size();i++){
+ cl->recv(i,&k,1);
+ if(k>0){
+ fprintf(stderr,"Task %zu identified the following %zu duplicate connections:",i,k);
+ for(j=0;j<k;j++){
+ std::uint64_t tmp[3];
+ cl->recv(i,tmp,3);
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",tmp[0],tmp[1],tmp[2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ }
+ }else{
+ cl->send(0,&k,1);
+ for(j=0;j<k;j++){
+ cl->send(0,bc[j].get(),3);
+ }
+ }
+
+ /* Check Connections With Partners */
+ bc.resize(0);
+ rootWatch->start();
+ k=0;
+ for(std::size_t i=0;i<(std::size_t)size()-1;i++){
+ int partner;
+ std::uint64_t step;
+ const int idle=(i*size()/2)%(size()-1);
+ if(rank()==(size()-1)) partner=idle;
+ else if(rank()==idle) partner=size()-1;
+ else partner=(i-rank()+size()-1)%(size()-1);
+ if(rank()<partner){
+ cl->send(partner,con[partner].data(),1);
+ cl->recv(partner,&step ,1);
+ }else{
+ cl->recv(partner,&step ,1);
+ cl->send(partner,con[partner].data(),1);
+ }
+ if(con[partner][0]!=step){
+ std::unique_ptr<std::uint64_t[]> tmp(new std::uint64_t[3]{(std::uint64_t)partner,(std::uint64_t)rank(),step});
+ bc.push_back(std::move(tmp));
+ k++;
+ }
+ }
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[checkcom]", rootWatch->getDuration());
+ delete[] con;
+
+ /* Print Bad Connections */
+ if(rank()==0){
+ if(k>0){
+ fprintf(stderr,"Task %d identified the following %zu bad connections:",rank(),k);
+ for(j=0;j<k;j++){
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",bc[j][0],bc[j][1],bc[j][2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ for(i=1;i<(std::size_t)size();i++){
+ cl->recv(i,&k,1);
+ if(k>0){
+ fprintf(stderr,"Task %zu identified the following %zu bad connections:",i,k);
+ for(j=0;j<k;j++){
+ std::uint64_t tmp[3];
+ cl->recv(i,tmp,3);
+ fprintf(stderr," %" PRIu64 "->%" PRIu64 "(%" PRIu64 ")",tmp[0],tmp[1],tmp[2]);
+ }
+ fprintf(stderr,"\n");
+ err=1;
+ }
+ }
+ }else{
+ cl->send(0,&k,1);
+ for(j=0;j<k;j++){
+ cl->send(0,bc[j].get(),3);
+ }
+ }
+
+ /* Broadcast Error To All Ranks */
+ cl->bcast(0,&err,1);
+
+ return(err);
+}
+
+void Benchmark::loadRankStepMappings(const std::size_t num_iterations, const std::size_t iteration){
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
+
+ //Load Communication Matrix
+ rootWatch->start();
+ std::vector<uint64_t> com=cl->read_COM_parallel(args,num_iterations,iteration);
+ if(com.size()==0){
+ fatal("Failed to read communication pattern.");
+ }
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[loadcom]", rootWatch->getDuration());
+
+ //TODO: Currently only extracting 1 iteration
+ const std::size_t num_steps=com.size();
+
+ //Test Loaded Communication Matrix
+ if(checkRankStepMappings(com)){
+ fatal("Errors detected in loaded communication pattern!");
+ }
+
+ //Configure Communication Pattern
+ rootWatch->start();
+ getRankFromStep.resize(num_steps);
+ getStepFromRank.resize(size());
+ getRankFromStep.assign(com.begin(),com.end());
+ std::fill(getStepFromRank.begin(), getStepFromRank.end(), INVALID_STEP);
+ for(std::size_t step=0;step<num_steps;step++){
+ getStepFromRank.at(getRankFromStep.at(step))=step;
+ }
+ rootWatch->stop();
+ printTimingIfRoot(cl->rank(), "[applycom]", rootWatch->getDuration());
+
+ //Permute Steps
+ permuteSteps();
+
+ //DEBUG: Print Communication Scheme
+ printCommunicationScheme();
}
std::vector<int> Benchmark::oneFactorAlgorithm(const int size, const int rank) {
@@ -819,8 +1047,18 @@ void Benchmark::run_Stresstest() {
}
}
-void Benchmark::run_Measuringtest() {
- const std::size_t NUM_ITERATIONS = randomizeTasks() ? args->num_randomize_tasks : 1;
+void Benchmark::run_Measuringtest(){
+ std::uint64_t tmp=0;
+ if(args->sion_com[0]!='\0'){
+ if(cl->rank()==0){
+ tmp=get_num_iterations_from_pattern_sion(args,size());
+ cl->bcast(0,&tmp,1);
+ }else{
+ cl->bcast(0,&tmp,1);
+ }
+ }
+ cl->barrier();
+ const std::size_t NUM_ITERATIONS = (tmp!=0) ? tmp : (randomizeTasks() ? args->num_randomize_tasks : 1);
newStatsVec.resize(NUM_ITERATIONS);
double timeWork=0.0, timeTotal=0.0; // Total time = work time + all-to-all time (+ overhead)
@@ -829,7 +1067,11 @@ void Benchmark::run_Measuringtest() {
std::printf("\n\nIteration:%4ld of %4ld\n", iteration, NUM_ITERATIONS);
std::printf("Preparing\n");
}
- computeRankStepMappings();
+ if(args->sion_com[0]!='\0'){
+ loadRankStepMappings(NUM_ITERATIONS,iteration);
+ }else{
+ computeRankStepMappings();
+ }
stats = &newStatsVec.at( iteration - 1 );
stats->initialize(cl.get(), args);
if (0 == cl->rank()) {
diff --git a/benchmark/benchmark.h b/benchmark/benchmark.h
index e6990278a1a38abbc110fbfec593939dc9899845..fe55717a7eb9822129094574e4b5603101be63b8 100644
--- a/benchmark/benchmark.h
+++ b/benchmark/benchmark.h
@@ -70,6 +70,10 @@ namespace linktest{
std::vector<int> oneFactorAlgorithm(const int size, const int rank);
void mapWith1FactorAlgorithm(bool randomizeTasks);
void mapWithBisection();
+ void permuteSteps();
+ void printCommunicationScheme();
+ void loadRankStepMappings(const std::size_t num_iterations, const std::size_t iteration);
+ int checkRankStepMappings(std::vector<uint64_t> com);
int work_alltoall();
/**
* Executes semi-, bi- or unidirectional tests
diff --git a/benchmark/cconfig.h b/benchmark/cconfig.h
index 325ef6a991ff71a8ce6a7eaa52eea7bc3cf2e9d7..622e39734eed5beb567a71732ce54ea8d4c92759 100644
--- a/benchmark/cconfig.h
+++ b/benchmark/cconfig.h
@@ -49,6 +49,10 @@
/* Linktest alignment string for output files. Marks the end of the Binary Header */
#define END_HEADER "END_HEADER"
+/* LinkTest identification for input communication SION files (offset: 0x00) */
+#define LINKTEST_COM_ID "LKTST_COM"
+
+
/* Linktest alignment string for output files. Marks end of thread output */
#define END_BLOCK "END_BLOCK"
diff --git a/benchmark/cmdline.cc b/benchmark/cmdline.cc
index 6adab296a1df33b5c804875938bd12d24af3f6a0..3084bad942cb08f4bd409d49def6925bb6620b25 100644
--- a/benchmark/cmdline.cc
+++ b/benchmark/cmdline.cc
@@ -319,6 +319,16 @@ auto linktest_all_cmdline_argdefs = {
.required = false,
.help = "Seed for task randomization",
.p = &cmdline_args.seed_randomize_tasks
+ },
+ Argument{
+ .type = Argument::kString,
+ .nargs = 1,
+ .optshort = "",
+ .optlong = "sion-communication-pattern",
+ .defaultval = "",
+ .required = false,
+ .help = "SION file containing communication patterns",
+ .p = &cmdline_args.sion_com
}
};
@@ -709,8 +719,16 @@ const struct linktest_args *bcast_cmdline_args(VirtualCluster* cl,
cl->bcast(root,&tmp->buf_mt_seed ,1);
cl->bcast(root,&tmp->check_buffers ,1);
- /* Broadcast Output Filename */
+ /* Broadcast Input Communication Pattern Filename */
std::uint64_t buf_size;
+ if(cl->rank()==root) buf_size=tmp->sion_com.size()+1;
+ cl->bcast(root,&buf_size,1);
+ char buf_sion_com[buf_size];
+ if(cl->rank()==root) std::snprintf(buf_sion_com, sizeof(buf_sion_com), "%s", tmp->sion_com.c_str());
+ cl->bcast(root, buf_sion_com, buf_size);
+ tmp->sion_com = buf_sion_com;
+
+ /* Broadcast Output Filename */
if(cl->rank()==root) buf_size=tmp->output.size()+1;
cl->bcast(root,&buf_size,1);
char buf_output[buf_size];
@@ -846,8 +864,11 @@ void print_cmdline_args(const struct linktest_args* args){
std::printf("Seed for random comm. schemes: %-" PRIu64 "\n" ,args->seed_randomize_tasks);
}
std::printf("Write output SION file? %-s\n" ,getProtocol());
+ if(args->sion_com[0]!='\0'){
+ std::printf("Input Com. Pat. SION filename: %-s\n" ,args->sion_com.c_str());
+ }
if(args->do_nosion==0) {
- std::printf("Output SION filename: %-s\n" ,args->output.c_str());
+ std::printf("Output SION filename: %-s\n" ,args->output.c_str());
}
std::printf(HLINE);
}
diff --git a/benchmark/cmdline.h b/benchmark/cmdline.h
index 46644b1b5504ef9cdb8c41c0d1d2fa6d3037e392..fe9e6da2d1ed443506fc90a527a42d52b0641741 100644
--- a/benchmark/cmdline.h
+++ b/benchmark/cmdline.h
@@ -47,6 +47,7 @@ struct linktest_args {
AllocatorType alloc_typ;
std::string virtual_cluster_implementation;
std::string memory_buffer_allocator;
+ std::string sion_com;
std::string output;
};
diff --git a/benchmark/config.h b/benchmark/config.h
index d97ec1f7b8909cf0a1ae3dbdc5afd44bc5ae1136..117d6ca37845c47d66c5c97091ad689fc5d367fd 100644
--- a/benchmark/config.h
+++ b/benchmark/config.h
@@ -22,6 +22,7 @@ constexpr std::int32_t GIT_HASH_LEN_ = GIT_HASH_LEN; // sizeo
constexpr char GIT_HASH_SHORT_[] = GIT_HASH;
constexpr std::int32_t GIT_HASH_SHORT_LEN_ = GIT_HASH_SHORT_LEN; // sizeof(GIT_HASH_SHORT_);
constexpr char LINKTEST_ENVIRON_PREFIX_[] = LINKTEST_ENVIRON_PREFIX;
+constexpr char LINKTEST_COM_ID_[] = LINKTEST_COM_ID;
constexpr char LINKTEST_ID_[] = LINKTEST_ID;
constexpr char END_HEADER_[] = END_HEADER;
constexpr char END_BLOCK_[] = END_BLOCK;
diff --git a/benchmark/output_sion.cc b/benchmark/output_sion.cc
index ac8c8d0fff94cd1e8bb92fab82b1fec8c8716855..d1a8ba77c657479e19481d827247f7d6664c2fd1 100644
--- a/benchmark/output_sion.cc
+++ b/benchmark/output_sion.cc
@@ -21,17 +21,25 @@
#include <cstdlib>
#include <cstdio>
#include <cstring>
+#include <cinttypes>
#include <type_traits>
+#include <algorithm>
#ifdef USE_SION
#include <mpi.h>
#ifdef __cplusplus
- extern "C" {
+ extern "C" {
#endif
#include "sion.h"
#ifdef __cplusplus
- }
+ }
#endif
+ enum DataType{
+ PatternFormatInvalid=0, //Invalid Data Type
+ PatternFormatCom =1, //COM matrix : Rank-by-Step (Value: Partner)
+ PatternFormatMat =2, //Communication Matrix: Rank-by-Partner (Value: Step )
+ PatternFormatPer =3, //Permutation Vector for Permuting 1-Factor Output - CURRENTLY NOT USED
+ };
#endif
#ifdef USE_SION
@@ -62,7 +70,7 @@ static Serializer getSerializer(VirtualCluster* cl, const struct linktest_args *
s.addDataSource("args->use_multi_buf" ,&args->use_multi_buf );
s.addDataSource("args->randomize_buffers" ,&args->randomize_buffers );
s.addDataSource("args->check_buffers" ,&args->check_buffers );
- s.addDataSource("args->alloc_typ" ,&args->alloc_typ ); // enum aka int
+ s.addDataSource("args->alloc_typ" ,&args->alloc_typ );
// uint64_t data
s.addDataSource("args->num_msg" ,&args->num_msg );
s.addDataSource("args->len_msg" ,&args->len_msg );
@@ -114,6 +122,27 @@ static Serializer getSerializer(VirtualCluster* cl, const struct linktest_args *
return s;
}
+
+static Serializer getSerializedCommunicationPatternHead(const char* const com_id, const uint32_t* const version, const uint64_t* const num_iter){
+ /* Note although this function promises not to change its input,
+ * when reading to these pointers data will be overwritten.
+ */
+ Serializer s;
+ s.addDataSource(LINKTEST_COM_ID_, com_id , std::strlen(LINKTEST_COM_ID_)+1);
+ s.addDataSource("version" , version , 3 );
+ s.addDataSource("num_iter" , num_iter );
+ return s;
+}
+
+static Serializer getSerializedCommunicationPatternBlockHead(const uint8_t* const dtype, const uint64_t* const numel){
+ /* Note although this function promises not to change its input,
+ * when reading to these pointers data will be overwritten.
+ */
+ Serializer s;
+ s.addDataSource("dtype", dtype);
+ s.addDataSource("numel", numel);
+ return s;
+}
#endif
int linktest_output_sion_collect_local_data(VirtualCluster* cl,
@@ -389,3 +418,444 @@ int linktest_output_sion_parallel(VirtualCluster* cl,
return ERROR;
#endif
}
+
+/*************************/
+/* Determine Chunk Sizes */
+/*************************/
+//Should only be executed as root!
+#ifdef USE_SION
+ std::vector<sion_int64> getChunkSizes(char* filename, const int vcl_size){ //BUG: Should be "const char* const filename", but SIONlib 1.7.7 requires "char* filename"
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(0); //Safe since this should only be executed by root.
+ auto printTiming = [&rootWatch](std::string step_name){
+ printTimingIfRoot(0, step_name, rootWatch->getDuration());
+ };
+
+ /*************/
+ /* Open File */
+ /*************/
+ FILE* fp ;
+ int ntasks ;
+ int nfiles ;
+ std::int32_t fsblksz ;
+ rootWatch->start();
+ auto file_handle = sion_open(filename,
+ "rb" ,
+ &ntasks ,
+ &nfiles ,
+ NULL ,
+ &fsblksz,
+ NULL ,
+ &fp );
+ rootWatch->stop();
+ printTiming("[sioncomopenroot]");
+
+ /****************************************************/
+ /* Check Number Of Tasks File Should Be Opened With */
+ /****************************************************/
+ if(ntasks!=vcl_size){
+ sion_close(file_handle);
+ fatal("To read %s %d tasks are required, while LinkTest is currently running with %d tasks.",filename,ntasks,vcl_size);
+ }
+
+ /******************/
+ /* Read Meta Data */
+ /******************/
+ int maxchunks;
+ sion_int64 globalskip;
+ sion_int64 start_of_varheader;
+ sion_int64 *chunksizes;
+ sion_int64 *globalranks;
+ sion_int64 *blockcount;
+ sion_int64 *blocksizes;
+ sion_get_locations(file_handle,
+ &ntasks,
+ &maxchunks,
+ &globalskip,
+ &start_of_varheader,
+ &chunksizes,
+ &globalranks,
+ &blockcount,
+ &blocksizes);
+
+ /*************************/
+ /* Determine Chunk Sizes */
+ /*************************/
+ std::vector<sion_int64> chunk_sizes;
+ chunk_sizes.resize(ntasks);
+ memcpy(chunk_sizes.data(),chunksizes,ntasks*sizeof(sion_int64));
+
+ /**************/
+ /* Close File */
+ /**************/
+ rootWatch->start();
+ sion_close(file_handle);
+ rootWatch->stop();
+ printTiming("[sioncomcloseroot]");
+
+ return(chunk_sizes);
+ }
+#endif
+
+/*********************************************/
+/* Get Number Of Iterations & Test SION File */
+/*********************************************/
+// Should only be executed with root (0) task!
+// If return 0 then error! */
+uint64_t get_num_iterations_from_pattern_sion(const struct linktest_args* const args, const int vcl_size){
+ #ifdef USE_SION
+ /*********************/
+ /* Start a Stopwatch */
+ /*********************/
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(0); //Safe since this should only be executed by root.
+ auto printTiming = [&rootWatch](std::string step_name){
+ printTimingIfRoot(0, step_name, rootWatch->getDuration());
+ };
+
+ /************************/
+ /* Determine Chunk Size */
+ /************************/
+ char filename[args->sion_com.size()+1];
+ /* ToDo: Below is a super awkward work around for SIONlib which
+ * does not require a const filename. This is not inline with
+ * our const command-line arguments, hence we need to create a
+ * new temporary string to hold the filename to open. This is to
+ * be fixed when a new version of SIONlib that rectifies this
+ * bug is available.
+ */
+ std::snprintf(filename, sizeof(filename), "%s", args->sion_com.c_str());
+ std::vector<sion_int64> chunk_sizes=getChunkSizes(filename,vcl_size);
+ sion_int64* chunksz=chunk_sizes.data();
+
+ /*************/
+ /* Open File */
+ /*************/
+ FILE* fp ;
+ int ntasks ;
+ int nfiles ;
+ std::int32_t fsblksz ;
+ int* globalranks=NULL;
+ rootWatch->start();
+ auto file_handle = sion_open(filename,
+ "rb,posix" ,
+ &ntasks ,
+ &nfiles ,
+ &chunksz ,
+ &fsblksz ,
+ &globalranks,
+ &fp );
+ rootWatch->stop();
+ printTiming("[sioncomrootopen]");
+ #ifdef DEBUG_COM_INPUT
+ printf("%s->ntasks: %d\n",args->sion_com.c_str(),ntasks);fflush(stdout);
+ #endif
+
+ /****************************************************/
+ /* Check Number Of Tasks File Should Be Opened With */
+ /****************************************************/
+ if(ntasks!=vcl_size){
+ sion_close(file_handle);
+ fatal("To read %s %d tasks are required, while LinkTest is currently running with %d tasks.",args->sion_com.c_str(),ntasks,vcl_size);
+ }
+
+ /*************/
+ /* Read Head */
+ /*************/
+ // Construct Buffer
+ std::string com_id;com_id.resize(strlen(LINKTEST_COM_ID_)+1);
+ std::uint32_t version[3];
+ std::uint64_t num_iter;
+ Serializer s=getSerializedCommunicationPatternHead(com_id.c_str(),version,&num_iter);
+ const size_t buf_size=s.getSize();
+ void* const buffer=::operator new(buf_size);
+ // Read Into Buffer
+ rootWatch->start();
+ if(sion_fread(buffer,buf_size,1,file_handle)!=1){
+ error("Failed to read communication-pattern header from SION file.");
+ }
+ rootWatch->stop();
+ printTiming("[sioncomrootread]");
+ //Distribute Data In Buffer
+ s.read((char*)buffer,0);
+ com_id[com_id.size()-1]='\0'; //Ensure string is null terminated
+ #ifdef DEBUG_COM_INPUT
+ printf("rank=0, com_id=%s, version=%" PRIu32 ".%" PRIu32 ".%" PRIu32 ", num_iterations=%" PRIu64 "\n",com_id.c_str(),version[0],version[1],version[2],num_iter);
+ fflush(stdout);
+ #endif
+ ::operator delete(buffer);
+
+ /**************/
+ /* Close File */
+ /**************/
+ rootWatch->start();
+ sion_close(file_handle);
+ rootWatch->stop();
+ printTiming("[sioncomrootclose]");
+
+ /**************/
+ /* Check Head */
+ /**************/
+ if(std::strcmp(LINKTEST_COM_ID_,com_id.c_str())!=0){
+ fatal("%s string not read. Read: %s.",LINKTEST_COM_ID_,com_id.c_str());
+ }
+ if(version[0]!=LINKTEST_VERSION ){
+ fatal("LinkTest major version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[0],VERSION_MAJOR);
+ }
+ if(version[1]!=LINKTEST_VERSION_SUB ){
+ fatal("LinkTest major version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[1],VERSION_MINOR);
+ }
+ if(version[2]!=LINKTEST_VERSION_PATCHLEVEL){
+ fatal("LinkTest patch version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[2],VERSION_PATCH);
+ }
+
+ /*******************************/
+ /* Return Number Of Iterations */
+ /*******************************/
+ return num_iter;
+ #else
+ error("This version of the benchmark was compiled without SIONlib support.");
+ return 0;
+ #endif
+}
+
+/*****************/
+/* SION Read COM */
+/*****************/
+std::vector<uint64_t> sion_read_communication_pattern_for_iteration_parallel([[maybe_unused]] VirtualCluster* cl, [[maybe_unused]] const struct linktest_args* const args, [[maybe_unused]] const std::size_t num_iterations, [[maybe_unused]] const std::size_t iteration){
+ // It should not be possible to reach this state without SION, but just in case.
+ #ifdef USE_SION
+ /*********************/
+ /* Start a Stopwatch */
+ /*********************/
+ std::unique_ptr<StopwatchI> rootWatch = Stopwatchfactory::getRootWatch(cl->rank());
+ auto printTiming = [rank=cl->rank(), &rootWatch](std::string step_name){
+ printTimingIfRoot(rank, step_name, rootWatch->getDuration());
+ };
+
+ /************************/
+ /* Determine Chunk Size */
+ /************************/
+ char filename[args->sion_com.size()+1];
+ /* ToDo: Below is a super awkward work around for SIONlib which
+ * does not require a const filename. This is not inline with
+ * our const command-line arguments, hence we need to create a
+ * new temporary string to hold the filename to open. This is to
+ * be fixed when a new version of SIONlib that rectifies this
+ * bug is available.
+ */
+ std::snprintf(filename, sizeof(filename), "%s", args->sion_com.c_str());
+ sion_int64 chunk_size;
+ if(cl->rank()==0){
+ std::vector<sion_int64> chunk_sizes=getChunkSizes(filename,cl->size());
+ chunk_size=chunk_sizes[0];
+ for(int i=1;i<cl->size();i++) cl->send(i,&chunk_sizes[i],1);
+ }else{
+ cl->recv(0,&chunk_size,1);
+ }
+ cl->barrier();
+
+ /*******************************/
+ /* Parameters for Opening File */
+ /*******************************/
+ FILE* fp;
+ int filenumber=0;
+ int nfiles =1;
+ int fsblksize =0;
+ char* newfname;
+ int grank=VirtualClusterSionGenericAdapter::global_rank(*cl);
+ int lrank=VirtualClusterSionGenericAdapter::local_rank(*cl);
+ int gsize=VirtualClusterSionGenericAdapter::global_size(*cl);
+ int lsize=VirtualClusterSionGenericAdapter::local_size(*cl);
+
+ /**************/
+ /* Create API */
+ /**************/
+ auto sion_api=create_and_register_api(args->virtual_cluster_implementation);
+
+ /*************/
+ /* Open File */
+ /*************/
+ rootWatch->start();
+ auto file_handle = sion_generic_paropen(
+ sion_api,
+ filename,
+ "rb",
+ &chunk_size,
+ &fsblksize,
+ cl, //gcommgroup
+ grank, //global rank
+ gsize, //global comm size
+ &filenumber, //filenumber
+ &nfiles, //numfiles
+ &lrank, //local rank
+ &lsize, //lsoze
+ &fp, //fileptr
+ &newfname //newfname
+ );
+ cl->barrier();
+ rootWatch->stop();
+ printTiming("[sioncomopenpar]");
+
+ /*************/
+ /* Read Head */
+ /*************/
+ // Construct Buffer
+ std::size_t seek_offset=0;
+ std::string com_id;com_id.resize(strlen(LINKTEST_COM_ID_)+1);
+ std::uint32_t version[3];
+ std::uint64_t num_iter;
+ Serializer shead=getSerializedCommunicationPatternHead(com_id.c_str(),version,&num_iter);
+ const size_t head_size=shead.getSize();
+ void* const head_buffer=::operator new(head_size);
+ // Read Into Buffer
+ rootWatch->start();
+ if(sion_fread(head_buffer,head_size,1,file_handle)!=1){
+ ::operator delete(head_buffer);
+ error("Failed to read communication-pattern header from SION file.");
+ }
+ rootWatch->stop();
+ printTiming("[sioncomreadheadpar]");
+ seek_offset+=head_size;
+ //Distribute Data In Buffer
+ shead.read((char*)head_buffer,cl->rank());
+ #ifdef DEBUG_COM_INPUT
+ for(int k=0;k<cl->size();k++){
+ if(cl->rank()==k){
+ printf("rank=%d, com_id=%s, version=%" PRIu32 ".%" PRIu32 ".%" PRIu32 ", num_iterations=%" PRIu64 "\n",cl->rank(),com_id.c_str(),version[0],version[1],version[2],num_iter);
+ fflush(stdout);
+ }
+ cl->barrier();
+ }
+ #endif
+ ::operator delete(head_buffer);
+ com_id[com_id.size()-1]='\0'; //Ensure string is null terminated
+
+ /**************/
+ /* Check Head */
+ /**************/
+ if(std::strcmp(LINKTEST_COM_ID_,com_id.c_str())!=0){
+ fatal("%s string not read. Read: %s.",LINKTEST_COM_ID_,com_id.c_str());
+ }
+ if(version[0]!=LINKTEST_VERSION ){
+ fatal("LinkTest major version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[0],VERSION_MAJOR);
+ }
+ if(version[1]!=LINKTEST_VERSION_SUB ){
+ fatal("LinkTest major version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[1],VERSION_MINOR);
+ }
+ if(version[2]!=LINKTEST_VERSION_PATCHLEVEL){
+ fatal("LinkTest patch version mismatch when reading communication pattern! %" PRId32 "!=%" PRId32,version[2],VERSION_PATCH);
+ }
+ if(num_iter!=num_iterations){
+ fatal("Number of iterations mismatch, expected %" PRId32 " but found " PRId32,num_iterations,num_iter);
+ }
+
+ /************************/
+ /* Skip Past Iterations */
+ /************************/
+ //TODO: Keep offset from one iteration to the next
+ // so we can skip past the seek step.
+ std::uint8_t dtype;
+ std::uint64_t numel=0;
+ Serializer sblockhead=getSerializedCommunicationPatternBlockHead(&dtype,&numel);
+ const std::size_t block_head_size=sblockhead.getSize();
+ void* const block_head_buffer=::operator new(block_head_size);
+ rootWatch->start();
+ for(size_t i=1;i<iteration;i++){
+ if(sion_fread(block_head_buffer,block_head_size,1,file_handle)!=1){
+ ::operator delete(block_head_buffer);
+ error("Failed to read communication-pattern block header for iteration %zu from SION file.",i);
+ }
+ sblockhead.read((char*)block_head_buffer,cl->rank());
+ seek_offset+=block_head_size+numel*sizeof(std::uint64_t);
+ sion_seek(file_handle,SION_CURRENT_RANK,0,seek_offset);
+ }
+ rootWatch->stop();
+ printTiming("[sioncomseekpar]");
+
+ /**************************/
+ /* Read Current Iteration */
+ /**************************/
+ rootWatch->start();
+ if(sion_fread(block_head_buffer,block_head_size,1,file_handle)!=1){
+ ::operator delete(block_head_buffer);
+ error("Failed to read communication-pattern block header for iteration %zu from SION file.",iteration);
+ }
+ sblockhead.read((char*)block_head_buffer,cl->rank());
+ #ifdef DEBUG_COM_INPUT
+ for(int k=0;k<cl->size();k++){
+ if(cl->rank()==k){
+ printf("rank=%d, dtype=%" PRIu8 ", numel=%" PRIu64 "\n",cl->rank(),dtype,numel);
+ fflush(stdout);
+ }
+ cl->barrier();
+ }
+ #endif
+ std::vector<uint64_t> com;
+ switch(dtype){
+ case PatternFormatInvalid:
+ error("Detected invalid pattern format for communication patter %zu!",iteration);
+ break;
+ case PatternFormatCom:
+ com.resize(numel);
+ if(sion_fread(com.data(),sizeof(std::uint64_t),numel,file_handle)!=numel){
+ error("Failed to read communication-pattern for iteration %zu from SION file.",iteration);
+ }
+ break;
+ case PatternFormatMat:
+ /* We convert here to a com matrix (see case above), only to convert back at a later stage.
+ * This currently works better though with the program flow.
+ * TODO: Optimize this!
+ */
+ for(size_t i=0;i<num_iter;i++){
+ //Check if numel is equal to number of tasks
+ if(numel!=(std::uint64_t)cl->size()){
+ error("Communication-pattern matrix from SION file is %s!",numel>(uint64_t)cl->size()?"too large":"too small");
+ goto r;
+ }
+ //Load data
+ std::uint64_t mat[numel];
+ if(sion_fread(mat,sizeof(std::uint64_t),numel,file_handle)!=numel){
+ error("Failed to read communication-pattern for iteration %zu from SION file.",iteration);
+ break;
+ }
+ //Determine number of steps
+ std::uint64_t global_max_step=*std::max_element(&mat[0],&mat[numel-1]);
+ if(cl->rank()==0){
+ std::uint64_t local__max_step;
+ for(int j=1;j<cl->size();j++){
+ cl->recv(j,&local__max_step,1);
+ global_max_step=(global_max_step>local__max_step)?global_max_step:local__max_step;
+ }
+ cl->bcast(0,&global_max_step,1);
+ }else{
+ cl->send( 0,&global_max_step,1);
+ cl->bcast(0,&global_max_step,1);
+ }
+ //Convert to com
+ com.resize(global_max_step,0);
+ for(std::uint64_t j=0;j<(std::uint64_t)cl->size();j++){
+ const size_t partner=(std::size_t)mat[(std::size_t)j];
+ if(partner==0) continue;
+ com[partner-1]=j;
+ }
+ }
+ break;
+ default:
+ error("Detected unknown communication-pattern format for iteration %zu!",iteration);
+ break;
+ }
+ if(com.empty()!=false){
+ rootWatch->stop();
+ printTiming("[sioncomreadpar]");
+ }
+
+r: rootWatch->start();
+ sion_generic_parclose(file_handle);
+ rootWatch->stop();
+ printTiming("[sioncomclosepar]");
+ ::operator delete(block_head_buffer);
+ return com;
+ #else
+ error("This version of the benchmark was compiled without SIONlib support.");
+ return ERROR;
+ #endif
+}
diff --git a/benchmark/output_sion.h b/benchmark/output_sion.h
index bf9f287697c9ec4e4c743881c5fecfc797516980..b88de55aa47adf08ed676cade422d3a933718c53 100644
--- a/benchmark/output_sion.h
+++ b/benchmark/output_sion.h
@@ -33,11 +33,22 @@ int linktest_output_sion_funnelled(VirtualCluster* cl,
const struct linktest_args *args,
const std::vector<LinktestStats>& statsVec);
-/* Write in parallel using SIONlib generic API with the same transport layer as the VirtualCluster
+/* Write in parallel using SIONlib generic API with the same transport layer
+ as the VirtualCluster.
*/
int linktest_output_sion_parallel(VirtualCluster* cl,
const struct linktest_args *args,
const std::vector<LinktestStats>& statsVec);
+/* Determine number of iterations to run based on input SION file
+ */
+uint64_t get_num_iterations_from_pattern_sion(const struct linktest_args* const args, const int vcl_size);
+
+/* Read in parallel using SIONlib the communication pattern for a given iteration
+ */
+std::vector<uint64_t> sion_read_communication_pattern_for_iteration_parallel([[maybe_unused]] VirtualCluster* cl,
+ [[maybe_unused]] const struct linktest_args* const args,
+ [[maybe_unused]] const std::size_t num_iterations,
+ [[maybe_unused]] const std::size_t iteration);
#endif
diff --git a/benchmark/vcluster.cc b/benchmark/vcluster.cc
index 7e435218a52b078d083063118ab110a02ec5f649..4dbb1448c9eb2b8c816686c713d9118389e7fa0d 100644
--- a/benchmark/vcluster.cc
+++ b/benchmark/vcluster.cc
@@ -528,3 +528,8 @@ int VirtualCluster::write_funnelled(const linktest_args* args, const std::vector
{
return linktest_output_sion_funnelled(this, args, statsVec);
}
+
+std::vector<std::uint64_t> VirtualCluster::read_COM_parallel(const struct linktest_args* const args, const std::size_t num_iterations, const std::size_t iteration){
+ return sion_read_communication_pattern_for_iteration_parallel(this, args, num_iterations, iteration);
+}
+
diff --git a/benchmark/vcluster.h b/benchmark/vcluster.h
index 9da08970212e617c301e5d86b9d2991fe796358e..e013395251f10783ba34dff4aad4de3490963cd3 100644
--- a/benchmark/vcluster.h
+++ b/benchmark/vcluster.h
@@ -256,7 +256,7 @@ public:
const struct linktest_args* args,
double* time);
- /* I/O operations.
+ /* I/O operations: Part I
*
* - write_parallel: Write in parallel from all processing elements.
* - write_funnelled: Funnel the I/O through the first processing element.
@@ -267,6 +267,13 @@ public:
virtual int write_parallel(const linktest_args* args, const std::vector<LinktestStats>& statsVec);
virtual int write_funnelled(const linktest_args* args, const std::vector<LinktestStats>& statsVec);
+ /* I/O operations: Part II
+ *
+ * - read_COM_parallel: Reads in parallel the communication matrix for the current step.
+ */
+public:
+ virtual std::vector<std::uint64_t> read_COM_parallel(const struct linktest_args* const args, const std::size_t num_iterations, const std::size_t iteration);
+
/* Given the name of the vcluster implementation create an instance. This
* function accesses an internal database to map the name of the implementation
* to a function that creates the instance. In order for this to work, the
diff --git a/misc/MPI_COM_Check.cc b/misc/MPI_COM_Check.cc
new file mode 100644
index 0000000000000000000000000000000000000000..9b9e567dd8ea8a8c351447d2c29806874ce741c0
--- /dev/null
+++ b/misc/MPI_COM_Check.cc
@@ -0,0 +1,275 @@
+#include <stdio.h>
+#include <memory>
+#include <vector>
+#include <mpi.h>
+
+int main(int argc, char *argv[]){
+ int i, j, k, world_size, world_rank;
+ MPI_Status mpi_status ;
+ MPI_Request mpi_request;
+
+ /* Initialize MPI */
+ int ret=MPI_Init(NULL, NULL);
+ if (ret)return(ret);
+ ret=MPI_Comm_size(MPI_COMM_WORLD,&world_size);
+ if (ret)return(ret);
+ MPI_Comm_rank(MPI_COMM_WORLD,&world_rank);
+ if (ret)return(ret);
+ if(world_size%2){
+ if(world_rank==0){printf("This example requires an even number of tasks!\n");fflush(stdout);}
+ goto FIN_MPI;
+ }
+ if(world_size<3){
+ if(world_rank==0){printf("This example requires at least 4 tasks!\n");fflush(stdout);}
+ goto FIN_MPI;
+ }
+
+ {
+ /* Generate Communication Matrix */
+ // Would normally load each row for each task from disk in parallel
+ const int num_steps=(world_size==6)?3:((world_size==8)?22:(world_size-1));
+ if(world_rank==0){printf("num_steps=%d\n\n",num_steps);fflush(stdout);}
+ int* const com=(int*)malloc((num_steps)*sizeof(int));
+ if(world_size==6){ //Custom version for n=6 to show off custom com matrix and errors.
+ switch(world_rank){
+ case(0):
+ com[0]=1;com[1]=2;com[2]=3;
+ break;
+ case(1):
+ com[0]=0;com[1]=3;com[2]=2;
+ break;
+ case(2):
+ com[0]=3;com[1]=0;com[2]=1;
+ break;
+ case(3):
+ com[0]=2;com[1]=1;com[2]=0;
+ break;
+ case(4):
+ com[0]=5;com[1]=5;com[2]=4;
+ break;
+ case(5):
+ com[0]=4;com[1]=4;com[2]=5;
+ break;
+ }
+ /* Include Errors */
+ // Turn on to see effect
+ if(world_rank==4)com[1]=2;
+ if(world_rank==5)com[2]=666;
+ }else if(world_size==8){
+ switch(world_rank){ //Custom version for n=8 to show off faster option for serial 2-CPU node testing.
+ case(0):
+ com[ 0]=1;com[ 1]=2;com[ 2]=3;com[ 3]=0;com[ 4]=0;com[ 5]=0;com[ 6]=4;com[ 7]=5;
+ com[ 8]=6;com[ 9]=7;com[10]=0;com[11]=0;com[12]=0;com[13]=0;com[14]=0;com[15]=0;
+ com[16]=0;com[17]=0;com[18]=0;com[19]=0;com[20]=0;com[21]=0;
+ break;
+ case(1):
+ com[ 0]=0;com[ 1]=1;com[ 2]=1;com[ 3]=2;com[ 4]=3;com[ 5]=1;com[ 6]=1;com[ 7]=1;
+ com[ 8]=1;com[ 9]=1;com[10]=4;com[11]=5;com[12]=6;com[13]=7;com[14]=1;com[15]=1;
+ com[16]=1;com[17]=1;com[18]=1;com[19]=1;com[20]=1;com[21]=1;
+ break;
+ case(2):
+ com[ 0]=2;com[ 1]=0;com[ 2]=2;com[ 3]=1;com[ 4]=2;com[ 5]=3;com[ 6]=2;com[ 7]=2;
+ com[ 8]=2;com[ 9]=2;com[10]=2;com[11]=2;com[12]=2;com[13]=2;com[14]=4;com[15]=5;
+ com[16]=6;com[17]=7;com[18]=2;com[19]=2;com[20]=2;com[21]=2;
+ break;
+ case(3):
+ com[ 0]=3;com[ 1]=3;com[ 2]=0;com[ 3]=3;com[ 4]=1;com[ 5]=2;com[ 6]=3;com[ 7]=3;
+ com[ 8]=3;com[ 9]=3;com[10]=3;com[11]=3;com[12]=3;com[13]=3;com[14]=3;com[15]=3;
+ com[16]=3;com[17]=3;com[18]=4;com[19]=5;com[20]=6;com[21]=7;
+ break;
+ case(4):
+ com[ 0]=5;com[ 1]=6;com[ 2]=7;com[ 3]=4;com[ 4]=4;com[ 5]=4;com[ 6]=0;com[ 7]=4;
+ com[ 8]=4;com[ 9]=4;com[10]=1;com[11]=4;com[12]=4;com[13]=4;com[14]=2;com[15]=4;
+ com[16]=4;com[17]=4;com[18]=3;com[19]=4;com[20]=4;com[21]=4;
+ break;
+ case(5):
+ com[ 0]=4;com[ 1]=5;com[ 2]=5;com[ 3]=6;com[ 4]=7;com[ 5]=5;com[ 6]=5;com[ 7]=0;
+ com[ 8]=5;com[ 9]=5;com[10]=5;com[11]=1;com[12]=5;com[13]=5;com[14]=5;com[15]=2;
+ com[16]=5;com[17]=5;com[18]=5;com[19]=3;com[20]=5;com[21]=5;
+ break;
+ case(6):
+ com[ 0]=6;com[ 1]=4;com[ 2]=6;com[ 3]=5;com[ 4]=6;com[ 5]=7;com[ 6]=6;com[ 7]=6;
+ com[ 8]=0;com[ 9]=6;com[10]=6;com[11]=6;com[12]=1;com[13]=6;com[14]=6;com[15]=6;
+ com[16]=2;com[17]=6;com[18]=6;com[19]=6;com[20]=3;com[21]=6;
+ break;
+ case(7):
+ com[ 0]=7;com[ 1]=7;com[ 2]=4;com[ 3]=7;com[ 4]=5;com[ 5]=6;com[ 6]=7;com[ 7]=7;
+ com[ 8]=7;com[ 9]=0;com[10]=7;com[11]=7;com[12]=7;com[13]=1;com[14]=7;com[15]=7;
+ com[16]=7;com[17]=2;com[18]=7;com[19]=7;com[20]=7;com[21]=3;
+ break;
+ }
+ /* Include Errors */
+ // Turn on to see effect
+ if(world_rank==4)com[1]=2;
+ if(world_rank==5)com[2]=666;
+ }else{
+ //Default 1 Factor algorithm
+ for(i=0;i<(num_steps);i++){
+ const int idle=(i*world_size/2)%(num_steps);
+ if(world_rank==(num_steps)) com[i]=idle;
+ else if(world_rank==idle) com[i]=num_steps;
+ else com[i]=(i-world_rank+num_steps)%(num_steps);
+ }
+ /* Include Errors */
+ // Turn on to see effect
+ if(world_rank==0) com[2]=1;
+ if(world_rank==1) com[0]=666;
+ }
+
+ /* Print Communication Matrix */
+ for(i=0;i<world_size;i++){
+ if(i==world_rank){
+ printf("%d: %d",world_rank,com[0]);
+ for(j=1;j<(num_steps);j++) printf(", %d",com[j]);
+ printf("\n");
+ fflush(stdout);
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+ if(world_rank==0){
+ printf("\n");
+ fflush(stdout);
+ }
+
+ /* Bin Communication Matrix According To Partner */
+ std::vector<int>* con=new std::vector<int>[world_size];
+ for(i=0;i<world_size;i++){
+ for(j=0;j<(num_steps);j++){
+ if(com[j]==i) con[i].push_back(j);
+ }
+ }
+
+ /* Print Connections Per Pair For Each Rank */
+ for(i=0;i<world_size;i++){
+ if(i==world_rank){
+ for(j=0;j<world_size;j++){
+ const int tmp=con[j].size();
+ if(j==i){
+ printf("%d does not communicate in %d steps",world_rank,tmp);
+ if(tmp){
+ printf(": %d",con[j][0]);
+ for(k=1;k<tmp;k++){
+ printf(", %d",con[j][k]);
+ }
+ }else printf(".");
+ }else{
+ printf("%d communicates with %d in %d steps",world_rank,j,tmp);
+ if(tmp){
+ printf(": %d",con[j][0]);
+ for(k=1;k<tmp;k++){
+ printf(", %d",con[j][k]);
+ }
+ }else printf(".");
+ }
+ printf("\n");
+ }
+ printf("\n");
+ fflush(stdout);
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+ if(world_rank==0){
+ printf("\n");
+ fflush(stdout);
+ }
+
+ /* Test Pairs */
+ std::vector<std::unique_ptr<int[]>> bc;
+ // Check Bounds
+ k=0;
+ for(i=0;i<num_steps;i++){
+ if(com[i]>=world_size){
+ std::unique_ptr<int[]> tmp(new int[3]{world_rank,com[i],i});
+ bc.push_back(std::move(tmp));
+ k++;
+ }
+ }
+ // Check Partners
+ std::vector<int> steps;
+ for(i=1;i<world_size;i++){
+ const int recv_partner=(world_size+world_rank-i)%world_size;
+ const int send_partner=(world_rank+i)%world_size;
+ int recv_size;
+ const int send_size=con[send_partner].size();
+ int sync_send;
+ const int sync_recv=0;
+ //Exchange Sizes
+ MPI_Isend(&send_size,1,MPI::INT,send_partner,send_partner,MPI_COMM_WORLD,&mpi_request);
+ MPI_Recv(&recv_size,1,MPI::INT,recv_partner,world_rank,MPI_COMM_WORLD,&mpi_status);
+ MPI_Isend(&sync_recv,1,MPI::INT,recv_partner,recv_partner,MPI_COMM_WORLD,&mpi_request);
+ steps.reserve(recv_size);
+ MPI_Recv(&sync_send,1,MPI::INT,send_partner,world_rank,MPI_COMM_WORLD,&mpi_status);
+ //Send and receive data
+ MPI_Isend(con[send_partner].data(),send_size,MPI::INT,send_partner,send_partner,MPI_COMM_WORLD,&mpi_request);
+ MPI_Recv(steps.data(),recv_size,MPI::INT,recv_partner,world_rank,MPI_COMM_WORLD,&mpi_status);
+ //Test Data
+ for(j=0;j<recv_size;j++){
+ if(com[steps[j]]!=recv_partner){
+ std::unique_ptr<int[]> tmp(new int[3]{recv_partner,world_rank,steps[j]});
+ bc.push_back(std::move(tmp));
+ k++;
+ }
+ }
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Print Bad Connections */
+ for(i=0;i<world_size;i++){
+ if(i==world_rank){
+ printf("%d identified the following %d bad connections:",world_rank,k);
+ for(j=0;j<k;j++){
+ printf(" %d->%d(%d)",bc[j][0],bc[j][1],bc[j][2]);
+ }
+ printf("\n");
+ }
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ if(world_rank==0){
+ printf("\n");
+ fflush(stdout);
+ }
+
+ /* Collect Bad Connections */
+ if(world_rank==0){
+ int recv_size;
+ for(i=1;i<world_size;i++){
+ const int sync_recv=0;
+ MPI_Recv(&recv_size,1,MPI::INT,i,i,MPI_COMM_WORLD,&mpi_status);
+ MPI_Isend(&sync_recv,1,MPI::INT,i,i,MPI_COMM_WORLD,&mpi_request);
+ for(j=0;j<recv_size;j++){
+ std::unique_ptr<int[]> buf(new int[3]);
+ MPI_Recv(buf.get(),3,MPI::INT,i,i,MPI_COMM_WORLD,&mpi_status);
+ bc.push_back(std::move(buf));
+ k++;
+ }
+ }
+ }else{
+ int sync_recv;
+ for(i=1;i<world_size;i++){
+ if(i==world_rank){
+ MPI_Isend(&k,1,MPI::INT,0,i,MPI_COMM_WORLD,&mpi_request);
+ MPI_Recv(&sync_recv,1,MPI::INT,0,i,MPI_COMM_WORLD,&mpi_status);
+ for(j=0;j<k;j++){
+ MPI_Send(bc[j].get(),3,MPI::INT,0,i,MPI_COMM_WORLD);
+ }
+ }
+ }
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Print Bad Connections Again */
+ if(world_rank==0){
+ printf("Bad Connections:\n");
+ for(i=0;i<k;i++){
+ printf("%d->%d (step: %d)\n",bc[i][0],bc[i][1],bc[i][2]);
+ }
+ }
+
+ /* Free Data */
+ free(com);
+ delete[] con;
+ }
+
+ /* Finalize MPI Environment. */
+ FIN_MPI: MPI_Finalize();
+}
\ No newline at end of file
diff --git a/misc/build.bash b/misc/build.bash
new file mode 100755
index 0000000000000000000000000000000000000000..cb1e09d75cd28f33578979e180e76fcb0afd348e
--- /dev/null
+++ b/misc/build.bash
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# The following function will generate communication pattern that you can load into LinkTest and use.
+mpicc -Wall -O2 -D__USE_POSIX -D_FILE_OFFSET_BITS=64 -I<<<PATH TO SIONlib INSTALL>>>/sionlib_linux_gomp/include -D_SION_LINUX -DSION_MPI generateLinkTestComPattern.c -L<<<PATH TO SIONlib INSTALL>>>/sionlib_linux_gomp/lib -lsionmpi_64 -lsiongen_64 -lsionser_64 -lsioncom_64 -L/usr/local/lib -lsionfwd-client -pthread /usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so -lsioncom_64_lock_none -o generateLinkTestComPattern.exe
+# execute: mpirun -n <<<NUMBER OF RANKS>>> ./generateLinkTestComPattern.exe
+
+# The following function is an example of how to check LinkTest communication patterns
+mpicxx MPI_COM_Check.cc -o MPI_COM_Check.exe
+# execute: mpirun -n <<<NUMBER OF RANKS>>> ./MPI_COM_Check.exe
+
+# The foll
+gcc -O2 permutationTester.c -o permutationTester.exe
+# execute: ./permutationTester.exe
+
diff --git a/misc/generateLinkTestComPattern.c b/misc/generateLinkTestComPattern.c
new file mode 100644
index 0000000000000000000000000000000000000000..91e06acb011e2cb093d2bb2f9959680d6f11ea42
--- /dev/null
+++ b/misc/generateLinkTestComPattern.c
@@ -0,0 +1,188 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+#include <mpi.h>
+#include "sion.h"
+
+#define LINKTEST_COM_ID "LKTST_COM"
+#define END_BLOCK "End_Block"
+
+/* This is an example script how to generate communication matrices so that they can
+ * be loaded into LinkTest.
+ *
+ * To use this script call it using the number of tasks that you wish to use in
+ * LinkTest. This will the create a communication pattern for that amount of tasks.
+ *
+ * You can choose to save the communication pattern as either a rank-by-step matrix
+ * where the value indicates the communication partner in said step or as a rank-by-
+ * rank matrix in which the value indicates the step in which the communication
+ * between the two is tested. You can set this a the beginning of the main function
+ * by setting "dType" to either "DataTypeCom" or "DataTypeMat". "DataTypePer" is
+ * currently not supported by LinkTest.
+ *
+ * TODO: Implement DataTypePer both here and in LinkTest.
+ */
+
+
+enum DataType{
+ DataTypeInvalid=0, //Invalid Data Type
+ DataTypeCom =1, //COM matrix : Rank-by-Step (Value: Partner)
+ DataTypeMat =2, //Communication Matrix: Rank-by-Partner (Value: Step )
+ DataTypePer =3, //Permutation Vector for Permuting 1-Factor Output TODO: CURRENTLY NOT SUPPORTED IN LINKTEST
+};
+
+struct Item{
+ const void* data;
+ size_t size;
+ struct Item *next;
+};
+
+struct ItemList_s{
+ size_t num_items;
+ size_t total_size;
+ struct Item* first_item;
+ struct Item* last_item;
+};
+typedef struct ItemList_s ItemList;
+
+int append(ItemList* const itemList,const void* const data,const size_t size){
+ if(itemList->num_items==0){
+ itemList->first_item=(struct Item*)malloc(sizeof(struct Item));
+ if(itemList->first_item==NULL) return -1;
+ itemList->last_item=itemList->first_item;
+ }else{
+ itemList->last_item->next=(struct Item*)malloc(sizeof(struct Item));
+ if(itemList->last_item->next==NULL) return -1;
+ itemList->last_item=itemList->last_item->next;
+ }
+ itemList->last_item->data=data;
+ itemList->last_item->size=size;
+ itemList->num_items++;
+ itemList->total_size+=size;
+ return(0);
+}
+
+void* createBuffer(const ItemList* const itemList){
+ struct Item* item=itemList->first_item;
+ void* const buf=(void*)malloc(itemList->total_size);
+ void* ptr=buf;
+
+ for(size_t i=0;i<itemList->num_items;i++){
+ ptr=memcpy(ptr,item->data,item->size)+item->size;
+ item=item->next;
+ }
+
+ return(buf);
+}
+
+void freeList(const ItemList* const itemList){
+ struct Item* item=itemList->first_item;
+ struct Item* next;
+ for(size_t i=0;i<itemList->num_items;i++){
+ next=item->next;
+ free(item);
+ item=next;
+ }
+}
+
+int main(int argc, char** argv){
+ const uint8_t dType =DataTypeMat;
+ int world_size, world_rank;
+
+ /* Initialize MPI */
+ int ret=MPI_Init(NULL, NULL);
+ if (ret)return(ret);
+ ret=MPI_Comm_size(MPI_COMM_WORLD,&world_size);
+ if (ret)return(ret);
+ MPI_Comm_rank(MPI_COMM_WORLD,&world_rank);
+ if (ret)return(ret);
+ if(world_size%2){
+ if(world_rank==0){printf("This example requires an even number of tasks!\n");fflush(stdout);}
+ goto FIN_MPI;
+ }
+
+ /* Generate Default COM */
+ const uint64_t num_ranks=(uint64_t)world_size;
+ const uint64_t num_steps=num_ranks-1;
+ uint64_t* com=(uint64_t*)malloc(num_steps*sizeof(uint64_t));
+ for(size_t i=0;i<num_steps;i++){
+ const uint64_t idle=(i*world_size/2)%num_steps;
+ if(world_rank==num_steps) com[i]=idle;
+ else if((uint64_t)world_rank==idle) com[i]=num_steps;
+ else com[i]=(i-world_rank+num_steps)%num_steps;
+ }
+
+ uint8_t tmp;
+ MPI_Status mpi_status;
+ if(world_rank==0) printf("COM Matrix:\n");
+ else MPI_Recv(&tmp,0,MPI_UINT8_T,world_rank-1,0,MPI_COMM_WORLD,&mpi_status);
+ printf("COM %d: ",world_rank);
+ for(size_t i=0;i<num_steps;i++) printf("%" PRIu64 ",",com[i]);
+ printf("\n");
+ fflush(stdout);
+ if(world_rank<world_size-1) MPI_Send(&tmp,0,MPI_UINT8_T,world_rank+1,0,MPI_COMM_WORLD);
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ uint64_t* mat=(dType==DataTypeMat)?(uint64_t*)calloc((size_t)world_size,sizeof(uint64_t)):NULL;
+ if(dType==DataTypeMat){
+ for(size_t i=0;i<num_steps;i++) mat[com[i]]=i+1;
+
+ if(world_rank==0) printf("Communication Matrix:\n");
+ else MPI_Recv(&tmp,0,MPI_UINT8_T,world_rank-1,0,MPI_COMM_WORLD,&mpi_status);
+ printf("MAT %d: ",world_rank);
+ for(size_t i=0;i<num_ranks;i++) printf("%" PRIu64 ",",mat[i]);
+ printf("\n");
+ fflush(stdout);
+ if(world_rank<world_size-1) MPI_Send(&tmp,0,MPI_UINT8_T,world_rank+1,0,MPI_COMM_WORLD);
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ /* Prepare Buffer */
+ const char LinkTest_Com_ID[]=LINKTEST_COM_ID;
+ const uint32_t version[3] ={2, 1, 17};
+ const uint64_t num_iter =1;
+ const char End_Block[] =END_BLOCK;
+ ItemList itemList={.num_items=0,.total_size=0,.first_item=NULL,.last_item=NULL};
+ append(&itemList, LinkTest_Com_ID,(strlen(LinkTest_Com_ID)+1)*sizeof(char ));
+ append(&itemList, version , 3*sizeof(uint32_t));
+ append(&itemList,&num_iter , sizeof(uint64_t));
+ append(&itemList,&dType , sizeof(uint8_t ));
+ if(dType==DataTypeMat){
+ append(&itemList,&num_ranks, sizeof(uint64_t));
+ append(&itemList, mat ,num_ranks*sizeof(uint64_t));
+ }else{
+ append(&itemList,&num_steps, sizeof(uint64_t));
+ append(&itemList, com ,num_steps*sizeof(uint64_t));
+ }
+ append(&itemList, End_Block ,(strlen(End_Block )+1)*sizeof(char ));
+
+ /* Create Buffer */
+ void* const buffer=createBuffer(&itemList);
+
+ /* for SIONlib open call */
+ char fname[] ="COM.sion";
+ int numfiles =1;
+ const MPI_Comm gcomm =MPI_COMM_WORLD;
+ const MPI_Comm lcomm =MPI_COMM_WORLD;
+ sion_int64 chunksize =itemList.total_size;
+ sion_int32 fsblksize =-1;
+ FILE* fileptr =NULL;
+ char* newfname =NULL;
+
+ // Write Data;
+ int sid=sion_paropen_mpi(fname,"bw",&numfiles,gcomm,&lcomm,&chunksize,&fsblksize,&world_rank,&fileptr,&newfname);
+ if(sion_fwrite(buffer,itemList.total_size,1,sid)!=1){
+ fprintf(stderr,"ERROR: Rank %d failed to write buffer.\n",world_rank);
+ }
+ sion_parclose_mpi(sid);
+
+ //Free Data
+ free(buffer);
+ free(com);
+ if(mat!=NULL) free(mat);
+ freeList(&itemList);
+
+ /* Finalize MPI Environment. */
+ FIN_MPI: MPI_Finalize();
+}
diff --git a/misc/permutationTester.c b/misc/permutationTester.c
new file mode 100644
index 0000000000000000000000000000000000000000..8307f84eb3c268aa1d1284473e2935f6f968ca2b
--- /dev/null
+++ b/misc/permutationTester.c
@@ -0,0 +1,53 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+void permute(const int n,int* const vec){
+ int i,help,pos;
+ for(i=0;i<n;i++)vec[i]=i;
+ for(i=0;i<n;i++){
+ pos=(int)((double)rand()*(double)(n)/(double)RAND_MAX);
+ help=vec[i];
+ vec[i]=vec[pos];
+ vec[pos]=help;
+ }
+}
+
+void test(const int niter, const int n, int** const vecs){
+ int i,j,k,l;
+ l=0;
+ for(i=0;i<niter;i++){
+ for(j=i+1;j<niter;j++){
+ int tst=1;
+ for(k=0;k<n;k++){
+ if(vecs[i][k]!=vecs[j][k]){tst=0;break;}
+ }
+ if(tst){
+ printf("Permutations %d and %d are equal!\n",i,j);
+ l++;
+ }
+ }
+ }
+ printf("%d repeated permutations detected!\n",l);
+}
+
+int main(){
+ int i,j;
+ const int n=100000; //Should not be zero!
+ const int niter=30000;
+ int** const vecs=(int**)malloc(niter*sizeof(int*));
+
+ //Set SEED
+ srand(1337);
+
+ //Generate Permutations
+ for(i=0;i<niter;i++){
+ vecs[i]=(int*)malloc(n*sizeof(int));
+ permute(n,vecs[i]);
+ }
+
+ // Test Permutations
+ test(niter,n,vecs);
+
+ for(i=0;i<niter;i++) free(vecs[i]);
+ free(vecs);
+}
\ No newline at end of file