diff --git a/benchmark/vcluster_portals.cc b/benchmark/vcluster_portals.cc index c41dc4db957a8e7043bd6cc9f2395eb9c39ecdb0..6efbe632c0bc44e2121f1f8eb4175bd14782c52c 100644 --- a/benchmark/vcluster_portals.cc +++ b/benchmark/vcluster_portals.cc @@ -72,7 +72,7 @@ int VirtualClusterPortals::finalize() return SUCCESS; } -void VirtualClusterPortals::prepareSendStructs(const MemoryBuffer& buf, const int num_msg) { +ptl_size_t VirtualClusterPortals::prepareSendStructs(const MemoryBuffer& buf, const int num_msg) { // send_ct.success = 0; // send_ct.failure = 0; // CHECK_RETURNVAL( PtlCTSet(md_ct_handle, send_ct) ); @@ -84,14 +84,14 @@ void VirtualClusterPortals::prepareSendStructs(const MemoryBuffer& buf, const in md.eq_handle = md_eq_handle; md.ct_handle = md_ct_handle; CHECK_RETURNVAL( PtlMDBind(mni_handle, &md, &md_handle) ); // Bind memory descriptor, aka. ??? TODO - - #if defined(DEBUG_PORTALS) CHECK_RETURNVAL( PtlCTGet(md_ct_handle, &send_ct) ); - info("Send: before success %d - failure %d", recv_ct.success, recv_ct.failure); + #if defined(DEBUG_PORTALS) + info("Send: before success %d - failure %d", send_ct.success, send_ct.failure); #endif + return send_ct.success; } -void VirtualClusterPortals::prepareRecvStructs(const MemoryBuffer& buf) { +ptl_size_t VirtualClusterPortals::prepareRecvStructs(const MemoryBuffer& buf) { // recv_ct.success = 0; // recv_ct.failure = 0; // CHECK_RETURNVAL( PtlCTSet(me_ct_handle, recv_ct) ); @@ -105,21 +105,21 @@ void VirtualClusterPortals::prepareRecvStructs(const MemoryBuffer& buf) { me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_CT_COMM; me.ct_handle = me_ct_handle; CHECK_RETURNVAL( PtlMEAppend(mni_handle, pt_index, &me, PTL_PRIORITY_LIST, nullptr, &me_handle) ); // Append match entry, aka. allow puts to buf from anyone - - #if defined(DEBUG_PORTALS) CHECK_RETURNVAL( PtlCTGet(me_ct_handle, &recv_ct) ); + #if defined(DEBUG_PORTALS) info("Recv: before success %d - failure %d", recv_ct.success, recv_ct.failure); #endif + return recv_ct.success; } -void VirtualClusterPortals::recvMessages(const int num_msg) { - CHECK_RETURNVAL( PtlCTWait(me_ct_handle, num_msg, &recv_ct) ); +void VirtualClusterPortals::recvMessages(const int num_msg, ptl_size_t start_count) { + CHECK_RETURNVAL( PtlCTWait(me_ct_handle, start_count + num_msg, &recv_ct) ); #if defined(DEBUG_PORTALS) info("Recv: after success %d - failure %d", recv_ct.success, recv_ct.failure); #endif } -void VirtualClusterPortals::sendMessages(const int to, MemoryBuffer& buf, const int num_msg, double* const timing) { +void VirtualClusterPortals::sendMessages(const int to, MemoryBuffer& buf, const int num_msg, ptl_size_t start_count, double* const timing) { ptl_process_t target; target.rank = to; const ptl_size_t localOffset = 0; @@ -130,7 +130,7 @@ void VirtualClusterPortals::sendMessages(const int to, MemoryBuffer& buf, const for(auto n = 1; n <= num_msg; n++) { CHECK_RETURNVAL( PtlPut(md_handle, localOffset, buf.len(), PTL_CT_ACK_REQ, target, pt_index, match_bits, remoteOffset, nullptr, header_data) ); } - CHECK_RETURNVAL( PtlCTWait(md_ct_handle, num_msg, &send_ct) ); + CHECK_RETURNVAL( PtlCTWait(md_ct_handle, start_count + num_msg, &send_ct) ); sendTime = walltime() - sendTime; #if defined(DEBUG_PORTALS) info("Send: after success %d - failure %d", send_ct.success, send_ct.failure); @@ -211,11 +211,14 @@ int VirtualClusterPortals::testPut() int VirtualClusterPortals::kpingpong(const int from, const int to, MemoryBuffer& buf, const int num_msg, double* const timing) { - if (rank() == from) prepareSendStructs(buf, num_msg); - if (rank() == to) prepareRecvStructs(buf); + ptl_size_t success_count; + if (rank() == from) success_count = prepareSendStructs(buf, num_msg); + if (rank() == to) success_count = prepareRecvStructs(buf); barrier(); - if (rank() == from) sendMessages(to, buf, num_msg, timing); - if (rank() == to) recvMessages(num_msg); + if (rank() == from) sendMessages(to, buf, num_msg, success_count, timing); + if (rank() == to) recvMessages(num_msg, success_count); + + info("kpingpong ended"); return SUCCESS; diff --git a/benchmark/vcluster_portals.h b/benchmark/vcluster_portals.h index 9e75163370ca031bc35b6f93ebb4162dd975d9ec..7fb630905c062afb5395ebdc4bcca07686f50ba5 100644 --- a/benchmark/vcluster_portals.h +++ b/benchmark/vcluster_portals.h @@ -75,7 +75,7 @@ private: ptl_handle_eq_t md_eq_handle; /** @brief Send Counter */ ptl_ct_event_t send_ct; - void prepareSendStructs(const MemoryBuffer& buf, const int num_msg); + ptl_size_t prepareSendStructs(const MemoryBuffer& buf, const int num_msg); // Receiver only /** @brief Match List Entry (ME) */ @@ -86,10 +86,10 @@ private: ptl_handle_ct_t me_ct_handle; /** @brief Receive Counter */ ptl_ct_event_t recv_ct; - void prepareRecvStructs(const MemoryBuffer& buf); + ptl_size_t prepareRecvStructs(const MemoryBuffer& buf); - void sendMessages(const int to, MemoryBuffer& buf, const int num_msg, double* const timing); - void recvMessages(const int num_msg); + void sendMessages(const int to, MemoryBuffer& buf, const int num_msg, ptl_size_t start_count, double* const timing); + void recvMessages(const int num_msg, ptl_size_t start_count); bool first = true; int testPut();