diff --git a/maestro/ofi.c b/maestro/ofi.c index ce5ab162c4b0b5e67e8809c5845e8ce6e381655a..3e636d5e360e0d192aa63b136615b61b8619c536 100644 --- a/maestro/ofi.c +++ b/maestro/ofi.c @@ -5,7 +5,7 @@ /* * Copyright (C) 2019 Cray Computer GmbH - * Copyright (C) 2020-2021 Hewlett-Packard (Schweiz) GmbH + * Copyright (C) 2020-2022 Hewlett-Packard (Schweiz) GmbH * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -60,6 +60,14 @@ #include "../transport/transport.h" #include "../transport/transport_rdma.h" #include <rdma/fabric.h> +/* until CXI hits upstream: */ +#ifndef FI_ADDR_CXI +#define FI_ADDR_CXI (FI_ADDR_PSMX3 + 1) +#endif +#ifndef FI_PROTO_CXI +#define FI_PROTO_CXI (FI_PROTO_PSMX3 + 1) +#endif + #include <rdma/fi_cm.h> #include <rdma/fi_domain.h> #include <rdma/fi_errno.h> @@ -725,6 +733,16 @@ mstro_epd_to_ofi_addr(const Mstro__Endpoint *epd, } break; } + case MSTRO__OFI_ADDR__VAL_CXI: { + if(*addrlen<sizeof(uint32_t)) { + *addrlen = sizeof(uint32_t); + return MSTRO_NOMEM; + } else { + memcpy(addr, &a->cxi->raw, sizeof(uint32_t)); + *addr_format = FI_ADDR_CXI; + } + break; + } case MSTRO__OFI_ADDR__VAL_BGQ: case MSTRO__OFI_ADDR__VAL_MLX: @@ -1220,12 +1238,13 @@ mstro_ofi__fi_info_cmp(const struct fi_info *x1, /* newer versions may introduce unknown providers, so check here: */ assert(FI_VERSION_GE(FI_VERSION(1,14), FI_VERSION(FI_MAJOR_VERSION,FI_MINOR_VERSION))); -#define MSTRO__FI_PROTO_MAX FI_PROTO_PSMX3 +#define MSTRO__FI_PROTO_MAX FI_PROTO_CXI /* provider order: sorted by preference */ const int proto_order[MSTRO__FI_PROTO_MAX+1] = { [FI_PROTO_SHM] = 0, - [FI_PROTO_IB_UD] = 5, - [FI_PROTO_GNI] = 10, + [FI_PROTO_GNI] = 5, + [FI_PROTO_CXI] = 7, + [FI_PROTO_IB_UD] = 10, [FI_PROTO_IB_RDM] = 50, [FI_PROTO_RXM] = 70, [FI_PROTO_UDP] = 90, @@ -1789,38 +1808,65 @@ mstro_ofi_init(void) if(retstat!=MSTRO_OK) { WARN("Failed to build EP %zu, trying others\n", i); - } else { - DEBUG("%zu. usable endpoint: %s\n", - g_endpoints->size, - mstro_endpoint_describe(&g_endpoints->eps[g_endpoints->size])); - - /* also register a region for incoming data */ - struct fid_mr *pm_info_mr = NULL; - - const struct fi_info* fi = g_endpoints->eps[g_endpoints->size].fi; + goto TRY_NEXT_EP; + } - if((fi->domain_attr->mr_mode & FI_MR_ENDPOINT)) { - ERR("FIXME: missing support for FI_MR_ENDPOINT domains -- FIXME\n"); + DEBUG("%zu. usable endpoint: %s\n", + g_endpoints->size, + mstro_endpoint_describe(&g_endpoints->eps[g_endpoints->size])); + + /* also register a region for incoming data */ + struct fid_mr *pm_info_mr = NULL; + + const struct fi_info* fi = g_endpoints->eps[g_endpoints->size].fi; + + if((fi->domain_attr->mr_mode & FI_MR_ENDPOINT)) { + WARN("FIXME: missing support for FI_MR_ENDPOINT domains in some regions -- FIXME\n"); + } + + uint64_t requested_key = fi->domain_attr->mr_mode & FI_MR_PROV_KEY ? 0 : mstro_memory_new_key(); + int ret = fi_mr_reg(g_endpoints->eps[g_endpoints->size].domain, + &g_pm_component_descriptor, sizeof(g_pm_component_descriptor), + FI_READ, 0, requested_key, 0, &pm_info_mr, NULL); + if(ret<0) { + ERR("Failed to register component descriptor read buffer on domain of ep %zu, skipping: %d (%s)\n", + g_endpoints->size, ret, fi_strerror(-ret)); + // drop memlock that ep_build_from_ofi did + retstat = mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor)); + goto TRY_NEXT_EP; + } + + DEBUG("registered peer buffer for RDMA, addr %p, mr 0x%" PRIx64 ", desc %p\n", + &g_pm_component_descriptor, pm_info_mr, fi_mr_desc(pm_info_mr)); + if((fi->domain_attr->mr_mode & FI_MR_ENDPOINT)) { + /* bind and enable mreg */ + ret = fi_mr_bind(pm_info_mr, (struct fid *)g_endpoints->eps[g_endpoints->size].ep, 0); + if(ret<0) { + ERR("Failed to bind PM-INFO mreg to endpoint %zu, skipping: %d (%s)\n", + g_endpoints->size, ret, fi_strerror(-ret)); + ret = fi_close((struct fid *)pm_info_mr); + if(ret<0) { + ERR("Failed to close PM-info mreg: %d (%s)\n", ret, strerror(-ret)); + retstat=MSTRO_FAIL; + } + retstat = mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor)); + goto TRY_NEXT_EP; } - uint64_t requested_key = fi->domain_attr->mr_mode & FI_MR_PROV_KEY ? 0 : mstro_memory_new_key(); - int ret = fi_mr_reg(g_endpoints->eps[g_endpoints->size].domain, - &g_pm_component_descriptor, sizeof(g_pm_component_descriptor), - FI_READ, 0, requested_key, 0, &pm_info_mr, NULL); + ret = fi_mr_enable(pm_info_mr); if(ret<0) { - ERR("Failed to register component descriptor read buffer on domain of ep %zu, skipping : %d (%s)\n", + ERR("Failed to enable PM-info mreg on endpoint %zu: %d (%s)\n", g_endpoints->size, ret, fi_strerror(-ret)); - // drop memlock that ep_build_from_ofi did - retstat = mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor)); - } else { - DEBUG("registered peer buffer for RDMA, addr %p, mr 0x%" PRIx64 ", desc %p\n", - &g_pm_component_descriptor, pm_info_mr, fi_mr_desc(pm_info_mr)); - g_endpoints->eps[g_endpoints->size].peer_info_mr = pm_info_mr; - g_endpoints->eps[g_endpoints->size].next - = &g_endpoints->eps[g_endpoints->size+1]; - g_endpoints->size++; + goto TRY_NEXT_EP; } } + + g_endpoints->eps[g_endpoints->size].peer_info_mr = pm_info_mr; + g_endpoints->eps[g_endpoints->size].next + = &g_endpoints->eps[g_endpoints->size+1]; + g_endpoints->size++; + TRY_NEXT_EP: + ; } /* fix pointer into invalid last entry */ g_endpoints->eps[g_endpoints->size-1].next = NULL; diff --git a/protocols/maestro-endpoints.c b/protocols/maestro-endpoints.c index 408ffbbe94b87dd93cf453ae8837bd611f5ca113..6ecdbfb4e2f69f974d52a9affbe12b4112e37316 100644 --- a/protocols/maestro-endpoints.c +++ b/protocols/maestro-endpoints.c @@ -3,7 +3,7 @@ ** @brief Maestro Endpoint (De-)Serialization **/ /* - * Copyright (C) 2021 Hewlett-Packaged (Schweiz) GmbH + * Copyright (C) 2021-2022 Hewlett-Packard (Schweiz) GmbH * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -118,6 +118,8 @@ mstro_ep_fi_to_ep(const struct fi_info *fi, struct fid_ep *ep, me->ofiproto = MSTRO__OFI_ENDPOINT_KIND__EFA; break; case FI_PROTO_PSMX3: me->ofiproto = MSTRO__OFI_ENDPOINT_KIND__PSMX3; break; + case FI_PROTO_CXI: + me->ofiproto = MSTRO__OFI_ENDPOINT_KIND__CXI; break; default: ERR("Unsupported OFI protocol: %d (%s)\n", fi->ep_attr->protocol, @@ -324,6 +326,33 @@ mstro_ep_fi_to_ep(const struct fi_info *fi, struct fid_ep *ep, addr->str = buf; break; } + case FI_ADDR_CXI: { + uint32_t buf; + size_t addrlen=sizeof(uint32_t); + + int r = fi_getname(&ep->fid, &buf, &addrlen); + if(r!=0 && r!=-FI_ETOOSMALL) { + ERR("Error obtaining the endpoint name (=local addr): %d (%s)\n", + r, fi_strerror(-r)); + s=MSTRO_FAIL; + goto BAILOUT; + } + if(r==-FI_ETOOSMALL) { + ERR("Unexpected CXI addrlen: %d (expected: %d)\n", addrlen, sizeof(uint32_t)); + s=MSTRO_UNIMPL; + goto BAILOUT; + } + addr->val_case = MSTRO__OFI_ADDR__VAL_CXI; + addr->cxi = malloc(sizeof(Mstro__AddrCXI)); + if(addr->cxi==NULL) { + s=MSTRO_NOMEM; + goto BAILOUT; + } else { + mstro__addr_cxi__init(addr->cxi); + addr->cxi->raw = buf; + } + break; + } /* unimplemented: */ case FI_SOCKADDR_IB: @@ -429,6 +458,20 @@ mstro_ep__addr_describe(const Mstro__Endpoint *ep, char *buf, size_t buflen) case MSTRO__OFI_ADDR__VAL_STR: num_written = snprintf(buf,unabbrev_len,"%s", ep->ofiaddr->str); break; + case MSTRO__OFI_ADDR__VAL_CXI: { + union cxi_addr_ { + struct { + uint32_t pid : 9; + uint32_t nic : 20; + uint32_t valid : 1; + }; + uint32_t raw; + }; + union cxi_addr_ tmp = { .raw = ep->ofiaddr->cxi->raw }; + + num_written = snprintf(buf,unabbrev_len,"%d:%d (%d)", tmp.nic, tmp.nic, tmp.valid); + break; + } case MSTRO__OFI_ADDR__VAL_SOCK: case MSTRO__OFI_ADDR__VAL_IB: diff --git a/protocols/maestro-endpoints.h b/protocols/maestro-endpoints.h index 2dc479d4562706e8c06478c53c6e160047b7065b..22b430da24ec9b0ddaab62731cdd1ad0ce07d862 100644 --- a/protocols/maestro-endpoints.h +++ b/protocols/maestro-endpoints.h @@ -38,6 +38,14 @@ #include "rdma/fabric.h" +/* until CXI hits upstream: */ +#ifndef FI_ADDR_CXI +#define FI_ADDR_CXI (FI_ADDR_PSMX3 + 1) +#endif +#ifndef FI_PROTO_CXI +#define FI_PROTO_CXI (FI_PROTO_PSMX3 + 1) +#endif + #include "rdma/fi_cm.h" #include "maestro/i_base64.h" diff --git a/protocols/mstro_ep.pb-c.c b/protocols/mstro_ep.pb-c.c index c66ec4e5440a58f4bac936e0f9fee4825d66138c..cd8c371487c739a237c9fb751191fda6f2481c0e 100644 --- a/protocols/mstro_ep.pb-c.c +++ b/protocols/mstro_ep.pb-c.c @@ -367,6 +367,51 @@ void mstro__addr_ib__ud__free_unpacked assert(message->base.descriptor == &mstro__addr_ib__ud__descriptor); protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); } +void mstro__addr_cxi__init + (Mstro__AddrCXI *message) +{ + static const Mstro__AddrCXI init_value = MSTRO__ADDR_CXI__INIT; + *message = init_value; +} +size_t mstro__addr_cxi__get_packed_size + (const Mstro__AddrCXI *message) +{ + assert(message->base.descriptor == &mstro__addr_cxi__descriptor); + return protobuf_c_message_get_packed_size ((const ProtobufCMessage*)(message)); +} +size_t mstro__addr_cxi__pack + (const Mstro__AddrCXI *message, + uint8_t *out) +{ + assert(message->base.descriptor == &mstro__addr_cxi__descriptor); + return protobuf_c_message_pack ((const ProtobufCMessage*)message, out); +} +size_t mstro__addr_cxi__pack_to_buffer + (const Mstro__AddrCXI *message, + ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &mstro__addr_cxi__descriptor); + return protobuf_c_message_pack_to_buffer ((const ProtobufCMessage*)message, buffer); +} +Mstro__AddrCXI * + mstro__addr_cxi__unpack + (ProtobufCAllocator *allocator, + size_t len, + const uint8_t *data) +{ + return (Mstro__AddrCXI *) + protobuf_c_message_unpack (&mstro__addr_cxi__descriptor, + allocator, len, data); +} +void mstro__addr_cxi__free_unpacked + (Mstro__AddrCXI *message, + ProtobufCAllocator *allocator) +{ + if(!message) + return; + assert(message->base.descriptor == &mstro__addr_cxi__descriptor); + protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); +} void mstro__ofi_addr__init (Mstro__OfiAddr *message) { @@ -547,6 +592,51 @@ void mstro__cred_drc__free_unpacked assert(message->base.descriptor == &mstro__cred_drc__descriptor); protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); } +void mstro__cred_cxi__init + (Mstro__CredCXI *message) +{ + static const Mstro__CredCXI init_value = MSTRO__CRED_CXI__INIT; + *message = init_value; +} +size_t mstro__cred_cxi__get_packed_size + (const Mstro__CredCXI *message) +{ + assert(message->base.descriptor == &mstro__cred_cxi__descriptor); + return protobuf_c_message_get_packed_size ((const ProtobufCMessage*)(message)); +} +size_t mstro__cred_cxi__pack + (const Mstro__CredCXI *message, + uint8_t *out) +{ + assert(message->base.descriptor == &mstro__cred_cxi__descriptor); + return protobuf_c_message_pack ((const ProtobufCMessage*)message, out); +} +size_t mstro__cred_cxi__pack_to_buffer + (const Mstro__CredCXI *message, + ProtobufCBuffer *buffer) +{ + assert(message->base.descriptor == &mstro__cred_cxi__descriptor); + return protobuf_c_message_pack_to_buffer ((const ProtobufCMessage*)message, buffer); +} +Mstro__CredCXI * + mstro__cred_cxi__unpack + (ProtobufCAllocator *allocator, + size_t len, + const uint8_t *data) +{ + return (Mstro__CredCXI *) + protobuf_c_message_unpack (&mstro__cred_cxi__descriptor, + allocator, len, data); +} +void mstro__cred_cxi__free_unpacked + (Mstro__CredCXI *message, + ProtobufCAllocator *allocator) +{ + if(!message) + return; + assert(message->base.descriptor == &mstro__cred_cxi__descriptor); + protobuf_c_message_free_unpacked ((ProtobufCMessage*)message, allocator); +} void mstro__ofi_credential__init (Mstro__OfiCredential *message) { @@ -1246,7 +1336,45 @@ const ProtobufCMessageDescriptor mstro__addr_ib__ud__descriptor = (ProtobufCMessageInit) mstro__addr_ib__ud__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mstro__ofi_addr__field_descriptors[14] = +static const ProtobufCFieldDescriptor mstro__addr_cxi__field_descriptors[1] = +{ + { + "raw", + 1, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_FIXED32, + 0, /* quantifier_offset */ + offsetof(Mstro__AddrCXI, raw), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned mstro__addr_cxi__field_indices_by_name[] = { + 0, /* field[0] = raw */ +}; +static const ProtobufCIntRange mstro__addr_cxi__number_ranges[1 + 1] = +{ + { 1, 0 }, + { 0, 1 } +}; +const ProtobufCMessageDescriptor mstro__addr_cxi__descriptor = +{ + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "mstro.AddrCXI", + "AddrCXI", + "Mstro__AddrCXI", + "mstro", + sizeof(Mstro__AddrCXI), + 1, + mstro__addr_cxi__field_descriptors, + mstro__addr_cxi__field_indices_by_name, + 1, mstro__addr_cxi__number_ranges, + (ProtobufCMessageInit) mstro__addr_cxi__init, + NULL,NULL,NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor mstro__ofi_addr__field_descriptors[15] = { { "unspec", @@ -1416,9 +1544,22 @@ static const ProtobufCFieldDescriptor mstro__ofi_addr__field_descriptors[14] = 0 | PROTOBUF_C_FIELD_FLAG_ONEOF, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, + { + "cxi", + 15, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_MESSAGE, + offsetof(Mstro__OfiAddr, val_case), + offsetof(Mstro__OfiAddr, cxi), + &mstro__addr_cxi__descriptor, + NULL, + 0 | PROTOBUF_C_FIELD_FLAG_ONEOF, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, }; static const unsigned mstro__ofi_addr__field_indices_by_name[] = { 7, /* field[7] = bgq */ + 14, /* field[14] = cxi */ 12, /* field[12] = efa */ 6, /* field[6] = gni */ 4, /* field[4] = ib */ @@ -1436,7 +1577,7 @@ static const unsigned mstro__ofi_addr__field_indices_by_name[] = { static const ProtobufCIntRange mstro__ofi_addr__number_ranges[1 + 1] = { { 1, 0 }, - { 0, 14 } + { 0, 15 } }; const ProtobufCMessageDescriptor mstro__ofi_addr__descriptor = { @@ -1446,7 +1587,7 @@ const ProtobufCMessageDescriptor mstro__ofi_addr__descriptor = "Mstro__OfiAddr", "mstro", sizeof(Mstro__OfiAddr), - 14, + 15, mstro__ofi_addr__field_descriptors, mstro__ofi_addr__field_indices_by_name, 1, mstro__ofi_addr__number_ranges, @@ -1593,7 +1734,58 @@ const ProtobufCMessageDescriptor mstro__cred_drc__descriptor = (ProtobufCMessageInit) mstro__cred_drc__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mstro__ofi_credential__field_descriptors[1] = +static const ProtobufCFieldDescriptor mstro__cred_cxi__field_descriptors[2] = +{ + { + "svc_id", + 1, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_FIXED32, + 0, /* quantifier_offset */ + offsetof(Mstro__CredCXI, svc_id), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, + { + "vni", + 2, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_FIXED32, + 0, /* quantifier_offset */ + offsetof(Mstro__CredCXI, vni), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, +}; +static const unsigned mstro__cred_cxi__field_indices_by_name[] = { + 0, /* field[0] = svc_id */ + 1, /* field[1] = vni */ +}; +static const ProtobufCIntRange mstro__cred_cxi__number_ranges[1 + 1] = +{ + { 1, 0 }, + { 0, 2 } +}; +const ProtobufCMessageDescriptor mstro__cred_cxi__descriptor = +{ + PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC, + "mstro.CredCXI", + "CredCXI", + "Mstro__CredCXI", + "mstro", + sizeof(Mstro__CredCXI), + 2, + mstro__cred_cxi__field_descriptors, + mstro__cred_cxi__field_indices_by_name, + 1, mstro__cred_cxi__number_ranges, + (ProtobufCMessageInit) mstro__cred_cxi__init, + NULL,NULL,NULL /* reserved[123] */ +}; +static const ProtobufCFieldDescriptor mstro__ofi_credential__field_descriptors[2] = { { "drc", @@ -1607,14 +1799,27 @@ static const ProtobufCFieldDescriptor mstro__ofi_credential__field_descriptors[1 0 | PROTOBUF_C_FIELD_FLAG_ONEOF, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, + { + "cxi", + 2, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_MESSAGE, + offsetof(Mstro__OfiCredential, val_case), + offsetof(Mstro__OfiCredential, cxi), + &mstro__cred_cxi__descriptor, + NULL, + 0 | PROTOBUF_C_FIELD_FLAG_ONEOF, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, }; static const unsigned mstro__ofi_credential__field_indices_by_name[] = { + 1, /* field[1] = cxi */ 0, /* field[0] = drc */ }; static const ProtobufCIntRange mstro__ofi_credential__number_ranges[1 + 1] = { { 1, 0 }, - { 0, 1 } + { 0, 2 } }; const ProtobufCMessageDescriptor mstro__ofi_credential__descriptor = { @@ -1624,7 +1829,7 @@ const ProtobufCMessageDescriptor mstro__ofi_credential__descriptor = "Mstro__OfiCredential", "mstro", sizeof(Mstro__OfiCredential), - 1, + 2, mstro__ofi_credential__field_descriptors, mstro__ofi_credential__field_indices_by_name, 1, mstro__ofi_credential__number_ranges, @@ -1733,7 +1938,7 @@ const ProtobufCMessageDescriptor mstro__app_info__descriptor = (ProtobufCMessageInit) mstro__app_info__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCEnumValue mstro__ofi_endpoint_kind__enum_values_by_number[22] = +static const ProtobufCEnumValue mstro__ofi_endpoint_kind__enum_values_by_number[23] = { { "UNSPEC", "MSTRO__OFI_ENDPOINT_KIND__UNSPEC", 0 }, { "RDMA_CM_IB_RC", "MSTRO__OFI_ENDPOINT_KIND__RDMA_CM_IB_RC", 1 }, @@ -1757,12 +1962,14 @@ static const ProtobufCEnumValue mstro__ofi_endpoint_kind__enum_values_by_number[ { "RDMA_CM_IB_XRC", "MSTRO__OFI_ENDPOINT_KIND__RDMA_CM_IB_XRC", 19 }, { "EFA", "MSTRO__OFI_ENDPOINT_KIND__EFA", 20 }, { "PSMX3", "MSTRO__OFI_ENDPOINT_KIND__PSMX3", 21 }, + { "CXI", "MSTRO__OFI_ENDPOINT_KIND__CXI", 22 }, }; static const ProtobufCIntRange mstro__ofi_endpoint_kind__value_ranges[] = { -{0, 0},{0, 22} +{0, 0},{0, 23} }; -static const ProtobufCEnumValueIndex mstro__ofi_endpoint_kind__enum_values_by_name[22] = +static const ProtobufCEnumValueIndex mstro__ofi_endpoint_kind__enum_values_by_name[23] = { + { "CXI", 22 }, { "EFA", 20 }, { "GNI", 10 }, { "IB_RDM", 9 }, @@ -1793,9 +2000,9 @@ const ProtobufCEnumDescriptor mstro__ofi_endpoint_kind__descriptor = "OfiEndpointKind", "Mstro__OfiEndpointKind", "mstro", - 22, + 23, mstro__ofi_endpoint_kind__enum_values_by_number, - 22, + 23, mstro__ofi_endpoint_kind__enum_values_by_name, 1, mstro__ofi_endpoint_kind__value_ranges, diff --git a/protocols/mstro_ep.pb-c.h b/protocols/mstro_ep.pb-c.h index 1bd476f2153c561111d829643afbb01f9d26f8cf..c121a5ca9a636045bacfe164e9e8c1e54e7467ca 100644 --- a/protocols/mstro_ep.pb-c.h +++ b/protocols/mstro_ep.pb-c.h @@ -23,10 +23,12 @@ typedef struct _Mstro__AddrGNI Mstro__AddrGNI; typedef struct _Mstro__AddrPSMX2 Mstro__AddrPSMX2; typedef struct _Mstro__AddrPSMX3 Mstro__AddrPSMX3; typedef struct _Mstro__AddrIBUD Mstro__AddrIBUD; +typedef struct _Mstro__AddrCXI Mstro__AddrCXI; typedef struct _Mstro__OfiAddr Mstro__OfiAddr; typedef struct _Mstro__Endpoint Mstro__Endpoint; typedef struct _Mstro__OfiMemoryRegion Mstro__OfiMemoryRegion; typedef struct _Mstro__CredDRC Mstro__CredDRC; +typedef struct _Mstro__CredCXI Mstro__CredCXI; typedef struct _Mstro__OfiCredential Mstro__OfiCredential; typedef struct _Mstro__EndpointList Mstro__EndpointList; typedef struct _Mstro__AppInfo Mstro__AppInfo; @@ -62,7 +64,8 @@ typedef enum _Mstro__OfiEndpointKind { MSTRO__OFI_ENDPOINT_KIND__RSTREAM = 18, MSTRO__OFI_ENDPOINT_KIND__RDMA_CM_IB_XRC = 19, MSTRO__OFI_ENDPOINT_KIND__EFA = 20, - MSTRO__OFI_ENDPOINT_KIND__PSMX3 = 21 + MSTRO__OFI_ENDPOINT_KIND__PSMX3 = 21, + MSTRO__OFI_ENDPOINT_KIND__CXI = 22 PROTOBUF_C__FORCE_ENUM_TO_BE_INT_SIZE(MSTRO__OFI_ENDPOINT_KIND) } Mstro__OfiEndpointKind; @@ -198,6 +201,29 @@ struct _Mstro__AddrIBUD , 0, 0, 0, 0 } +struct _Mstro__AddrCXI +{ + ProtobufCMessage base; + /* + * union { + *struct { + *uint32_t pid : C_DFA_PID_BITS_MAX; + *uint32_t nic : C_DFA_NIC_BITS; + *uint32_t valid : 1; + *}; + *uint32_t raw; + *} + */ + /* + * While VNI is part of the addressing it is handled through auth_key methods. See CredCXI below. + */ + uint32_t raw; +}; +#define MSTRO__ADDR_CXI__INIT \ + { PROTOBUF_C_MESSAGE_INIT (&mstro__addr_cxi__descriptor) \ + , 0 } + + typedef enum { MSTRO__OFI_ADDR__VAL__NOT_SET = 0, MSTRO__OFI_ADDR__VAL_UNSPEC = 1, @@ -213,7 +239,8 @@ typedef enum { MSTRO__OFI_ADDR__VAL_PSMX2 = 11, MSTRO__OFI_ADDR__VAL_IB_UD = 12, MSTRO__OFI_ADDR__VAL_EFA = 13, - MSTRO__OFI_ADDR__VAL_PSMX3 = 14 + MSTRO__OFI_ADDR__VAL_PSMX3 = 14, + MSTRO__OFI_ADDR__VAL_CXI = 15 PROTOBUF_C__FORCE_ENUM_TO_BE_INT_SIZE(MSTRO__OFI_ADDR__VAL) } Mstro__OfiAddr__ValCase; @@ -254,6 +281,7 @@ struct _Mstro__OfiAddr * 2 uint64 values */ Mstro__AddrPSMX3 *psmx3; + Mstro__AddrCXI *cxi; }; }; #define MSTRO__OFI_ADDR__INIT \ @@ -314,9 +342,24 @@ struct _Mstro__CredDRC , 0 } +struct _Mstro__CredCXI +{ + ProtobufCMessage base; + uint32_t svc_id; + /* + * actually only 16 bit + */ + uint32_t vni; +}; +#define MSTRO__CRED_CXI__INIT \ + { PROTOBUF_C_MESSAGE_INIT (&mstro__cred_cxi__descriptor) \ + , 0, 0 } + + typedef enum { MSTRO__OFI_CREDENTIAL__VAL__NOT_SET = 0, - MSTRO__OFI_CREDENTIAL__VAL_DRC = 1 + MSTRO__OFI_CREDENTIAL__VAL_DRC = 1, + MSTRO__OFI_CREDENTIAL__VAL_CXI = 2 PROTOBUF_C__FORCE_ENUM_TO_BE_INT_SIZE(MSTRO__OFI_CREDENTIAL__VAL) } Mstro__OfiCredential__ValCase; @@ -329,6 +372,10 @@ struct _Mstro__OfiCredential * Cray DRC credential */ Mstro__CredDRC *drc; + /* + * HPE Slingshot Cassini + */ + Mstro__CredCXI *cxi; }; }; #define MSTRO__OFI_CREDENTIAL__INIT \ @@ -528,6 +575,25 @@ Mstro__AddrIBUD * void mstro__addr_ib__ud__free_unpacked (Mstro__AddrIBUD *message, ProtobufCAllocator *allocator); +/* Mstro__AddrCXI methods */ +void mstro__addr_cxi__init + (Mstro__AddrCXI *message); +size_t mstro__addr_cxi__get_packed_size + (const Mstro__AddrCXI *message); +size_t mstro__addr_cxi__pack + (const Mstro__AddrCXI *message, + uint8_t *out); +size_t mstro__addr_cxi__pack_to_buffer + (const Mstro__AddrCXI *message, + ProtobufCBuffer *buffer); +Mstro__AddrCXI * + mstro__addr_cxi__unpack + (ProtobufCAllocator *allocator, + size_t len, + const uint8_t *data); +void mstro__addr_cxi__free_unpacked + (Mstro__AddrCXI *message, + ProtobufCAllocator *allocator); /* Mstro__OfiAddr methods */ void mstro__ofi_addr__init (Mstro__OfiAddr *message); @@ -604,6 +670,25 @@ Mstro__CredDRC * void mstro__cred_drc__free_unpacked (Mstro__CredDRC *message, ProtobufCAllocator *allocator); +/* Mstro__CredCXI methods */ +void mstro__cred_cxi__init + (Mstro__CredCXI *message); +size_t mstro__cred_cxi__get_packed_size + (const Mstro__CredCXI *message); +size_t mstro__cred_cxi__pack + (const Mstro__CredCXI *message, + uint8_t *out); +size_t mstro__cred_cxi__pack_to_buffer + (const Mstro__CredCXI *message, + ProtobufCBuffer *buffer); +Mstro__CredCXI * + mstro__cred_cxi__unpack + (ProtobufCAllocator *allocator, + size_t len, + const uint8_t *data); +void mstro__cred_cxi__free_unpacked + (Mstro__CredCXI *message, + ProtobufCAllocator *allocator); /* Mstro__OfiCredential methods */ void mstro__ofi_credential__init (Mstro__OfiCredential *message); @@ -687,6 +772,9 @@ typedef void (*Mstro__AddrPSMX3_Closure) typedef void (*Mstro__AddrIBUD_Closure) (const Mstro__AddrIBUD *message, void *closure_data); +typedef void (*Mstro__AddrCXI_Closure) + (const Mstro__AddrCXI *message, + void *closure_data); typedef void (*Mstro__OfiAddr_Closure) (const Mstro__OfiAddr *message, void *closure_data); @@ -699,6 +787,9 @@ typedef void (*Mstro__OfiMemoryRegion_Closure) typedef void (*Mstro__CredDRC_Closure) (const Mstro__CredDRC *message, void *closure_data); +typedef void (*Mstro__CredCXI_Closure) + (const Mstro__CredCXI *message, + void *closure_data); typedef void (*Mstro__OfiCredential_Closure) (const Mstro__OfiCredential *message, void *closure_data); @@ -723,10 +814,12 @@ extern const ProtobufCMessageDescriptor mstro__addr_gni__descriptor; extern const ProtobufCMessageDescriptor mstro__addr_psmx2__descriptor; extern const ProtobufCMessageDescriptor mstro__addr_psmx3__descriptor; extern const ProtobufCMessageDescriptor mstro__addr_ib__ud__descriptor; +extern const ProtobufCMessageDescriptor mstro__addr_cxi__descriptor; extern const ProtobufCMessageDescriptor mstro__ofi_addr__descriptor; extern const ProtobufCMessageDescriptor mstro__endpoint__descriptor; extern const ProtobufCMessageDescriptor mstro__ofi_memory_region__descriptor; extern const ProtobufCMessageDescriptor mstro__cred_drc__descriptor; +extern const ProtobufCMessageDescriptor mstro__cred_cxi__descriptor; extern const ProtobufCMessageDescriptor mstro__ofi_credential__descriptor; extern const ProtobufCMessageDescriptor mstro__endpoint_list__descriptor; extern const ProtobufCMessageDescriptor mstro__app_info__descriptor; diff --git a/protocols/mstro_ep.proto b/protocols/mstro_ep.proto index b6b5623381a9950509034809c51a6857821dea65..c8558c2f1e038d8849c3d264aef21676452a3706 100644 --- a/protocols/mstro_ep.proto +++ b/protocols/mstro_ep.proto @@ -74,6 +74,19 @@ message AddrIB_UD { fixed64 a3 = 4; }; +message AddrCXI { + /* union { + struct { + uint32_t pid : C_DFA_PID_BITS_MAX; + uint32_t nic : C_DFA_NIC_BITS; + uint32_t valid : 1; + }; + uint32_t raw; + } */ + fixed32 raw = 1; + /* While VNI is part of the addressing it is handled through auth_key methods. See CredCXI below. */ +}; + /** Open Fabric endpoint address */ message OfiAddr { oneof val { @@ -92,6 +105,7 @@ message OfiAddr { AddrIB_UD ib_ud = 12; /* 4 uint64 values */ fixed64 efa = 13; AddrPSMX3 psmx3 = 14; /* 2 uint64 values */ + AddrCXI cxi = 15; } }; @@ -119,7 +133,8 @@ enum OfiEndpointKind { RSTREAM = 18; RDMA_CM_IB_XRC = 19; EFA = 20; - PSMX3 = 21; + PSMX3 = 21; + CXI = 22; } message Endpoint { @@ -141,9 +156,15 @@ message CredDRC { fixed32 credential = 1; } +message CredCXI { + fixed32 svc_id = 1; + fixed32 vni = 2; /* actually only 16 bit */ +} + message OfiCredential { oneof val { CredDRC drc = 1; /* Cray DRC credential */ + CredCXI cxi = 2; /* HPE Slingshot Cassini */ } }