CUDA: enable cuda support v1 - EAGER with GDR COPY

src/uct/api/uct.h

@@ -377,11 +377,24 @@ enum {
     UCT_MD_FLAG_RKEY_PTR   = UCS_BIT(6),  /**< MD supports direct access to
                                                remote memory via a pointer that
                                                is returned by @ref uct_rkey_ptr */
-    UCT_MD_FLAG_SOCKADDR   = UCS_BIT(7)   /**< MD support for client-server
+    UCT_MD_FLAG_SOCKADDR   = UCS_BIT(7),  /**< MD support for client-server
                                                connection establishment via
                                                sockaddr */
+    UCT_MD_FLAG_ADDR_DN    = UCS_BIT(8)   /**< MD supports memory addr domain
+                                               detection */
 };
 
+/*
+ * @ingroup UCT_MD
+ * @brief  Memory addr domains
+ */
+typedef enum {
+    UCT_MD_ADDR_DOMAIN_CUDA = 0,  /**< NVIDIA CUDA domain */
+    UCT_MD_ADDR_DOMAIN_DEFAULT,   /**< Default system domain */
+    UCT_MD_ADDR_DOMAIN_LAST = UCT_MD_ADDR_DOMAIN_DEFAULT
+
+} uct_addr_domain_t;
+
 
 /**
  * @ingroup UCT_MD
@@ -630,6 +643,7 @@ struct uct_md_attr {
         size_t               max_alloc; /**< Maximal allocation size */
         size_t               max_reg;   /**< Maximal registration size */
         uint64_t             flags;     /**< UCT_MD_FLAG_xx */
+        uct_addr_domain_t    addr_dn;   /**< Supported addr domain */
     } cap;
 
     uct_linear_growth_t      reg_cost;  /**< Memory registration cost estimation
@@ -1413,6 +1427,18 @@ ucs_status_t uct_md_mem_reg(uct_md_h md, void *address, size_t length,
 ucs_status_t uct_md_mem_dereg(uct_md_h md, uct_mem_h memh);
 
 
+/**
+ * @ingroup UCT_MD
+ * @brief Detect memory on the memory domain.
+ *
+ *  Detect memory on the memory domain.
+ *  Return UCS_OK if address belongs to MDs address domain
+ *
+ * @param [in]     md        Memory domain to register memory on.
+ * @param [in]     address   Memory address to detect.
+ */
+ucs_status_t uct_md_mem_detect(uct_md_h md, void *addr);
+
 /**
  * @ingroup UCT_MD
  * @brief Allocate memory for zero-copy communications and remote access.

src/uct/base/uct_md.c

@@ -543,3 +543,8 @@ int uct_md_is_sockaddr_accessible(uct_md_h md, const ucs_sock_addr_t *sockaddr,
 {
     return md->ops->is_sockaddr_accessible(md, sockaddr, mode);
 }
+
+ucs_status_t uct_md_mem_detect(uct_md_h md, void *addr)
+{
+    return md->ops->mem_detect(md, addr);
+}

src/uct/base/uct_md.h

@@ -136,6 +136,7 @@ struct uct_md_ops {
 
     int          (*is_sockaddr_accessible)(uct_md_h md, const ucs_sock_addr_t *sockaddr,
                                            uct_sockaddr_accessibility_t mode);
+    ucs_status_t (*mem_detect)(uct_md_h md, void *addr);
 };
 
 

src/uct/ib/base/ib_md.c

@@ -155,9 +155,11 @@ static ucs_status_t uct_ib_md_query(uct_md_h uct_md, uct_md_attr_t *md_attr)
     md_attr->cap.max_alloc = ULONG_MAX; /* TODO query device */
     md_attr->cap.max_reg   = ULONG_MAX; /* TODO query device */
     md_attr->cap.flags     = UCT_MD_FLAG_REG       |
+                             UCT_MD_FLAG_RNDV_REG  |
                              UCT_MD_FLAG_NEED_MEMH |
                              UCT_MD_FLAG_NEED_RKEY |
                              UCT_MD_FLAG_ADVISE;
+    md_attr->cap.addr_dn   = UCT_MD_ADDR_DOMAIN_DEFAULT;
     md_attr->rkey_packed_size = sizeof(uint64_t);
 
     if (md->config.enable_contig_pages &&
@@ -866,6 +868,7 @@ static uct_md_ops_t uct_ib_md_ops = {
     .mem_alloc    = uct_ib_mem_alloc,
     .mem_free     = uct_ib_mem_free,
     .mem_reg      = uct_ib_mem_reg,
+    .mem_detect   = ucs_empty_function_return_success,
     .mem_dereg    = uct_ib_mem_dereg,
     .mem_advise   = uct_ib_mem_advise,
     .mkey_pack    = uct_ib_mkey_pack,

src/uct/rocm/rocm_cma_md.c

@@ -30,6 +30,7 @@ static ucs_status_t uct_rocm_cma_md_query(uct_md_h md, uct_md_attr_t *md_attr)
     md_attr->rkey_packed_size  = sizeof(uct_rocm_cma_key_t);
     md_attr->cap.flags         = UCT_MD_FLAG_REG |
                                  UCT_MD_FLAG_NEED_RKEY;
+    md_attr->cap.addr_dn       = UCT_MD_ADDR_DOMAIN_DEFAULT;
     md_attr->cap.max_alloc     = 0;
     md_attr->cap.max_reg       = ULONG_MAX;
 
@@ -211,6 +212,7 @@ static ucs_status_t uct_rocm_cma_md_open(const char *md_name,
         .query        = uct_rocm_cma_md_query,
         .mkey_pack    = uct_rocm_cma_rkey_pack,
         .mem_reg      = uct_rocm_cma_mem_reg,
+        .mem_detect   = ucs_empty_function_return_success,
         .mem_dereg    = uct_rocm_cma_mem_dereg
     };
 

src/uct/sm/cma/cma_md.c

@@ -59,7 +59,8 @@ static ucs_status_t uct_cma_md_open(const char *md_name, const uct_md_config_t *
         .mem_free     = (void*)ucs_empty_function_return_success,
         .mkey_pack    = (void*)ucs_empty_function_return_success,
         .mem_reg      = uct_cma_mem_reg,
-        .mem_dereg    = (void*)ucs_empty_function_return_success
+        .mem_dereg    = (void*)ucs_empty_function_return_success,
+        .mem_detect   = ucs_empty_function_return_success,
     };
     static uct_md_t md = {
         .ops          = &md_ops,
@@ -80,6 +81,7 @@ ucs_status_t uct_cma_md_query(uct_md_h md, uct_md_attr_t *md_attr)
 {
     md_attr->rkey_packed_size  = 0;
     md_attr->cap.flags         = UCT_MD_FLAG_REG;
+    md_attr->cap.addr_dn       = UCT_MD_ADDR_DOMAIN_DEFAULT;
     md_attr->cap.max_alloc     = 0;
     md_attr->cap.max_reg       = ULONG_MAX;
     md_attr->reg_cost.overhead = 9e-9;

src/uct/sm/knem/knem_md.c

@@ -13,6 +13,7 @@ ucs_status_t uct_knem_md_query(uct_md_h md, uct_md_attr_t *md_attr)
     md_attr->rkey_packed_size  = sizeof(uct_knem_key_t);
     md_attr->cap.flags         = UCT_MD_FLAG_REG |
                                  UCT_MD_FLAG_NEED_RKEY;
+    md_attr->cap.addr_dn       = UCT_MD_ADDR_DOMAIN_DEFAULT;
     md_attr->cap.max_alloc     = 0;
     md_attr->cap.max_reg       = ULONG_MAX;
     md_attr->reg_cost.overhead = 1200.0e-9;
@@ -186,7 +187,8 @@ static ucs_status_t uct_knem_md_open(const char *md_name, const uct_md_config_t
         .mem_free     = (void*)ucs_empty_function_return_success,
         .mkey_pack    = uct_knem_rkey_pack,
         .mem_reg      = uct_knem_mem_reg,
-        .mem_dereg    = uct_knem_mem_dereg
+        .mem_dereg    = uct_knem_mem_dereg,
+        .mem_detect   = ucs_empty_function_return_success
     };
 
     knem_md = ucs_malloc(sizeof(uct_knem_md_t), "uct_knem_md_t");

src/uct/sm/mm/mm_md.c

@@ -124,6 +124,7 @@ ucs_status_t uct_mm_md_query(uct_md_h md, uct_md_attr_t *md_attr)
         md_attr->reg_cost.growth   = 0.007e-9;
     }
     md_attr->cap.flags        |= UCT_MD_FLAG_NEED_RKEY;
+    md_attr->cap.addr_dn      = UCT_MD_ADDR_DOMAIN_DEFAULT;
     /* all mm md(s) support fixed memory alloc */
     md_attr->cap.flags        |= UCT_MD_FLAG_FIXED;
     md_attr->cap.max_alloc    = ULONG_MAX;
@@ -227,6 +228,7 @@ uct_md_ops_t uct_mm_md_ops = {
     .mem_free     = uct_mm_mem_free,
     .mem_reg      = uct_mm_mem_reg,
     .mem_dereg    = uct_mm_mem_dereg,
+    .mem_detect   = ucs_empty_function_return_success,
     .mkey_pack    = uct_mm_mkey_pack,
 };
 

src/uct/sm/self/self_md.c

@@ -10,6 +10,7 @@ static ucs_status_t uct_self_md_query(uct_md_h md, uct_md_attr_t *attr)
 {
     /* Dummy memory registration provided. No real memory handling exists */
     attr->cap.flags         = UCT_MD_FLAG_REG;
+    attr->cap.addr_dn       = UCT_MD_ADDR_DOMAIN_DEFAULT;
     attr->cap.max_alloc     = 0;
     attr->cap.max_reg       = ULONG_MAX;
     attr->rkey_packed_size  = 0; /* uct_md_query adds UCT_MD_COMPONENT_NAME_MAX to this */
@@ -41,7 +42,8 @@ static ucs_status_t uct_self_md_open(const char *md_name, const uct_md_config_t
         .query        = uct_self_md_query,
         .mkey_pack    = ucs_empty_function_return_success,
         .mem_reg      = uct_self_mem_reg,
-        .mem_dereg    = ucs_empty_function_return_success
+        .mem_dereg    = ucs_empty_function_return_success,
+        .mem_detect   = ucs_empty_function_return_success
     };
     static uct_md_t md = {
         .ops          = &md_ops,

src/uct/tcp/tcp_md.c

@@ -32,7 +32,8 @@ static ucs_status_t uct_tcp_md_open(const char *md_name, const uct_md_config_t *
         .query        = uct_tcp_md_query,
         .mkey_pack    = ucs_empty_function_return_unsupported,
         .mem_reg      = ucs_empty_function_return_unsupported,
-        .mem_dereg    = ucs_empty_function_return_unsupported
+        .mem_dereg    = ucs_empty_function_return_unsupported,
+        .mem_detect   = ucs_empty_function_return_success
     };
     static uct_md_t md = {
         .ops          = &md_ops,

src/uct/ugni/base/ugni_md.c

@@ -34,6 +34,7 @@ static ucs_status_t uct_ugni_md_query(uct_md_h md, uct_md_attr_t *md_attr)
     md_attr->cap.flags         = UCT_MD_FLAG_REG       |
                                  UCT_MD_FLAG_NEED_MEMH |
                                  UCT_MD_FLAG_NEED_RKEY;
+    md_attr->cap.addr_dn       = UCT_MD_ADDR_DOMAIN_DEFAULT;
     md_attr->cap.max_alloc     = 0;
     md_attr->cap.max_reg       = ULONG_MAX;
     md_attr->reg_cost.overhead = 1000.0e-9;
@@ -180,6 +181,7 @@ static ucs_status_t uct_ugni_md_open(const char *md_name, const uct_md_config_t
         .mem_free     = (void*)ucs_empty_function,
         .mem_reg      = uct_ugni_mem_reg,
         .mem_dereg    = uct_ugni_mem_dereg,
+        .mem_detect   = ucs_empty_function_return_success,
         .mkey_pack     = uct_ugni_rkey_pack
     };