Feat xdp map pinning

ansible/cndp-config/templates/fwd.jsonc.j2

@@ -86,10 +86,10 @@
     //    busy_polling  -     Same as above
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milliseconds
     //    busy_budget   - (O) 0xFFFF disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well
+    //    xsk_pin_path  - (O) Path to pinned bpf map
     "lports": {
     {% for i in range(num_lports) %}
         "{{iface}}:{{i}}": {

containerization/docker/ubuntu/fwd.jsonc

@@ -79,10 +79,10 @@
     //    busy_polling  -     Same as above
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milliseconds
     //    busy_budget   - (O) 0xFFFF disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well
+	//    xsk_pin_path  - (O) Path to pinned xsk map for this port
     "lports": {
         "net1:0": {
             "pmd": "net_af_xdp",

containerization/k8s/README.md

@@ -1,4 +1,4 @@
-### CNDP Pod
+# CNDP Pod
 
 The provided configuration creates a CNDP pod with two containers. One that runs
 the CNDP cndpfwd example and another which runs the prometheus go agent to collect
@@ -7,7 +7,7 @@ a unix domain socket.
 
 This guide will walk you through the setup of the CNDP pods.
 
-### Setup K8s Env
+## Setup K8s Env
 
 This guide will walk you through how to setup a single node cluster where you
 can launch a CNDP container. The example uses kubeadm to bootstrap the cluster.
@@ -218,7 +218,7 @@ container image used by the CNDP pod.
 
 ### Build and deploy AF_XDP plugins for K8s
 
-The source code is available at: https://github.com/intel/afxdp-plugins-for-kubernetes
+The source code is available [here](https://github.com/intel/afxdp-plugins-for-kubernetes).
 
 For detailed install instructions please refer to README.md in the device plugin repo.
 This section will provide a quick start for deploying the device plugin and CNI.
@@ -254,16 +254,17 @@ index 8465a21..774c6ca 100644
  data:
    config.json: |
      {
-         "mode": "cndp",
-         "logLevel": "debug",
-         "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
-         "pools" : [
-             {
--                "name" : "e2e",
-+                "name" : "pool1",
-                 "drivers" : ["i40e"]
-             }
-         ]
+        "clusterType": "physical",
+        "mode": "primary",
+        "logLevel": "debug",
+        "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
+        "pools" : [
+            {
+-              "name" : "e2e",
++              "name" : "pool1",
+               "drivers" : ["i40e"]
+            }
+        ]
 
 ```
 
@@ -302,7 +303,7 @@ ctr -n=k8s.io images import afxdp-device-plugin.tar
 
 ### Verify that the image is now available to the container run-time
 
-For information on how to install crictl : https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/
+More information on how to install crictl can be found [here](https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/).
 
 ```bash
 sudo crictl images
@@ -341,6 +342,10 @@ file to define LIST_OF_QIDS or CNDP_COPY_MODE environment variables. These
 variables are read by `tools/jsonc_gen.sh` when the container starts to
 generate the configuration for the CNDP application.
 
+> **_NOTE:_** the `tools/jsonc_gen.sh` supports BPF map pinning configuration by
+running it with the `-p` flag. If you wish to use it in a kind cluster then run
+the script with a `-k` flag in the pod YAML.
+
 An example to force copy-mode for all AF_XDP sockets:
 
 ```yaml

containerization/k8s/cndp-pods/cndp-0-0.yaml

@@ -16,12 +16,15 @@ spec:
     - name: cndp-0
       command: ["/bin/bash"]
       args: ["-c", "./jsonc_gen.sh; cndpfwd -c config.jsonc lb;"]
+      # args: ["-c", "./jsonc_gen.sh -k -p; cndpfwd -c config.jsonc lb;"] # to run in a kind cluster using bpf map pinning
       image: cndp
       imagePullPolicy: Never
       securityContext:
         capabilities:
           add:
             - NET_RAW
+            - IPC_LOCK
+            # - BPF # Enable if running in a kind cluster as the default kernel used is 5.15, BPF map support without privilege is only supported from 5.19.
       ports:
       - containerPort: 8094
         hostPort: 8094

doc/guides/sample_app_ug/cndpfwd.rst

@@ -31,10 +31,11 @@ The creation of an AF_XDP socket involves loading of a BPF program which is a pr
 In order to run the CNDP application in an unprivileged container, the privileged operations are
 done by a Kubernetes device plugin. The CNDP application talks to the Kubernetes device plugin over
 a unix domain socket. The path to the unix domain socket created by the device plugin is the value
-of the "uds_path" attribute. The "unprivileged" flag should be true if running the CNDP app in an
-unprivileged container. The sysctl param ``kernel.unprivileged_bpf_disabled`` should be 0 to perform
-unprivileged BPF operations. For more details about the device plugin, please refer to
-:ref:`Integration of the K8s device plugin with CNDP <integration-k8s-dp>`.
+of the "uds_path" attribute. The "unprivileged" flag is automatically configured if the "uds_path"
+parameter is used. The sysctl param ``kernel.unprivileged_bpf_disabled`` should be 0 to perform
+unprivileged BPF operations. Alternatively the "xsk_pin_path" can be used in combination with the
+AF_XDP Device Plugin to support unprivileged Pods. For more details about the device plugin, please refer to
+:ref:`Integration of the K8s device plugin with CNDP <integration-k8s-dp>`. Alternatively
 
 Running the Application
 -----------------------
@@ -156,10 +157,10 @@ The configuration json file is located in the ``cndpfwd`` example sub-directory
         //    busy_poll     - (O) Enable busy polling support, true or false, default false
         //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milliseconds
         //    busy_budget   - (O) 0xFFFF disabled, 0 use default, >0 budget value
-        //    unprivileged  - (O) inhibit loading the BPF program if true, default false
         //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
         //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
         //    description   - (O) the description, 'desc' can be used as well
+		//    xsk_pin_path  - (O) Path to pinned xsk map for this port
         "lports": {
             "eth0:0": {
                 "pmd": "net_af_xdp",

examples/cndpfwd/fwd.jsonc

@@ -79,9 +79,9 @@
     //    busy_polling  -     Same as above
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milliseconds
     //    busy_budget   - (O) 0xFFFF disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
+	//    xsk_pin_path  - (O) Path to pinned xsk map for this port
     //    description   - (O) the description, 'desc' can be used as well
     "lports": {
         "eth0:0": {

examples/cndpfwd/parse-args.c

@@ -33,6 +33,7 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
     uint32_t total_region_cnt;
     char *umem_addr;
     size_t nlen;
+    jcfg_lport_t *lport;
 
     if (!_obj)
         return -1;
@@ -157,7 +158,7 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
 
     case JCFG_LPORT_TYPE:
         do {
-            jcfg_lport_t *lport = obj.lport;
+            lport = obj.lport;
             struct fwd_port *pd;
             mmap_t *mm;
             jcfg_umem_t *umem;
@@ -188,6 +189,11 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
             pcfg.flags        = lport->flags;
             pcfg.flags |= (umem->shared_umem == 1) ? LPORT_SHARED_UMEM : 0;
 
+            if (lport->xsk_map_path) {
+                cne_printf("[yellow]**** [green]PINNED_BPF_MAP is [red]enabled[]\n");
+                pcfg.xsk_map_path = lport->xsk_map_path;
+            }
+
             pcfg.addr = jcfg_lport_region(lport, &pcfg.bufcnt);
             if (!pcfg.addr) {
                 free(pd);
@@ -196,12 +202,11 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
             }
             pcfg.pi = umem->rinfo[lport->region_idx].pool;
 
-            if (lport->flags & LPORT_UNPRIVILEGED) {
-                if (f->xdp_uds)
-                    pcfg.xsk_uds = f->xdp_uds;
-                else
-                    CNE_ERR_RET("UDS info struct is null\n");
+            if (f->xdp_uds) {
+                pcfg.xsk_uds = f->xdp_uds;
+                lport->flags |= LPORT_UNPRIVILEGED;
             }
+
             /* Setup the mempool configuration */
             strlcpy(pcfg.pmd_name, lport->pmd_name, sizeof(pcfg.pmd_name));
             strlcpy(pcfg.ifname, lport->netdev, sizeof(pcfg.ifname));

lang/rs/apis/cne/fwd.jsonc

@@ -75,7 +75,6 @@
     //    busy_poll     - (O) Enable busy polling support, true or false, default false
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milli-seconds
     //    busy_budget   - (O) -1 disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well

lang/rs/examples/echo_server/fwd.jsonc

@@ -75,7 +75,6 @@
     //    busy_poll     - (O) Enable busy polling support, true or false, default false
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milli-seconds
     //    busy_budget   - (O) -1 disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well

lang/rs/examples/fwd/fwd.jsonc

@@ -75,7 +75,6 @@
     //    busy_poll     - (O) Enable busy polling support, true or false, default false
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milli-seconds
     //    busy_budget   - (O) -1 disabled, 0 use default, >0 budget value
-    //    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well

lib/core/xskdev/xskdev.c

@@ -26,6 +26,8 @@
 #include <cne_mutex_helper.h>
 #include <dirent.h>
 #include <limits.h>        // for PATH_MAX
+#include <bpf/bpf.h>
+#include <error.h>
 
 #include "xskdev.h"
 #include "cne_lport.h"        // for lport_stats_t, lport_cfg, lport_cfg_t
@@ -738,16 +740,26 @@ xskdev_socket_create(struct lport_cfg *c)
     xi->xsk_map_fd = -1;
 
     if (c->flags & LPORT_UNPRIVILEGED) {
-        /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client
-         * and receive the XSK_MAP_FD.
-         * We will need (to wait) a flag or re checking of the value of the FD (latter preferred) to
-         * indicate we have completed reception before moving onto the socket create stage
-         */
-        xi->uds_info = (uds_info_t *)c->xsk_uds;
+        if (c->xsk_uds) {
+            /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client
+             * and receive the XSK_MAP_FD.
+             * We will need (to wait) a flag or re checking of the value of the FD (latter
+             * preferred) to indicate we have completed reception before moving onto the socket
+             * create stage
+             */
+            xi->uds_info = (uds_info_t *)c->xsk_uds;
+
+            ret = xskdev_recv_xsk_fd(xi);
+            if (ret < 0)
+                CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n");
+        } else if (c->xsk_map_path) {
+            /* Try to retrieve the xsk_map_fd from a pinned bpf map*/
+            xi->xsk_map_fd = bpf_obj_get(c->xsk_map_path);
+            if (xi->xsk_map_fd < 0)
+                CNE_ERR_GOTO(err, "Failed to get xsk map fd from bpf object\n");
+        }
 
-        ret = xskdev_recv_xsk_fd(xi);
-        if (ret < 0)
-            CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n");
+        CNE_DEBUG("xi->xsk_map_fd = %d\n", xi->xsk_map_fd);
     }
 
     if (xskdev_use_tx_lock) {

lib/include/cne_lport.h

@@ -85,6 +85,7 @@ typedef struct lport_cfg {
     size_t umem_size;              /**< Size of the umem region */
     pktmbuf_info_t *pi;            /**< pktmbuf_info_t structure pointer */
     void *xsk_uds;                 /**< The UDS to connect to get xsk FDs */
+    char *xsk_map_path;            /**< The pinned map to get xsk FD */
     lport_buf_mgmt_t buf_mgmt;     /**< Buffer management functions */
 } lport_cfg_t;
 

lib/usr/app/jcfg/jcfg.h

@@ -171,7 +171,8 @@ typedef struct jcfg_lport {
     uint16_t qid;                /**< The queue ID number */
     uint16_t busy_timeout;       /**< busy timeout value in milliseconds */
     uint16_t busy_budget;        /**< busy budget 0xFFFF disabled, 0 use default, >0 budget */
-    uint16_t flags; /**< Flags to configure lport in lport_cfg_t.flags in cne_lport.h */
+    uint16_t flags;     /**< Flags to configure lport in lport_cfg_t.flags in cne_lport.h */
+    char *xsk_map_path; /**< The path to the pinned xsk_map for this port */
 } jcfg_lport_t;
 
 /** JCFG lport configuration names */
@@ -185,7 +186,7 @@ typedef struct jcfg_lport {
 #define JCFG_LPORT_BUSY_POLLING_NAME "busy_polling"
 #define JCFG_LPORT_BUSY_TIMEOUT_NAME "busy_timeout"
 #define JCFG_LPORT_BUSY_BUDGET_NAME  "busy_budget"
-#define JCFG_LPORT_UNPRIVILEGED_NAME "unprivileged"
+#define JCFG_PINNED_XSK_MAP_NAME     "xsk_pin_path"
 #define JCFG_LPORT_FORCE_WAKEUP_NAME "force_wakeup"
 #define JCFG_LPORT_SKB_MODE_NAME     "skb_mode"
 
@@ -237,6 +238,7 @@ typedef struct jcfg_lport_group {
     uint16_t busy_timeout;             /**< busy timeout value in milliseconds */
     uint16_t busy_budget;              /**< busy budget 0xFFFF disabled, 0 use default, >0 budget */
     uint16_t flags;                    /**< Flags to configure lport in lport_cfg_t.flags */
+
 } jcfg_lport_group_t;
 
 /** JCFG lport group configuration names */

lib/usr/app/jcfg/jcfg_lport.c

@@ -108,9 +108,10 @@ _lport(struct json_object *obj, int flags, struct json_object *parent __cne_unus
                 CNE_ERR_RET_VAL(JSON_C_VISIT_RETURN_ERROR, "%s: Invalid Range\n",
                                 JCFG_LPORT_BUSY_BUDGET_NAME);
             lport->busy_budget = (uint16_t)val;
-        } else if (!strncmp(key, JCFG_LPORT_UNPRIVILEGED_NAME, keylen))
-            lport->flags |= json_object_get_boolean(obj) ? LPORT_UNPRIVILEGED : 0;
-        else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen))
+        } else if (!strncmp(key, JCFG_PINNED_XSK_MAP_NAME, keylen)) {
+            lport->xsk_map_path = strndup(json_object_get_string(obj), JCFG_MAX_STRING_SIZE);
+            lport->flags |= LPORT_UNPRIVILEGED;
+        } else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen))
             lport->flags |= json_object_get_boolean(obj) ? LPORT_FORCE_WAKEUP : 0;
         else if (!strncmp(key, JCFG_LPORT_SKB_MODE_NAME, keylen))
             lport->flags |= json_object_get_boolean(obj) ? LPORT_SKB_MODE : 0;

lib/usr/app/jcfg/jcfg_lport_group.c

@@ -748,9 +748,7 @@ _lport_group(struct json_object *obj, int flags, struct json_object *parent __cn
             CNE_ERR_RET_VAL(JSON_C_VISIT_RETURN_ERROR, "%s: Invalid Range\n",
                             JCFG_LPORT_BUSY_BUDGET_NAME);
         lpg->busy_budget = (uint16_t)val;
-    } else if (!strncmp(key, JCFG_LPORT_UNPRIVILEGED_NAME, keylen))
-        lpg->flags |= json_object_get_boolean(obj) ? LPORT_UNPRIVILEGED : 0;
-    else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen))
+    } else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen))
         lpg->flags |= json_object_get_boolean(obj) ? LPORT_FORCE_WAKEUP : 0;
     else if (!strncmp(key, JCFG_LPORT_SKB_MODE_NAME, keylen))
         lpg->flags |= json_object_get_boolean(obj) ? LPORT_SKB_MODE : 0;

tools/cndp.schema

@@ -152,11 +152,6 @@
                             "minimum": 0,
                             "maximum": 65535
                         },
-                        "unprivileged": {
-                            "description": "Enable unprivileged operations",
-                            "type": "boolean",
-                            "default": false
-                        },
                         "force_wakeup": {
                             "description": "Force Tx wakeup calls for CVL NIC",
                             "type": "boolean",
@@ -330,11 +325,6 @@
                         "minimum": 0,
                         "maximum": 65535
                     },
-                    "unprivileged": {
-                        "description": "Enable unprivileged operations",
-                        "type": "boolean",
-                        "default": false
-                    },
                     "force_wakeup": {
                         "description": "Force Tx wakeup calls for CVL NIC",
                         "type": "boolean",