Feat xdp map pinning

ansible/cndp-config/templates/fwd.jsonc.j2

@@ -130,6 +130,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

containerization/docker/ubuntu/fwd.jsonc

@@ -115,6 +115,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

containerization/k8s/README.md

@@ -1,4 +1,4 @@
-### CNDP Pod
+# CNDP Pod
 
 The provided configuration creates a CNDP pod with two containers. One that runs
 the CNDP cndpfwd example and another which runs the prometheus go agent to collect
@@ -7,7 +7,7 @@ a unix domain socket.
 
 This guide will walk you through the setup of the CNDP pods.
 
-### Setup K8s Env
+## Setup K8s Env
 
 This guide will walk you through how to setup a single node cluster where you
 can launch a CNDP container. The example uses kubeadm to bootstrap the cluster.
@@ -218,7 +218,7 @@ container image used by the CNDP pod.
 
 ### Build and deploy AF_XDP plugins for K8s
 
-The source code is available at: https://github.com/intel/afxdp-plugins-for-kubernetes
+The source code is available [here](https://github.com/intel/afxdp-plugins-for-kubernetes).
 
 For detailed install instructions please refer to README.md in the device plugin repo.
 This section will provide a quick start for deploying the device plugin and CNI.
@@ -254,16 +254,17 @@ index 8465a21..774c6ca 100644
  data:
    config.json: |
      {
-         "mode": "cndp",
-         "logLevel": "debug",
-         "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
-         "pools" : [
-             {
--                "name" : "e2e",
-+                "name" : "pool1",
-                 "drivers" : ["i40e"]
-             }
-         ]
+        "clusterType": "physical",
+        "mode": "primary",
+        "logLevel": "debug",
+        "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
+        "pools" : [
+            {
+-              "name" : "e2e",
++              "name" : "pool1",
+               "drivers" : ["i40e"]
+            }
+        ]
 
 ```
 
@@ -302,7 +303,7 @@ ctr -n=k8s.io images import afxdp-device-plugin.tar
 
 ### Verify that the image is now available to the container run-time
 
-For information on how to install crictl : https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/
+More information on how to install crictl can be found [here](https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/).
 
 ```bash
 sudo crictl images
@@ -341,6 +342,10 @@ file to define LIST_OF_QIDS or CNDP_COPY_MODE environment variables. These
 variables are read by `tools/jsonc_gen.sh` when the container starts to
 generate the configuration for the CNDP application.
 
+> **_NOTE:_** the `tools/jsonc_gen.sh` supports BPF map pinning configuration by
+running it with the `-p` flag. If you wish to use it in a kind cluster then run
+the script with a `-k` flag in the pod YAML.
+
 An example to force copy-mode for all AF_XDP sockets:
 
 ```yaml

containerization/k8s/cndp-pods/cndp-0-0.yaml

@@ -16,12 +16,15 @@ spec:
     - name: cndp-0
       command: ["/bin/bash"]
       args: ["-c", "./jsonc_gen.sh; cndpfwd -c config.jsonc lb;"]
+      # args: ["-c", "./jsonc_gen.sh -k -p; cndpfwd -c config.jsonc lb;"] # to run in a kind cluster using bpf map pinning
       image: cndp
       imagePullPolicy: Never
       securityContext:
         capabilities:
           add:
             - NET_RAW
+            - IPC_LOCK
+            # - BPF # Enable if running in a kind cluster as the default kernel used is 5.15, BPF map support without privilege is only supported from 5.19.
       ports:
       - containerPort: 8094
         hostPort: 8094

doc/guides/sample_app_ug/cndpfwd.rst

@@ -201,6 +201,7 @@ The configuration json file is located in the ``cndpfwd`` example sub-directory
         //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, tx-only-rx
         //                    acl-strict, acl-permissive
         //   uds_path   - (0) Path to unix domain socket to get xsk map fd
+        //   xsk-pin-path - (O) Path to pinned bpf map
         "options": {
             "pkt_api": "xskdev",
             "no-metrics": false,

examples/cndpfwd/fwd.jsonc

@@ -124,6 +124,7 @@
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, tx-only-rx,
     //                    acl-strict, acl-permissive, [hyperscan | hs]
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

examples/cndpfwd/main.h

@@ -55,14 +55,15 @@ enum {
     FWD_ACL_STATS   = (1 << 4), /**< Enable printing ACL stats */
 };
 
-#define PKT_API_TAG    "pkt_api"         /**< Packet API json tag */
-#define NO_METRICS_TAG "no-metrics"      /**< json tag for no-metrics */
-#define NO_RESTAPI_TAG "no-restapi"      /**< json tag for no-restapi */
-#define ENABLE_CLI_TAG "cli"             /**< json tag to enable/disable CLI */
-#define MODE_TAG       "mode"            /**< json tag to set the mode flag */
-#define UDS_PATH_TAG   "uds_path"        /**< json tag for UDS to get xsk map fd */
-#define FIB_RULES_TAG  "l3fwd-fib-rules" /**< json tag to set up static FIB entries */
-#define HS_PATTERN_TAG "hs-patterns"     /**< json tag for Hyperscan patterns */
+#define PKT_API_TAG          "pkt_api"         /**< Packet API json tag */
+#define NO_METRICS_TAG       "no-metrics"      /**< json tag for no-metrics */
+#define NO_RESTAPI_TAG       "no-restapi"      /**< json tag for no-restapi */
+#define ENABLE_CLI_TAG       "cli"             /**< json tag to enable/disable CLI */
+#define MODE_TAG             "mode"            /**< json tag to set the mode flag */
+#define UDS_PATH_TAG         "uds_path"        /**< json tag for UDS to get xsk map fd */
+#define XSK_MAP_PIN_PATH_TAG "xsk-pin-path"    /**< json tag for pinned BPF map to get xsk map fd */
+#define FIB_RULES_TAG        "l3fwd-fib-rules" /**< json tag to set up static FIB entries */
+#define HS_PATTERN_TAG       "hs-patterns"     /**< json tag for Hyperscan patterns */
 
 #define MODE_DROP           "drop"           /**< Drop the received packets */
 #define MODE_RX_ONLY        "rx-only"        /**< Alias for MODE_DROP */
@@ -152,6 +153,7 @@ struct fwd_info {
     int burst;                 /**< Burst Size */
     char **fib_rules;          /**< FIB entries */
     uint16_t fib_size;         /**< Number of FIB entries */
+    char *xsk_map_path;        /**< xsk_map pin path */
     char **hs_patterns;        /**< Hyperscan information */
     uint16_t hs_pattern_count; /**< Number of Hyperscan information entries */
     char **hs_expressions;     /**< List of parsed hyperscan regex expressions */

examples/cndpfwd/parse-args.c

@@ -76,6 +76,10 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
                 if (f->xdp_uds == NULL)
                     CNE_ERR_RET("UDS handshake failed\n");
             }
+        } else if (!strncmp(obj.opt->name, XSK_MAP_PIN_PATH_TAG, nlen)) {
+            if (obj.opt->val.type == STRING_OPT_TYPE) {
+                f->xsk_map_path = obj.opt->val.str;
+            }
         } else if (!strncmp(obj.opt->name, FIB_RULES_TAG, nlen)) {
             if (obj.opt->val.type == ARRAY_OPT_TYPE) {
                 f->fib_rules = calloc(obj.opt->val.array_sz, sizeof(char *));
@@ -199,6 +203,8 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx)
             if (lport->flags & LPORT_UNPRIVILEGED) {
                 if (f->xdp_uds)
                     pcfg.xsk_uds = f->xdp_uds;
+                else if (f->xsk_map_path)
+                    pcfg.xsk_map_path = f->xsk_map_path;
                 else
                     CNE_ERR_RET("UDS info struct is null\n");
             }

examples/cnet-graph/cnet-graph.h

@@ -67,9 +67,10 @@ typedef enum {
     }
 // clang-format on
 
-#define NO_METRICS_TAG "no-metrics" /**< json tag for no-metrics */
-#define NO_RESTAPI_TAG "no-restapi" /**< json tag for no-restapi */
-#define ENABLE_CLI_TAG "cli"        /**< json tag to enable/disable CLI */
+#define NO_METRICS_TAG       "no-metrics"   /**< json tag for no-metrics */
+#define NO_RESTAPI_TAG       "no-restapi"   /**< json tag for no-restapi */
+#define ENABLE_CLI_TAG       "cli"          /**< json tag to enable/disable CLI */
+#define XSK_MAP_PIN_PATH_TAG "xsk-pin-path" /**< json tag for pinned BPF map to get xsk map fd */
 
 struct fwd_port {
     int lport;                           /**< PKTDEV lport id */
@@ -110,6 +111,7 @@ struct cnet_info {
     pthread_barrier_t barrier;                /**< Barrier for all threads */
     bool barrier_inited;                      /**< Barrier for all threads */
     graph_info_t graph_info[MAX_GRAPH_COUNT]; /**< Graph information */
+    char *xsk_map_path;                       /**< xsk_map pin path */
 };
 
 extern struct cnet_info *cinfo; /**< global application information pointer */

examples/cnet-graph/cnetfwd-graph.jsonc

@@ -80,6 +80,7 @@
     //    busy_timeout  - (O) 1-65535 or 0 - use default value, values in milliseconds
     //    busy_budget   - (O) 0xFFFF disabled, 0 use default, >0 budget value
     //    inhibit_prog_load - (O) inhibit loading the BPF program if true, default false
+	//    unprivileged  - (O) inhibit loading the BPF program if true, default false
     //    force_wakeup  - (O) force TX wakeup calls for CVL NIC, default false
     //    skb_mode      - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false
     //    description   - (O) the description, 'desc' can be used as well
@@ -124,6 +125,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
+	//   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "no-metrics": false,
         "no-restapi": false,

examples/cnet-graph/parse-args.c

@@ -67,6 +67,10 @@ process_callback(jcfg_info_t *j, void *_obj, void *arg, int idx)
         } else if (!strncmp(obj.opt->name, ENABLE_CLI_TAG, nlen)) {
             if (obj.opt->val.type == BOOLEAN_OPT_TYPE)
                 ci->opts.cli = obj.opt->val.boolean;
+        } else if (!strncmp(obj.opt->name, XSK_MAP_PIN_PATH_TAG, nlen)) {
+            if (obj.opt->val.type == STRING_OPT_TYPE) {
+                ci->xsk_map_path = obj.opt->val.str;
+            }
         }
         break;
 
@@ -155,6 +159,13 @@ process_callback(jcfg_info_t *j, void *_obj, void *arg, int idx)
             }
             pcfg.pi = umem->rinfo[lport->region_idx].pool;
 
+            if (lport->flags & LPORT_UNPRIVILEGED) {
+                if (ci->xsk_map_path)
+                    pcfg.xsk_map_path = ci->xsk_map_path;
+                else
+                    CNE_ERR_RET("BPF MAP path is not configured\n");
+            }
+
             /* Setup the mempool configuration */
             strlcpy(pcfg.pmd_name, lport->pmd_name, sizeof(pcfg.pmd_name));
             strlcpy(pcfg.ifname, lport->netdev, sizeof(pcfg.ifname));

lib/core/xskdev/xskdev.c

@@ -26,6 +26,8 @@
 #include <cne_mutex_helper.h>
 #include <dirent.h>
 #include <limits.h>        // for PATH_MAX
+#include <bpf/bpf.h>
+#include <error.h>
 
 #include "xskdev.h"
 #include "cne_lport.h"        // for lport_stats_t, lport_cfg, lport_cfg_t
@@ -738,16 +740,26 @@ xskdev_socket_create(struct lport_cfg *c)
     xi->xsk_map_fd = -1;
 
     if (c->flags & LPORT_UNPRIVILEGED) {
-        /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client
-         * and receive the XSK_MAP_FD.
-         * We will need (to wait) a flag or re checking of the value of the FD (latter preferred) to
-         * indicate we have completed reception before moving onto the socket create stage
-         */
-        xi->uds_info = (uds_info_t *)c->xsk_uds;
+        if (c->xsk_uds) {
+            /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client
+             * and receive the XSK_MAP_FD.
+             * We will need (to wait) a flag or re checking of the value of the FD (latter
+             * preferred) to indicate we have completed reception before moving onto the socket
+             * create stage
+             */
+            xi->uds_info = (uds_info_t *)c->xsk_uds;
+
+            ret = xskdev_recv_xsk_fd(xi);
+            if (ret < 0)
+                CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n");
+        } else if (c->xsk_map_path) {
+            /* Try to retrieve the xsk_map_fd from a pinned bpf map*/
+            xi->xsk_map_fd = bpf_obj_get(c->xsk_map_path);
+            if (xi->xsk_map_fd < 0)
+                CNE_ERR_GOTO(err, "Failed to get xsk map fd from bpf object\n");
+        }
 
-        ret = xskdev_recv_xsk_fd(xi);
-        if (ret < 0)
-            CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n");
+        CNE_DEBUG("xi->xsk_map_fd = %d\n", xi->xsk_map_fd);
     }
 
     if (xskdev_use_tx_lock) {

lib/include/cne_lport.h

@@ -85,6 +85,7 @@ typedef struct lport_cfg {
     size_t umem_size;              /**< Size of the umem region */
     pktmbuf_info_t *pi;            /**< pktmbuf_info_t structure pointer */
     void *xsk_uds;                 /**< The UDS to connect to get xsk FDs */
+    char *xsk_map_path;            /**< The pinned map to get xsk FD */
     lport_buf_mgmt_t buf_mgmt;     /**< Buffer management functions */
 } lport_cfg_t;