diff --git a/ansible/cndp-config/templates/fwd.jsonc.j2 b/ansible/cndp-config/templates/fwd.jsonc.j2 index 703e60c8..8f76bc20 100644 --- a/ansible/cndp-config/templates/fwd.jsonc.j2 +++ b/ansible/cndp-config/templates/fwd.jsonc.j2 @@ -86,10 +86,10 @@ // busy_polling - Same as above // busy_timeout - (O) 1-65535 or 0 - use default value, values in milliseconds // busy_budget - (O) 0xFFFF disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well + // xsk_pin_path - (O) Path to pinned bpf map "lports": { {% for i in range(num_lports) %} "{{iface}}:{{i}}": { diff --git a/containerization/docker/ubuntu/fwd.jsonc b/containerization/docker/ubuntu/fwd.jsonc index 1b42bbb6..777d589c 100644 --- a/containerization/docker/ubuntu/fwd.jsonc +++ b/containerization/docker/ubuntu/fwd.jsonc @@ -79,10 +79,10 @@ // busy_polling - Same as above // busy_timeout - (O) 1-65535 or 0 - use default value, values in milliseconds // busy_budget - (O) 0xFFFF disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well + // xsk_pin_path - (O) Path to pinned xsk map for this port "lports": { "net1:0": { "pmd": "net_af_xdp", diff --git a/containerization/k8s/README.md b/containerization/k8s/README.md index e80ba6df..6c62bee3 100644 --- a/containerization/k8s/README.md +++ b/containerization/k8s/README.md @@ -1,4 +1,4 @@ -### CNDP Pod +# CNDP Pod The provided configuration creates a CNDP pod with two containers. One that runs the CNDP cndpfwd example and another which runs the prometheus go agent to collect @@ -7,7 +7,7 @@ a unix domain socket. This guide will walk you through the setup of the CNDP pods. -### Setup K8s Env +## Setup K8s Env This guide will walk you through how to setup a single node cluster where you can launch a CNDP container. The example uses kubeadm to bootstrap the cluster. @@ -218,7 +218,7 @@ container image used by the CNDP pod. ### Build and deploy AF_XDP plugins for K8s -The source code is available at: https://github.com/intel/afxdp-plugins-for-kubernetes +The source code is available [here](https://github.com/intel/afxdp-plugins-for-kubernetes). For detailed install instructions please refer to README.md in the device plugin repo. This section will provide a quick start for deploying the device plugin and CNI. @@ -254,16 +254,17 @@ index 8465a21..774c6ca 100644 data: config.json: | { - "mode": "cndp", - "logLevel": "debug", - "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log", - "pools" : [ - { -- "name" : "e2e", -+ "name" : "pool1", - "drivers" : ["i40e"] - } - ] + "clusterType": "physical", + "mode": "primary", + "logLevel": "debug", + "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log", + "pools" : [ + { +- "name" : "e2e", ++ "name" : "pool1", + "drivers" : ["i40e"] + } + ] ``` @@ -302,7 +303,7 @@ ctr -n=k8s.io images import afxdp-device-plugin.tar ### Verify that the image is now available to the container run-time -For information on how to install crictl : https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/ +More information on how to install crictl can be found [here](https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/). ```bash sudo crictl images @@ -341,6 +342,10 @@ file to define LIST_OF_QIDS or CNDP_COPY_MODE environment variables. These variables are read by `tools/jsonc_gen.sh` when the container starts to generate the configuration for the CNDP application. +> **_NOTE:_** the `tools/jsonc_gen.sh` supports BPF map pinning configuration by +running it with the `-p` flag. If you wish to use it in a kind cluster then run +the script with a `-k` flag in the pod YAML. + An example to force copy-mode for all AF_XDP sockets: ```yaml diff --git a/containerization/k8s/cndp-pods/cndp-0-0.yaml b/containerization/k8s/cndp-pods/cndp-0-0.yaml index 197ebee9..2449fa6c 100644 --- a/containerization/k8s/cndp-pods/cndp-0-0.yaml +++ b/containerization/k8s/cndp-pods/cndp-0-0.yaml @@ -16,12 +16,15 @@ spec: - name: cndp-0 command: ["/bin/bash"] args: ["-c", "./jsonc_gen.sh; cndpfwd -c config.jsonc lb;"] + # args: ["-c", "./jsonc_gen.sh -k -p; cndpfwd -c config.jsonc lb;"] # to run in a kind cluster using bpf map pinning image: cndp imagePullPolicy: Never securityContext: capabilities: add: - NET_RAW + - IPC_LOCK + # - BPF # Enable if running in a kind cluster as the default kernel used is 5.15, BPF map support without privilege is only supported from 5.19. ports: - containerPort: 8094 hostPort: 8094 diff --git a/doc/guides/sample_app_ug/cndpfwd.rst b/doc/guides/sample_app_ug/cndpfwd.rst index aa3995d6..936e443a 100644 --- a/doc/guides/sample_app_ug/cndpfwd.rst +++ b/doc/guides/sample_app_ug/cndpfwd.rst @@ -31,10 +31,11 @@ The creation of an AF_XDP socket involves loading of a BPF program which is a pr In order to run the CNDP application in an unprivileged container, the privileged operations are done by a Kubernetes device plugin. The CNDP application talks to the Kubernetes device plugin over a unix domain socket. The path to the unix domain socket created by the device plugin is the value -of the "uds_path" attribute. The "unprivileged" flag should be true if running the CNDP app in an -unprivileged container. The sysctl param ``kernel.unprivileged_bpf_disabled`` should be 0 to perform -unprivileged BPF operations. For more details about the device plugin, please refer to -:ref:`Integration of the K8s device plugin with CNDP `. +of the "uds_path" attribute. The "unprivileged" flag is automatically configured if the "uds_path" +parameter is used. The sysctl param ``kernel.unprivileged_bpf_disabled`` should be 0 to perform +unprivileged BPF operations. Alternatively the "xsk_pin_path" can be used in combination with the +AF_XDP Device Plugin to support unprivileged Pods. For more details about the device plugin, please refer to +:ref:`Integration of the K8s device plugin with CNDP `. Alternatively Running the Application ----------------------- @@ -156,10 +157,10 @@ The configuration json file is located in the ``cndpfwd`` example sub-directory // busy_poll - (O) Enable busy polling support, true or false, default false // busy_timeout - (O) 1-65535 or 0 - use default value, values in milliseconds // busy_budget - (O) 0xFFFF disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well + // xsk_pin_path - (O) Path to pinned xsk map for this port "lports": { "eth0:0": { "pmd": "net_af_xdp", diff --git a/examples/cndpfwd/fwd.jsonc b/examples/cndpfwd/fwd.jsonc index 2c952307..8444e6ba 100644 --- a/examples/cndpfwd/fwd.jsonc +++ b/examples/cndpfwd/fwd.jsonc @@ -79,9 +79,9 @@ // busy_polling - Same as above // busy_timeout - (O) 1-65535 or 0 - use default value, values in milliseconds // busy_budget - (O) 0xFFFF disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false + // xsk_pin_path - (O) Path to pinned xsk map for this port // description - (O) the description, 'desc' can be used as well "lports": { "eth0:0": { diff --git a/examples/cndpfwd/parse-args.c b/examples/cndpfwd/parse-args.c index c0ffcf88..d2b5f911 100644 --- a/examples/cndpfwd/parse-args.c +++ b/examples/cndpfwd/parse-args.c @@ -33,6 +33,7 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx) uint32_t total_region_cnt; char *umem_addr; size_t nlen; + jcfg_lport_t *lport; if (!_obj) return -1; @@ -157,7 +158,7 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx) case JCFG_LPORT_TYPE: do { - jcfg_lport_t *lport = obj.lport; + lport = obj.lport; struct fwd_port *pd; mmap_t *mm; jcfg_umem_t *umem; @@ -188,6 +189,11 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx) pcfg.flags = lport->flags; pcfg.flags |= (umem->shared_umem == 1) ? LPORT_SHARED_UMEM : 0; + if (lport->xsk_map_path) { + cne_printf("[yellow]**** [green]PINNED_BPF_MAP is [red]enabled[]\n"); + pcfg.xsk_map_path = lport->xsk_map_path; + } + pcfg.addr = jcfg_lport_region(lport, &pcfg.bufcnt); if (!pcfg.addr) { free(pd); @@ -196,12 +202,11 @@ process_callback(jcfg_info_t *j __cne_unused, void *_obj, void *arg, int idx) } pcfg.pi = umem->rinfo[lport->region_idx].pool; - if (lport->flags & LPORT_UNPRIVILEGED) { - if (f->xdp_uds) - pcfg.xsk_uds = f->xdp_uds; - else - CNE_ERR_RET("UDS info struct is null\n"); + if (f->xdp_uds) { + pcfg.xsk_uds = f->xdp_uds; + lport->flags |= LPORT_UNPRIVILEGED; } + /* Setup the mempool configuration */ strlcpy(pcfg.pmd_name, lport->pmd_name, sizeof(pcfg.pmd_name)); strlcpy(pcfg.ifname, lport->netdev, sizeof(pcfg.ifname)); diff --git a/lang/rs/apis/cne/fwd.jsonc b/lang/rs/apis/cne/fwd.jsonc index eaf9c8cc..4ec7fdde 100644 --- a/lang/rs/apis/cne/fwd.jsonc +++ b/lang/rs/apis/cne/fwd.jsonc @@ -75,7 +75,6 @@ // busy_poll - (O) Enable busy polling support, true or false, default false // busy_timeout - (O) 1-65535 or 0 - use default value, values in milli-seconds // busy_budget - (O) -1 disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well diff --git a/lang/rs/examples/echo_server/fwd.jsonc b/lang/rs/examples/echo_server/fwd.jsonc index eaf9c8cc..4ec7fdde 100644 --- a/lang/rs/examples/echo_server/fwd.jsonc +++ b/lang/rs/examples/echo_server/fwd.jsonc @@ -75,7 +75,6 @@ // busy_poll - (O) Enable busy polling support, true or false, default false // busy_timeout - (O) 1-65535 or 0 - use default value, values in milli-seconds // busy_budget - (O) -1 disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well diff --git a/lang/rs/examples/fwd/fwd.jsonc b/lang/rs/examples/fwd/fwd.jsonc index 48eb66a9..681166ac 100644 --- a/lang/rs/examples/fwd/fwd.jsonc +++ b/lang/rs/examples/fwd/fwd.jsonc @@ -75,7 +75,6 @@ // busy_poll - (O) Enable busy polling support, true or false, default false // busy_timeout - (O) 1-65535 or 0 - use default value, values in milli-seconds // busy_budget - (O) -1 disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well diff --git a/lib/core/xskdev/xskdev.c b/lib/core/xskdev/xskdev.c index 90d28cc9..b8d3e897 100644 --- a/lib/core/xskdev/xskdev.c +++ b/lib/core/xskdev/xskdev.c @@ -26,6 +26,8 @@ #include #include #include // for PATH_MAX +#include +#include #include "xskdev.h" #include "cne_lport.h" // for lport_stats_t, lport_cfg, lport_cfg_t @@ -738,16 +740,26 @@ xskdev_socket_create(struct lport_cfg *c) xi->xsk_map_fd = -1; if (c->flags & LPORT_UNPRIVILEGED) { - /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client - * and receive the XSK_MAP_FD. - * We will need (to wait) a flag or re checking of the value of the FD (latter preferred) to - * indicate we have completed reception before moving onto the socket create stage - */ - xi->uds_info = (uds_info_t *)c->xsk_uds; + if (c->xsk_uds) { + /* If UDS is set then call xskdev_recv_xsk_fd to setup a UDS client + * and receive the XSK_MAP_FD. + * We will need (to wait) a flag or re checking of the value of the FD (latter + * preferred) to indicate we have completed reception before moving onto the socket + * create stage + */ + xi->uds_info = (uds_info_t *)c->xsk_uds; + + ret = xskdev_recv_xsk_fd(xi); + if (ret < 0) + CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n"); + } else if (c->xsk_map_path) { + /* Try to retrieve the xsk_map_fd from a pinned bpf map*/ + xi->xsk_map_fd = bpf_obj_get(c->xsk_map_path); + if (xi->xsk_map_fd < 0) + CNE_ERR_GOTO(err, "Failed to get xsk map fd from bpf object\n"); + } - ret = xskdev_recv_xsk_fd(xi); - if (ret < 0) - CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n"); + CNE_DEBUG("xi->xsk_map_fd = %d\n", xi->xsk_map_fd); } if (xskdev_use_tx_lock) { diff --git a/lib/include/cne_lport.h b/lib/include/cne_lport.h index adfee24e..d952a0d4 100644 --- a/lib/include/cne_lport.h +++ b/lib/include/cne_lport.h @@ -85,6 +85,7 @@ typedef struct lport_cfg { size_t umem_size; /**< Size of the umem region */ pktmbuf_info_t *pi; /**< pktmbuf_info_t structure pointer */ void *xsk_uds; /**< The UDS to connect to get xsk FDs */ + char *xsk_map_path; /**< The pinned map to get xsk FD */ lport_buf_mgmt_t buf_mgmt; /**< Buffer management functions */ } lport_cfg_t; diff --git a/lib/usr/app/jcfg/jcfg.h b/lib/usr/app/jcfg/jcfg.h index b717451d..0102e1e1 100644 --- a/lib/usr/app/jcfg/jcfg.h +++ b/lib/usr/app/jcfg/jcfg.h @@ -171,7 +171,8 @@ typedef struct jcfg_lport { uint16_t qid; /**< The queue ID number */ uint16_t busy_timeout; /**< busy timeout value in milliseconds */ uint16_t busy_budget; /**< busy budget 0xFFFF disabled, 0 use default, >0 budget */ - uint16_t flags; /**< Flags to configure lport in lport_cfg_t.flags in cne_lport.h */ + uint16_t flags; /**< Flags to configure lport in lport_cfg_t.flags in cne_lport.h */ + char *xsk_map_path; /**< The path to the pinned xsk_map for this port */ } jcfg_lport_t; /** JCFG lport configuration names */ @@ -185,7 +186,7 @@ typedef struct jcfg_lport { #define JCFG_LPORT_BUSY_POLLING_NAME "busy_polling" #define JCFG_LPORT_BUSY_TIMEOUT_NAME "busy_timeout" #define JCFG_LPORT_BUSY_BUDGET_NAME "busy_budget" -#define JCFG_LPORT_UNPRIVILEGED_NAME "unprivileged" +#define JCFG_PINNED_XSK_MAP_NAME "xsk_pin_path" #define JCFG_LPORT_FORCE_WAKEUP_NAME "force_wakeup" #define JCFG_LPORT_SKB_MODE_NAME "skb_mode" @@ -237,6 +238,7 @@ typedef struct jcfg_lport_group { uint16_t busy_timeout; /**< busy timeout value in milliseconds */ uint16_t busy_budget; /**< busy budget 0xFFFF disabled, 0 use default, >0 budget */ uint16_t flags; /**< Flags to configure lport in lport_cfg_t.flags */ + } jcfg_lport_group_t; /** JCFG lport group configuration names */ diff --git a/lib/usr/app/jcfg/jcfg_lport.c b/lib/usr/app/jcfg/jcfg_lport.c index 39d8e77d..f1b3b89f 100644 --- a/lib/usr/app/jcfg/jcfg_lport.c +++ b/lib/usr/app/jcfg/jcfg_lport.c @@ -108,9 +108,10 @@ _lport(struct json_object *obj, int flags, struct json_object *parent __cne_unus CNE_ERR_RET_VAL(JSON_C_VISIT_RETURN_ERROR, "%s: Invalid Range\n", JCFG_LPORT_BUSY_BUDGET_NAME); lport->busy_budget = (uint16_t)val; - } else if (!strncmp(key, JCFG_LPORT_UNPRIVILEGED_NAME, keylen)) - lport->flags |= json_object_get_boolean(obj) ? LPORT_UNPRIVILEGED : 0; - else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen)) + } else if (!strncmp(key, JCFG_PINNED_XSK_MAP_NAME, keylen)) { + lport->xsk_map_path = strndup(json_object_get_string(obj), JCFG_MAX_STRING_SIZE); + lport->flags |= LPORT_UNPRIVILEGED; + } else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen)) lport->flags |= json_object_get_boolean(obj) ? LPORT_FORCE_WAKEUP : 0; else if (!strncmp(key, JCFG_LPORT_SKB_MODE_NAME, keylen)) lport->flags |= json_object_get_boolean(obj) ? LPORT_SKB_MODE : 0; diff --git a/lib/usr/app/jcfg/jcfg_lport_group.c b/lib/usr/app/jcfg/jcfg_lport_group.c index 4cab667e..3a19d5ee 100644 --- a/lib/usr/app/jcfg/jcfg_lport_group.c +++ b/lib/usr/app/jcfg/jcfg_lport_group.c @@ -748,9 +748,7 @@ _lport_group(struct json_object *obj, int flags, struct json_object *parent __cn CNE_ERR_RET_VAL(JSON_C_VISIT_RETURN_ERROR, "%s: Invalid Range\n", JCFG_LPORT_BUSY_BUDGET_NAME); lpg->busy_budget = (uint16_t)val; - } else if (!strncmp(key, JCFG_LPORT_UNPRIVILEGED_NAME, keylen)) - lpg->flags |= json_object_get_boolean(obj) ? LPORT_UNPRIVILEGED : 0; - else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen)) + } else if (!strncmp(key, JCFG_LPORT_FORCE_WAKEUP_NAME, keylen)) lpg->flags |= json_object_get_boolean(obj) ? LPORT_FORCE_WAKEUP : 0; else if (!strncmp(key, JCFG_LPORT_SKB_MODE_NAME, keylen)) lpg->flags |= json_object_get_boolean(obj) ? LPORT_SKB_MODE : 0; diff --git a/tools/cndp.schema b/tools/cndp.schema index ec4f00be..8769c56d 100644 --- a/tools/cndp.schema +++ b/tools/cndp.schema @@ -152,11 +152,6 @@ "minimum": 0, "maximum": 65535 }, - "unprivileged": { - "description": "Enable unprivileged operations", - "type": "boolean", - "default": false - }, "force_wakeup": { "description": "Force Tx wakeup calls for CVL NIC", "type": "boolean", @@ -330,11 +325,6 @@ "minimum": 0, "maximum": 65535 }, - "unprivileged": { - "description": "Enable unprivileged operations", - "type": "boolean", - "default": false - }, "force_wakeup": { "description": "Force Tx wakeup calls for CVL NIC", "type": "boolean", diff --git a/tools/jsonc_gen.sh b/tools/jsonc_gen.sh index c2822426..168df77a 100755 --- a/tools/jsonc_gen.sh +++ b/tools/jsonc_gen.sh @@ -13,22 +13,36 @@ # The available cores to the application are determined by # the 'lscpu' command. KIND=false +PINNED_BPF_MAP=false config_file=config.jsonc AFXDP_DEVICES=${AFXDP_DEVICES:-net1} AFXDP_COPY_MODE=${AFXDP_COPY_MODE:-false} -while getopts "k" flag; do +while getopts "kp" flag; do case $flag in k) KIND=true ;; + p) PINNED_BPF_MAP=true ;; *) echo 'error unkown flag' >&2 exit 1 esac done if [ ${KIND} = false ] ; then -LIST_OF_QIDS=${LIST_OF_QIDS:-4} + LIST_OF_QIDS=${LIST_OF_QIDS:-4} else -LIST_OF_QIDS=${LIST_OF_QIDS:-0} # For kind using veth use queue 0 + LIST_OF_QIDS=${LIST_OF_QIDS:-0} # For kind using veth use queue 0 +fi + +if [ ${PINNED_BPF_MAP} = false ] ; then + UDS_PATH="uds_path" + UDS="/tmp/afxdp.sock" + UDS_CFG=("\"$UDS_PATH\": \"$UDS\",") + unset "${MAP_CFG[@]}" +else + MAP_PATH="xsk_pin_path" + MAP="/tmp/xsks_map" + MAP_CFG=("\"$MAP_PATH\": \"$MAP\",") + unset "${UDS_CFG[@]}" fi num_of_interfaces=0 @@ -73,25 +87,35 @@ function get_lcore() echo """${LCORE[index]}""" } -output=$(lscpu | grep "NUMA node[0-9] CPU") -# Parse the list of cores for each numa node and store it in an array -IFS=$':\n'; -for each_output in $output -do - if [ $((l%2)) -eq 0 ] - then - ((num_of_numa_nodes++)) - else - # split the comma separated value of cores into an array - IFS=', ' read -ra list_of_numa_lcores <<< "$each_output" +if [ $KIND = false ] ; then + output=$(lscpu | grep "NUMA node[0-9] CPU") + # Parse the list of cores for each numa node and store it in an array + IFS=$':\n'; + for each_output in $output + do + if [ $((l%2)) -eq 0 ] + then + ((num_of_numa_nodes++)) + else + # split the comma separated value of cores into an array + IFS=', ' read -ra list_of_numa_lcores <<< "$each_output" - num_of_cores_in_each_numa_node[numa_node++]=${#list_of_numa_lcores[@]} - LCORE=("${LCORE[@]}" "${list_of_numa_lcores[@]}") - fi - ((l++)) -done -unset IFS + num_of_cores_in_each_numa_node[numa_node++]=${#list_of_numa_lcores[@]} + LCORE=("${LCORE[@]}" "${list_of_numa_lcores[@]}") + fi + ((l++)) + done + unset IFS +else + output=$(lscpu -b -p=Core | grep -v '^#') + IFS=$'\n'; + for each_output in $output + do + LCORE=("${LCORE[@]}" "$each_output") + done + unset IFS +fi while [ $i -lt $num_of_interfaces ] do @@ -110,8 +134,8 @@ EOF "qid": ${QID[i]}, "umem": "umem0", "region": ${i}, - "unprivileged": true, "skb_mode": ${AFXDP_COPY_MODE}, + ${MAP_CFG[@]} "description": "LAN ${i} port" } EOF @@ -130,9 +154,9 @@ EOF ) if [ $KIND = false ] ; then nic_numa_node=$(cat /sys/class/net/"${NET[i]}"/device/numa_node || echo 0) - lcore=$(get_lcore "$nic_numa_node" $i) + lcore=$(get_lcore "$nic_numa_node" $i) else - lcore=0 # For kind you can't really retrieve the numa node as shown above + lcore=0 # For kind you can't really retrieve the nic_numa_node as shown above fi # create list of lcore groups lcore_groups[i]=$( @@ -197,7 +221,7 @@ cat <<-EOF > ${config_file} // description | desc - (O) Description of the umem space. "umems": { "umem0": { - "bufcnt": (16*$num_of_interfaces), + "bufcnt": $((num_of_interfaces * 16)), "bufsz": 2, "mtype": "2MB", "regions": [${regions[*]} @@ -227,10 +251,10 @@ cat <<-EOF > ${config_file} // busy_polling - Same as above // busy_timeout - (O) 1-65535 or 0 - use default value, values in milliseconds // busy_budget - (O) 0xFFFF disabled, 0 use default, >0 budget value - // unprivileged - (O) inhibit loading the BPF program if true, default false // force_wakeup - (O) force TX wakeup calls for CVL NIC, default false // skb_mode - (O) Enable XDP_FLAGS_SKB_MODE when creating af_xdp socket, forces copy mode, default false // description - (O) the description, 'desc' can be used as well + // xsk_pin_path - (O) Path to pinned xsk map for this port "lports": {${lports[*]} }, @@ -242,7 +266,7 @@ cat <<-EOF > ${config_file} // The default group is special and is used if a thread if not assigned to a group. "lcore-groups": { "initial": ["${LCORE[i]}"], - ${lcore_groups[*]}, + ${lcore_groups[*]}, "default": ["${LCORE[i+1]}"] }, @@ -261,7 +285,7 @@ cat <<-EOF > ${config_file} "no-metrics": false, "no-restapi": false, "cli": false, - "uds_path": "/tmp/afxdp.sock", + ${UDS_CFG[@]} "mode": "drop" }, diff --git a/tools/jsonc_gen_lport_groups.sh b/tools/jsonc_gen_lport_groups.sh index 47ac5e2c..12bf2fef 100755 --- a/tools/jsonc_gen_lport_groups.sh +++ b/tools/jsonc_gen_lport_groups.sh @@ -66,9 +66,9 @@ function build_netdevs_by_node local node local dev - for dev in ${CNDP_DEVICES[@]}; do - node=$(cat /sys/class/net/$dev/device/numa_node 2>/dev/null || echo 0) - netdevs_by_node[$node]="${netdevs_by_node[$node]} $dev" + for dev in "${CNDP_DEVICES[@]}"; do + node=$(cat /sys/class/net/"$dev"/device/numa_node 2>/dev/null || echo 0) + netdevs_by_node[node]="${netdevs_by_node[$node]} $dev" done } @@ -77,12 +77,13 @@ function build_netdevs_by_node # function build_lcores_by_node { - local lines=$(lscpu | awk -F: '/NUMA node[0-9] CPU\(s\):/{print $2}' | xargs) + local lines + lines=$(lscpu | awk -F: '/NUMA node[0-9] CPU\(s\):/{print $2}' | xargs) local line local i i=0 - for line in ${lines[@]}; do + for line in "${lines[@]}"; do lcores_by_node[i]=$line i=$((i+1)) done @@ -220,7 +221,6 @@ function emit_jsonc_lport_groups "threads": ["fwd:$i"], "pmd": "net_af_xdp", "skb_mode": $CNDP_COPY_MODE, - "unprivileged": true } EOF diff --git a/usrtools/txgen/app/txgen.c b/usrtools/txgen/app/txgen.c index a5b7bd9c..4d122c49 100644 --- a/usrtools/txgen/app/txgen.c +++ b/usrtools/txgen/app/txgen.c @@ -291,6 +291,7 @@ txgen_packet_ctor(port_info_t *info) /* IPv4 Header constructor */ txgen_ipv4_ctor(pkt, l3_hdr); } + } else cne_printf("Unknown EtherType 0x%04x", pkt->ethType); } @@ -560,6 +561,8 @@ txgen_send_pkts(port_info_t *info) xb->data_len = info->pkt.pktSize; memcpy(pktmbuf_mtod(xb, uint8_t *), (uint8_t *)&info->pkt.hdr, xb->data_len); } + + cksum(pktmbuf_mtod(xb, uint8_t *), xb->data_len, 0); } info->tx_mbufs.len += nb; diff --git a/usrtools/xskmap_load_send/README.md b/usrtools/xskmap_load_send/README.md index 046ddc41..203133d9 100644 --- a/usrtools/xskmap_load_send/README.md +++ b/usrtools/xskmap_load_send/README.md @@ -3,8 +3,8 @@ This is a simple utility that loads a pinned xsk_map fd and sends it to a CNDP application that's configured to work in unprivileged mode. It allows the end user to load their own custom BPF programs besides the -default AF_XDP redirect program provided by the kernel. -For e.g. https://github.com/maryamtahhan/xdp-progs/blob/main/xdp-filter-udp/xdp_prog_kern.c +default AF_XDP redirect program provided by the kernel. For +[example](https://github.com/maryamtahhan/xdp-progs/blob/main/xdp-filter-udp/xdp_prog_kern.c). > **_NOTE_** The loading is left outside the scope of CNDP as there are many options available for use. @@ -22,7 +22,6 @@ application to be used alongside this utility is shown below: "umem": "umem0", "region": 0, "description": "LAN 0 port", - "unprivileged": true } }, ...