Merge pull request #9590 from tomastigera/tomas-bpf-source-counters

[BPF] extra conntrack counters and stats

felix/Makefile

@@ -278,7 +278,7 @@ $(FELIX_CONTAINER_CREATED): register \
                             docker-image/felix.cfg \
                             docker-image/Dockerfile \
                             $(shell test "$(FELIX_IMAGE_ID)" || echo force-rebuild)
-	$(DOCKER_BUILD) -t $(FELIX_IMAGE_WITH_TAG) -f ./docker-image/Dockerfile docker-image
+	$(DOCKER_BUILD) --network=host -t $(FELIX_IMAGE_WITH_TAG) -f ./docker-image/Dockerfile docker-image
 	$(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest
 	touch $(FELIX_CONTAINER_CREATED)
 

felix/bpf-gpl/conntrack.h

@@ -233,6 +233,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx,
 		int i;
 
 		CALI_DEBUG("Source collision for " IP_FMT ":%d", debug_ip(ct_ctx->src), sport);
+		counter_inc(ctx, CALI_REASON_SOURCE_COLLISION);
 
 		ct_value.orig_sport = sport;
 
@@ -258,6 +259,7 @@ static CALI_BPF_INLINE int calico_ct_v4_create_tracking(struct cali_tc_ctx *ctx,
 			CALI_INFO("Source collision unresolved " IP_FMT ":%d",
 					debug_ip(ct_ctx->src), ct_value.orig_sport);
 			err = -17; /* EEXIST */
+			counter_inc(ctx, CALI_REASON_SOURCE_COLLISION_FAILED);
 		}
 	}
 

felix/bpf-gpl/counters.h

@@ -7,7 +7,7 @@
 
 #include "bpf.h"
 
-#define MAX_COUNTERS_SIZE 14
+#define MAX_COUNTERS_SIZE 17
 
 typedef __u64 counters_t[MAX_COUNTERS_SIZE];
 
@@ -20,7 +20,7 @@ struct counters_key {
 #define COUNTERS_TC_EGRESS	1
 #define COUNTERS_XDP		2
 
-CALI_MAP(cali_counters, 2,
+CALI_MAP(cali_counters, 3,
 		BPF_MAP_TYPE_PERCPU_HASH,
 		struct counters_key, counters_t, 20000,
 		0)

felix/bpf-gpl/reasons.h

@@ -23,6 +23,9 @@ enum calico_reason {
 	CALI_REASON_UNAUTH_SOURCE,
 	CALI_REASON_RT_UNKNOWN,
 	CALI_REASON_BLACK_HOLE,
+	CALI_REASON_SOURCE_COLLISION,
+	CALI_REASON_SOURCE_COLLISION_FAILED,
+	CALI_REASON_CT_CREATE_FAILED,
 	CALI_REASON_ACCEPTED_BY_XDP, // Not used by counters map
 	CALI_REASON_WEP_NOT_READY,
 	CALI_REASON_NATIFACE,

felix/bpf-gpl/tc.c

@@ -806,6 +806,7 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx,
 			int err;
 			if ((err = conntrack_create(ctx, ct_ctx_nat))) {
 				CALI_DEBUG("Creating NAT conntrack failed with %d", err);
+				deny_reason(ctx, CALI_REASON_CT_CREATE_FAILED);
 				goto deny;
 			}
 			STATE->ct_result.nat_sip = ct_ctx_nat->src;
@@ -1399,7 +1400,7 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb)
 				CALI_DEBUG("Allowing local host traffic without CT");
 				goto allow;
 			}
-
+			deny_reason(ctx, CALI_REASON_CT_CREATE_FAILED);
 			goto deny;
 		}
 		goto allow;

felix/bpf/counters/counters.go

@@ -27,7 +27,7 @@ import (
 )
 
 const (
-	MaxCounterNumber    int = 14
+	MaxCounterNumber    int = 17
 	counterMapKeySize   int = 8
 	counterMapValueSize int = 8
 )
@@ -73,6 +73,9 @@ const (
 	DroppedUnauthSource
 	DroppedUnknownRoute
 	DroppedBlackholeRoute
+	SourceCollisionHit
+	SourceCollisionResolutionFailed
+	ConntrackCreateFailed
 )
 
 type Description struct {
@@ -155,6 +158,18 @@ var descriptions DescList = DescList{
 		Counter:  DroppedBlackholeRoute,
 		Category: "Dropped", Caption: "packets hitting blackhole route",
 	},
+	{
+		Counter:  SourceCollisionHit,
+		Category: "Other", Caption: "packets hitting NAT source collision",
+	},
+	{
+		Counter:  ConntrackCreateFailed,
+		Category: "Dropped", Caption: "failed to create conntrack",
+	},
+	{
+		Counter:  SourceCollisionResolutionFailed,
+		Category: "Dropped", Caption: "NAT source collision resolution failed",
+	},
 }
 
 func Descriptions() DescList {

felix/bpf/counters/map.go

@@ -29,7 +29,7 @@ var MapParameters = maps.MapParameters{
 	ValueSize:  counterMapValueSize * MaxCounterNumber,
 	MaxEntries: 20000,
 	Name:       "cali_counters",
-	Version:    2,
+	Version:    3,
 }
 
 func Map() maps.Map {

felix/bpf/ut/nat_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/projectcalico/calico/felix/bpf/conntrack"
 	conntrack3 "github.com/projectcalico/calico/felix/bpf/conntrack/v3"
 	v3 "github.com/projectcalico/calico/felix/bpf/conntrack/v3"
+	"github.com/projectcalico/calico/felix/bpf/counters"
 	"github.com/projectcalico/calico/felix/bpf/nat"
 	"github.com/projectcalico/calico/felix/bpf/polprog"
 	"github.com/projectcalico/calico/felix/bpf/routes"
@@ -2388,6 +2389,7 @@ func TestNATSourceCollision(t *testing.T) {
 	bpfIfaceName = "SPRT"
 	defer func() { bpfIfaceName = "" }()
 	resetCTMap(ctMap)
+	resetCTMap(countersMap)
 
 	// Setup node2 with backend pod such that conntrack has an active TCP
 	// connection with which we will collide the next SYN.
@@ -2605,6 +2607,11 @@ func TestNATSourceCollision(t *testing.T) {
 		tcp := tcpL.(*layers.TCP)
 		Expect(uint16(tcp.SrcPort)).To(Equal(newSPort))
 
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.SourceCollisionHit])).To(Equal(1))
+		Expect(int(bpfCounters[counters.SourceCollisionResolutionFailed])).To(Equal(0))
+
 		recvPkt = res.dataOut
 	})
 	expectMark(tcdefs.MarkSeen)
@@ -2651,6 +2658,10 @@ func TestNATSourceCollision(t *testing.T) {
 		res, err := bpfrun(pktBytes)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(res.Retval).To(Equal(resTC_ACT_SHOT))
+		bpfCounters, err := counters.Read(countersMap, 1, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(int(bpfCounters[counters.SourceCollisionHit])).To(Equal(2))
+		Expect(int(bpfCounters[counters.SourceCollisionResolutionFailed])).To(Equal(1))
 	}, withPSNATPorts(22222, 22222))
 
 	// It should eventually succeed if we keep retransmitting and it is possible to pick

felix/cmd/calico-bpf/commands/conntrack.go

@@ -46,6 +46,7 @@ func init() {
 	conntrackCmd.AddCommand(newConntrackWriteCmd())
 	conntrackCmd.AddCommand(newConntrackFillCmd())
 	conntrackCmd.AddCommand(newConntrackCreateCmd())
+	conntrackCmd.AddCommand(newConntrackStatsCmd())
 	rootCmd.AddCommand(conntrackCmd)
 }
 
@@ -216,12 +217,14 @@ func (cmd *conntrackDumpCmd) prettyDump(k conntrack.KeyInterface, v conntrack.Va
 	}
 
 	now := bpf.KTimeNanos()
-	cmd.Printf(" Age: %s Active ago %s",
-		time.Duration(now-v.Created()), time.Duration(now-v.LastSeen()))
+	cmd.Printf(" Age: %s Active ago %s Duration %s",
+		time.Duration(now-v.Created()), time.Duration(now-v.LastSeen()), time.Duration(v.LastSeen()-v.Created()))
 
 	if k.Proto() == 6 {
-		if (v.IsForwardDSR() && d.FINsSeenDSR()) || d.FINsSeen() || d.RSTSeen() {
+		if (v.IsForwardDSR() && d.FINsSeenDSR()) || d.FINsSeen() {
 			cmd.Printf(" CLOSED")
+		} else if d.RSTSeen() {
+			cmd.Printf(" RESET")
 		} else if d.Established() {
 			cmd.Printf(" ESTABLISHED")
 		} else {
@@ -248,11 +251,16 @@ func dumpExtrav2(k v2.Key, v v2.Value) {
 
 	data := v.Data()
 
-	if (v.IsForwardDSR() && data.FINsSeenDSR()) || data.FINsSeen() || data.RSTSeen() {
+	if (v.IsForwardDSR() && data.FINsSeenDSR()) || data.FINsSeen() {
 		fmt.Printf(" CLOSED")
 		return
 	}
 
+	if data.RSTSeen() {
+		fmt.Printf(" RESET")
+		return
+	}
+
 	if data.Established() {
 		fmt.Printf(" ESTABLISHED")
 		return
@@ -282,6 +290,11 @@ func dumpExtra(k conntrack.KeyInterface, v conntrack.ValueInterface) {
 		return
 	}
 
+	if data.RSTSeen() {
+		fmt.Printf(" RESET")
+		return
+	}
+
 	if data.Established() {
 		fmt.Printf(" ESTABLISHED")
 		return
@@ -611,3 +624,98 @@ func (cmd *conntrackFillCmd) Run(c *cobra.Command, _ []string) {
 		}
 	}
 }
+
+type conntrackStatsCmd struct {
+	*cobra.Command
+	ipv6 bool
+
+	established int
+	reset       int
+	closed      int
+	synSent     int
+	total       int
+	nat         int
+
+	protos map[int]int
+}
+
+func newConntrackStatsCmd() *cobra.Command {
+	cmd := &conntrackStatsCmd{
+		Command: &cobra.Command{
+			Use:   "stats",
+			Short: "Print conntrack statistics",
+		},
+		protos: make(map[int]int),
+	}
+	cmd.Command.Run = cmd.Run
+
+	return cmd.Command
+}
+
+func (cmd *conntrackStatsCmd) Run(c *cobra.Command, _ []string) {
+	var ctMap maps.Map
+
+	cmd.ipv6 = ipv6 != nil && *ipv6
+
+	if cmd.ipv6 {
+		ctMap = conntrack.MapV6()
+	} else {
+		ctMap = conntrack.Map()
+	}
+
+	if err := ctMap.Open(); err != nil {
+		log.WithError(err).Fatal("Failed to access ConntrackMap")
+	}
+
+	keyFromBytes := conntrack.KeyFromBytes
+	valFromBytes := conntrack.ValueFromBytes
+	if cmd.ipv6 {
+		keyFromBytes = conntrack.KeyV6FromBytes
+		valFromBytes = conntrack.ValueV6FromBytes
+	}
+
+	err := ctMap.Iter(func(k, v []byte) maps.IteratorAction {
+		ctKey := keyFromBytes(k)
+		ctVal := valFromBytes(v)
+		d := ctVal.Data()
+
+		if ctVal.Type() == conntrack.TypeNATForward {
+			cmd.nat++
+			return maps.IterNone
+		}
+
+		if ctKey.Proto() == 6 {
+			if (ctVal.IsForwardDSR() && d.FINsSeenDSR()) || d.FINsSeen() {
+				cmd.closed++
+			} else if d.RSTSeen() {
+				cmd.reset++
+			} else if d.Established() {
+				cmd.established++
+			} else {
+				cmd.synSent++
+			}
+		}
+
+		cmd.total++
+		cmd.protos[int(ctKey.Proto())]++
+
+		return maps.IterNone
+	})
+
+	cmd.Printf("Total connections: %d\n", cmd.total)
+	cmd.Printf("Total entries: %d\n", cmd.total+cmd.nat)
+	cmd.Printf("NAT connections: %d\n\n", cmd.nat)
+
+	cmd.Printf("TCP : %d\n", cmd.protos[6])
+	cmd.Printf("UDP : %d\n", cmd.protos[17])
+	cmd.Printf("Others : %d\n\n", cmd.total-cmd.protos[6]-cmd.protos[17])
+
+	cmd.Printf("TCP Established: %d\n", cmd.established)
+	cmd.Printf("TCP Closed: %d\n", cmd.closed)
+	cmd.Printf("TCP Reset: %d\n", cmd.reset)
+	cmd.Printf("TCP Syn-sent: %d\n", cmd.synSent)
+
+	if err != nil {
+		log.WithError(err).Fatal("Failed to iterate over conntrack entries")
+	}
+}

felix/fv/bpf_counters_test.go

@@ -265,16 +265,21 @@ func checkDroppedByPolicyCounters(g Gomega, felix *infrastructure.Felix, ifName
 		xCounter           string
 	)
 
+	dropped := false
+
 	for _, line := range strOut {
 		fields := strings.FieldsFunc(line, f)
 		if len(fields) < 5 {
 			continue
 		}
 
+		if strings.TrimSpace(strings.ToLower(fields[0])) == "dropped" {
+			dropped = true
+		}
+
 		// "Dropped by policy" is the description of DroppedByPolicy counter
 		// defined in felix/bpf/counters/counters.go.
-		if strings.TrimSpace(strings.ToLower(fields[0])) == "dropped" &&
-			strings.TrimSpace(strings.ToLower(fields[1])) == "by policy" {
+		if dropped && strings.TrimSpace(strings.ToLower(fields[1])) == "by policy" {
 			iCounter, _ = strconv.Atoi(strings.TrimSpace(strings.ToLower(fields[2])))
 			eCounter, _ = strconv.Atoi(strings.TrimSpace(strings.ToLower(fields[3])))
 			xCounter = strings.TrimSpace(strings.ToLower(fields[4]))

node/Makefile

@@ -265,12 +265,12 @@ sub-image-fips-%:
 
 image $(NODE_IMAGE): register $(NODE_CONTAINER_MARKER)
 $(NODE_CONTAINER_CREATED): $(REMOTE_DEPS) ./Dockerfile.$(ARCH) $(NODE_CONTAINER_BINARY) $(INCLUDED_SOURCE) $(NODE_CONTAINER_FILES) $(TOOLS_MOUNTNS_BINARY)
-	$(DOCKER_BUILD) --build-arg BIN_DIR=$(NODE_CONTAINER_BIN_DIR) --build-arg BIRD_IMAGE=$(BIRD_IMAGE) --build-arg GIT_VERSION=$(GIT_VERSION) -t $(NODE_IMAGE):latest-$(ARCH) -f ./Dockerfile.$(ARCH) .
+	$(DOCKER_BUILD) --network=host --build-arg BIN_DIR=$(NODE_CONTAINER_BIN_DIR) --build-arg BIRD_IMAGE=$(BIRD_IMAGE) --build-arg GIT_VERSION=$(GIT_VERSION) -t $(NODE_IMAGE):latest-$(ARCH) -f ./Dockerfile.$(ARCH) .
 	$(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest
 	touch $@
 
 $(NODE_CONTAINER_FIPS_CREATED): $(REMOTE_DEPS) ./Dockerfile.$(ARCH) $(NODE_CONTAINER_BINARY) $(INCLUDED_SOURCE) $(NODE_CONTAINER_FILES) $(TOOLS_MOUNTNS_BINARY)
-	$(DOCKER_BUILD) --build-arg BIN_DIR=$(NODE_CONTAINER_BIN_DIR) --build-arg BIRD_IMAGE=$(BIRD_IMAGE) --build-arg GIT_VERSION=$(GIT_VERSION) -t $(NODE_IMAGE):latest-fips-$(ARCH) -f ./Dockerfile.$(ARCH) .
+	$(DOCKER_BUILD) --network=host --build-arg BIN_DIR=$(NODE_CONTAINER_BIN_DIR) --build-arg BIRD_IMAGE=$(BIRD_IMAGE) --build-arg GIT_VERSION=$(GIT_VERSION) -t $(NODE_IMAGE):latest-fips-$(ARCH) -f ./Dockerfile.$(ARCH) .
 	$(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips
 	touch $@