From 5aa5a7ae51d743a7571da3c4bdd1a5a8714ee13f Mon Sep 17 00:00:00 2001
From: Kent Dong <ch3cho@qq.com>
Date: Thu, 7 Mar 2024 10:17:59 +0800
Subject: [PATCH] feat: Support configuring routes with Gateway API (#61)

* feat: Support configuring routes with Gateway API
1. Fix the Gateway API CRDs support in the API server.
2. Generate a basic GatewayClass and Gateway resource during preparation.
3. Fix the XDS cache issue causing delayed push of Gateway API configuration updates.

* Remove PILOT_ENABLE_XDS_CACHE=false since the bug has been fixed
---
 compose/.env                              |   2 +-
 compose/scripts/prepare.sh                |  97 +++++++++++++++++++-
 src/apiserver/pkg/apiserver/apiserver.go  |  19 +---
 src/apiserver/pkg/converter/gatewayapi.go | 104 ++++++++++++++++++++++
 4 files changed, 200 insertions(+), 22 deletions(-)

diff --git a/compose/.env b/compose/.env
index bbaf259..f6659ae 100644
--- a/compose/.env
+++ b/compose/.env
@@ -8,7 +8,7 @@ NACOS_PASSWORD=''
 NACOS_DATA_ENC_KEY='0123456789abcdef0123456789abcdef'
 NACOS_SERVER_TAG='v2.2.3'
 HIGRESS_RUNNER_TAG='0.0.3'
-HIGRESS_API_SERVER_TAG='0.0.13'
+HIGRESS_API_SERVER_TAG='0.0.14'
 HIGRESS_CONTROLLER_TAG='1.3.5'
 HIGRESS_PILOT_TAG='1.3.5'
 HIGRESS_GATEWAY_TAG='1.3.5'
diff --git a/compose/scripts/prepare.sh b/compose/scripts/prepare.sh
index a44fa9f..95a206c 100644
--- a/compose/scripts/prepare.sh
+++ b/compose/scripts/prepare.sh
@@ -33,7 +33,12 @@ checkConfigExists() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}"
+  local url;
+  if [ -z "$namespace" ]; then
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}"
+  else
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}"
+  fi
   statusCode=$(curl -s -o /dev/null -w "%{http_code}" "${url}" -k)
   if [ $statusCode -eq 200 ]; then
     return 0
@@ -60,7 +65,12 @@ getConfig() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}"
+  local url;
+  if [ -z "$namespace" ]; then
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}"
+  else
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}/${configName}"
+  fi
   local tmpFile=$(mktemp /tmp/higress-precheck-config.XXXXXXXXX.cfg)
   local statusCode=$(curl -s -o "$tmpFile" -w "%{http_code}" "${url}" -k -H "Accept: application/yaml")
   if [ $statusCode -eq 200 ]; then
@@ -92,7 +102,12 @@ publishConfig() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url="${API_SERVER_BASE_URL}{$uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}"
+  local url;
+  if [ -z "$namespace" ]; then
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}"
+  else
+    url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/namespaces/${namespace}/${configType}"
+  fi
   statusCode="$(curl -s -o /dev/null -w "%{http_code}" "$url" -k -X POST -H "Content-Type: application/yaml" -d "$content")"
   if [ $statusCode -ne 201 ]; then
     echo "  Publishing config ${configType}.${configName} to namespace ${namespace} failed with ${statusCode}"
@@ -218,6 +233,7 @@ data:
       {"authority":"%REQ(:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"}
     configSources:
     - address: xds://controller:15051
+    - address: k8s://
     defaultConfig:
       disableAlpnH2: true
       discoveryAddress: pilot:15012
@@ -310,7 +326,82 @@ EOF
   fi
 }
 
+checkGatewayApi() {
+  echo "Checking Gateway API configurations..."
+
+  checkConfigExists "higress-system" "v1" "services" "higress-gateway"
+  if [ $? -ne 0 ]; then
+    echo "  The Service resource \"higress-gateway\" doesn't exist. Create it now..."
+    read -r -d '' content <<EOF
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    higress: higress-system-higress-gateway
+  name: higress-gateway
+  namespace: higress-system
+spec:
+  ports:
+  - name: http2
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  - name: https
+    port: 443
+    protocol: TCP
+    targetPort: 443
+  selector:
+    higress: higress-system-higress-gateway
+  type: LoadBalancer
+EOF
+    publishConfig "higress-system" "v1" "services" "higress-gateway" "$content"
+  fi
+
+  checkConfigExists "" "gateway.networking.k8s.io/v1beta1" "gatewayclasses" "higress-gateway"
+  if [ $? -ne 0 ]; then
+    echo "  The GatewayClass resource \"higress-gateway\" doesn't exist. Create it now..."
+    read -r -d '' content <<EOF
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: GatewayClass
+metadata:
+  name: higress-gateway
+spec:
+  controllerName: "higress.io/gateway-controller"
+EOF
+    publishConfig "" "gateway.networking.k8s.io/v1beta1" "gatewayclasses" "higress-gateway" "$content"
+  fi
+
+  checkConfigExists "higress-system" "gateway.networking.k8s.io/v1beta1" "gateways" "higress-gateway"
+  if [ $? -ne 0 ]; then
+    echo "  The Gateway resource \"higress-gateway\" doesn't exist. Create it now..."
+    read -r -d '' content <<EOF
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: higress-gateway
+  namespace: higress-system
+spec:
+  gatewayClassName: higress-gateway
+  listeners:
+  - name: http
+    port: 80
+    protocol: HTTP
+    allowedRoutes:
+      namespaces:
+        from: All
+  - name: https
+    port: 443
+    protocol: HTTPS
+    allowedRoutes:
+      namespaces:
+        from: All
+EOF
+    publishConfig "higress-system" "gateway.networking.k8s.io/v1beta1" "gateways" "higress-gateway" "$content"
+  fi
+}
+
 checkStorage
 checkPilot
 checkGateway
 checkConsole
+checkGatewayApi
diff --git a/src/apiserver/pkg/apiserver/apiserver.go b/src/apiserver/pkg/apiserver/apiserver.go
index b0767b2..0cc9212 100644
--- a/src/apiserver/pkg/apiserver/apiserver.go
+++ b/src/apiserver/pkg/apiserver/apiserver.go
@@ -381,24 +381,7 @@ func (c completedConfig) New() (*HigressServer, error) {
 			func() runtime.Object { return &gwapiv1beta1.ReferenceGrantList{} },
 			nil, false)
 		gwapiApiGroupInfo.VersionedResourcesStorageMap[gwapiv1beta1.SchemeGroupVersion.Version] = gwapiv1beta1Storages
-		gwapiv1alpha2Storages := map[string]rest.Storage{}
-		appendStorage(gwapiv1alpha2Storages, storageCreateFunc, gwapiv1alpha2.SchemeGroupVersion, false, "gatewayclass", "gatewayclasses",
-			func() runtime.Object { return &gwapiv1alpha2.GatewayClass{} },
-			func() runtime.Object { return &gwapiv1alpha2.GatewayClassList{} },
-			nil, false)
-		appendStorage(gwapiv1alpha2Storages, storageCreateFunc, gwapiv1alpha2.SchemeGroupVersion, true, "gateway", "gateways",
-			func() runtime.Object { return &gwapiv1alpha2.Gateway{} },
-			func() runtime.Object { return &gwapiv1alpha2.GatewayList{} },
-			nil, false)
-		appendStorage(gwapiv1alpha2Storages, storageCreateFunc, gwapiv1alpha2.SchemeGroupVersion, true, "httproute", "httproutes",
-			func() runtime.Object { return &gwapiv1alpha2.HTTPRoute{} },
-			func() runtime.Object { return &gwapiv1alpha2.HTTPRouteList{} },
-			nil, false)
-		appendStorage(gwapiv1alpha2Storages, storageCreateFunc, gwapiv1alpha2.SchemeGroupVersion, true, "referencegrant", "referencegrants",
-			func() runtime.Object { return &gwapiv1alpha2.ReferenceGrant{} },
-			func() runtime.Object { return &gwapiv1alpha2.ReferenceGrantList{} },
-			nil, false)
-		gwapiApiGroupInfo.VersionedResourcesStorageMap[gwapiv1alpha2.SchemeGroupVersion.Version] = gwapiv1alpha2Storages
+		gwapiApiGroupInfo.VersionedResourcesStorageMap[gwapiv1alpha2.SchemeGroupVersion.Version] = gwapiv1beta1Storages
 		if err := s.GenericAPIServer.InstallAPIGroup(&gwapiApiGroupInfo); err != nil {
 			return nil, err
 		}
diff --git a/src/apiserver/pkg/converter/gatewayapi.go b/src/apiserver/pkg/converter/gatewayapi.go
index cf4e535..593c5e2 100644
--- a/src/apiserver/pkg/converter/gatewayapi.go
+++ b/src/apiserver/pkg/converter/gatewayapi.go
@@ -8,6 +8,32 @@ import (
 )
 
 func registerGatewayApiConverters(scheme *runtime.Scheme) {
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.GatewayClassList{}, &gwapiv1beta1.GatewayClassList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1beta1.GatewayClassList)
+			la := a.(*gwapiv1alpha2.GatewayClassList)
+			lb.Items = make([]gwapiv1beta1.GatewayClass, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1beta1.GroupVersion)
+			return nil
+		})
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1beta1.GatewayClassList{}, &gwapiv1alpha2.GatewayClassList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1alpha2.GatewayClassList)
+			la := a.(*gwapiv1beta1.GatewayClassList)
+			lb.Items = make([]gwapiv1alpha2.GatewayClass, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1alpha2.GroupVersion)
+			return nil
+		})
 	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.GatewayClass{}, &gwapiv1beta1.GatewayClass{},
 		func(a, b interface{}, scope conversion.Scope) error {
 			*(b.(*gwapiv1beta1.GatewayClass)) = gwapiv1beta1.GatewayClass(*a.(*gwapiv1alpha2.GatewayClass))
@@ -21,6 +47,32 @@ func registerGatewayApiConverters(scheme *runtime.Scheme) {
 			return nil
 		})
 
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.GatewayList{}, &gwapiv1beta1.GatewayList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1beta1.GatewayList)
+			la := a.(*gwapiv1alpha2.GatewayList)
+			lb.Items = make([]gwapiv1beta1.Gateway, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1beta1.GroupVersion)
+			return nil
+		})
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1beta1.GatewayList{}, &gwapiv1alpha2.GatewayList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1alpha2.GatewayList)
+			la := a.(*gwapiv1beta1.GatewayList)
+			lb.Items = make([]gwapiv1alpha2.Gateway, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1alpha2.GroupVersion)
+			return nil
+		})
 	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.Gateway{}, &gwapiv1beta1.Gateway{},
 		func(a, b interface{}, scope conversion.Scope) error {
 			*(b.(*gwapiv1beta1.Gateway)) = gwapiv1beta1.Gateway(*a.(*gwapiv1alpha2.Gateway))
@@ -34,6 +86,32 @@ func registerGatewayApiConverters(scheme *runtime.Scheme) {
 			return nil
 		})
 
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.HTTPRouteList{}, &gwapiv1beta1.HTTPRouteList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1beta1.HTTPRouteList)
+			la := a.(*gwapiv1alpha2.HTTPRouteList)
+			lb.Items = make([]gwapiv1beta1.HTTPRoute, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1beta1.GroupVersion)
+			return nil
+		})
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1beta1.HTTPRouteList{}, &gwapiv1alpha2.HTTPRouteList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1alpha2.HTTPRouteList)
+			la := a.(*gwapiv1beta1.HTTPRouteList)
+			lb.Items = make([]gwapiv1alpha2.HTTPRoute, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1alpha2.GroupVersion)
+			return nil
+		})
 	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.HTTPRoute{}, &gwapiv1beta1.HTTPRoute{},
 		func(a, b interface{}, scope conversion.Scope) error {
 			*(b.(*gwapiv1beta1.HTTPRoute)) = gwapiv1beta1.HTTPRoute(*a.(*gwapiv1alpha2.HTTPRoute))
@@ -47,6 +125,32 @@ func registerGatewayApiConverters(scheme *runtime.Scheme) {
 			return nil
 		})
 
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.ReferenceGrantList{}, &gwapiv1beta1.ReferenceGrantList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1beta1.ReferenceGrantList)
+			la := a.(*gwapiv1alpha2.ReferenceGrantList)
+			lb.Items = make([]gwapiv1beta1.ReferenceGrant, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1beta1.GroupVersion)
+			return nil
+		})
+	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1beta1.ReferenceGrantList{}, &gwapiv1alpha2.ReferenceGrantList{},
+		func(a, b interface{}, scope conversion.Scope) error {
+			lb := b.(*gwapiv1alpha2.ReferenceGrantList)
+			la := a.(*gwapiv1beta1.ReferenceGrantList)
+			lb.Items = make([]gwapiv1alpha2.ReferenceGrant, len(la.Items))
+			for i := range la.Items {
+				if err := scheme.Convert(&la.Items[i], &lb.Items[i], scope); err != nil {
+					return err
+				}
+			}
+			setApiVersion(b, gwapiv1alpha2.GroupVersion)
+			return nil
+		})
 	_ = scheme.Converter().RegisterUntypedConversionFunc(&gwapiv1alpha2.ReferenceGrant{}, &gwapiv1beta1.ReferenceGrant{},
 		func(a, b interface{}, scope conversion.Scope) error {
 			*(b.(*gwapiv1beta1.ReferenceGrant)) = gwapiv1beta1.ReferenceGrant(*a.(*gwapiv1alpha2.ReferenceGrant))