From 3950bad9d7a3e14d252f66e364980e3fd29b8708 Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Thu, 8 Jun 2023 11:57:43 +0800 Subject: [PATCH] wip: [skip ci] gateway status transition Signed-off-by: Lin Yang --- .../gateway/v1beta1/gateway_controller.go | 156 +++++++++++++++++- .../v1beta1/gatewayclass_controller.go | 25 ++- .../gateway/v1beta1/httproute_controller.go | 30 +++- docs/.gitkeep | 0 docs/gateway-config-json.md | 4 + pkg/event/handler/handler.go | 9 +- pkg/gateway/cache/cache.go | 118 ++++++++++--- pkg/gateway/cache/endpoints_processor.go | 5 +- pkg/gateway/cache/endpointslices_processor.go | 5 +- pkg/gateway/cache/gatewayclasses_processor.go | 13 +- pkg/gateway/cache/gateways_processor.go | 9 +- pkg/gateway/cache/grpcroutes_processor.go | 5 +- pkg/gateway/cache/httproutes_processor.go | 5 +- pkg/gateway/cache/service_processor.go | 5 +- pkg/gateway/cache/serviceimports_processor.go | 5 +- pkg/gateway/cache/tcproutes_processor.go | 5 +- pkg/gateway/cache/tlsroutes_processor.go | 5 +- pkg/gateway/route/types.go | 20 ++- pkg/gateway/utils/utils.go | 20 +++ pkg/webhooks/gateway/gateway_webhook.go | 87 ++++++++-- 20 files changed, 452 insertions(+), 79 deletions(-) delete mode 100644 docs/.gitkeep create mode 100644 docs/gateway-config-json.md diff --git a/controllers/gateway/v1beta1/gateway_controller.go b/controllers/gateway/v1beta1/gateway_controller.go index 9bf3dc32..d57fab0a 100644 --- a/controllers/gateway/v1beta1/gateway_controller.go +++ b/controllers/gateway/v1beta1/gateway_controller.go @@ -195,7 +195,13 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } - // 5. after all status of gateways in the namespace have been updated successfully + // 5. update listener status of this gateway no matter it's accepted or not + result, err := r.updateListenerStatus(ctx, gateway) + if err != nil { + return result, err + } + + // 6. after all status of gateways in the namespace have been updated successfully // list all gateways in the namespace and deploy/redeploy the effective one activeGateway, result, err := r.findActiveGateway(ctx, gateway) if err != nil { @@ -211,7 +217,7 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct activeGateways[gateway.Namespace] = activeGateway } - // 6. update addresses of Gateway status if any IP is allocated + // 7. update addresses of Gateway status if any IP is allocated if activeGateway != nil { lbSvc := &corev1.Service{} key := client.ObjectKey{ @@ -245,6 +251,152 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } +type gatewayStatus struct { + FullName types.NamespacedName + Conditions map[gwv1beta1.GatewayConditionType]metav1.Condition + ListenerStatus map[gwv1beta1.PortNumber]*gwv1beta1.ListenerStatus +} + +func (r *gatewayReconciler) updateListenerStatus(ctx context.Context, gateway *gwv1beta1.Gateway) (ctrl.Result, error) { + if len(gateway.Annotations) == 0 { + gateway.Annotations = make(map[string]string) + } + + oldHash := gateway.Annotations["gateway.flomesh.io/listeners-hash"] + hash := util.SimpleHash(gateway.Spec.Listeners) + + if oldHash != hash { + gateway.Annotations["gateway.flomesh.io/listeners-hash"] = hash + if err := r.fctx.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + + existingListenerStatus := make(map[gwv1beta1.SectionName]gwv1beta1.ListenerStatus) + for _, status := range gateway.Status.Listeners { + existingListenerStatus[status.Name] = status + } + + gateway.Status.Listeners = nil + listenerStatus := make([]gwv1beta1.ListenerStatus, 0) + for _, listener := range gateway.Spec.Listeners { + status, ok := existingListenerStatus[listener.Name] + if ok { + // update existing status + programmedConditionExists := false + acceptedConditionExists := false + for _, cond := range status.Conditions { + if cond.Type == string(gwv1beta1.ListenerConditionProgrammed) { + programmedConditionExists = true + } + if cond.Type == string(gwv1beta1.ListenerConditionAccepted) { + acceptedConditionExists = true + } + } + + if !programmedConditionExists { + metautil.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: string(gwv1beta1.ListenerConditionProgrammed), + Status: metav1.ConditionFalse, + ObservedGeneration: gateway.Generation, + LastTransitionTime: metav1.Time{Time: time.Now()}, + Reason: string(gwv1beta1.ListenerReasonInvalid), + Message: fmt.Sprintf("Invalid listener %q[:%d]", listener.Name, listener.Port), + }) + } + + if !acceptedConditionExists { + metautil.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: string(gwv1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.Generation, + LastTransitionTime: metav1.Time{Time: time.Now()}, + Reason: string(gwv1beta1.ListenerReasonAccepted), + Message: fmt.Sprintf("listener %q[:%d] is accepted.", listener.Name, listener.Port), + }) + } + } else { + // create new status + status = gwv1beta1.ListenerStatus{ + Name: listener.Name, + SupportedKinds: supportedKindsByProtocol(listener.Protocol), + Conditions: []metav1.Condition{ + { + Type: string(gwv1beta1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.Generation, + LastTransitionTime: metav1.Time{Time: time.Now()}, + Reason: string(gwv1beta1.ListenerReasonAccepted), + Message: fmt.Sprintf("listener %q[:%d] is accepted.", listener.Name, listener.Port), + }, + { + Type: string(gwv1beta1.ListenerConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.Generation, + LastTransitionTime: metav1.Time{Time: time.Now()}, + Reason: string(gwv1beta1.ListenerReasonProgrammed), + Message: fmt.Sprintf("Valid listener %q[:%d]", listener.Name, listener.Port), + }, + }, + } + } + + listenerStatus = append(listenerStatus, status) + } + + if len(listenerStatus) > 0 { + gateway.Status.Listeners = listenerStatus + if err := r.fctx.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + } + } + + return ctrl.Result{}, nil +} + +func supportedKindsByProtocol(protocol gwv1beta1.ProtocolType) []gwv1beta1.RouteGroupKind { + switch protocol { + case gwv1beta1.HTTPProtocolType, gwv1beta1.HTTPSProtocolType: + return []gwv1beta1.RouteGroupKind{ + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "HTTPRoute", + }, + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "GRPCRoute", + }, + } + case gwv1beta1.TLSProtocolType: + return []gwv1beta1.RouteGroupKind{ + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "TLSRoute", + }, + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "TCPRoute", + }, + } + case gwv1beta1.TCPProtocolType: + return []gwv1beta1.RouteGroupKind{ + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "TCPRoute", + }, + } + case gwv1beta1.UDPProtocolType: + return []gwv1beta1.RouteGroupKind{ + { + Group: utils.GroupPointer("gateway.networking.k8s.io"), + Kind: "UDPRoute", + }, + } + } + + return nil +} + func gatewayAddresses(activeGateway *gwv1beta1.Gateway, lbSvc *corev1.Service) []gwv1beta1.GatewayAddress { existingIPs := gatewayIPs(activeGateway) expectedIPs := lbIPs(lbSvc) diff --git a/controllers/gateway/v1beta1/gatewayclass_controller.go b/controllers/gateway/v1beta1/gatewayclass_controller.go index 5d144f09..d8fc587d 100644 --- a/controllers/gateway/v1beta1/gatewayclass_controller.go +++ b/controllers/gateway/v1beta1/gatewayclass_controller.go @@ -75,7 +75,8 @@ func (r *gatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request ObjectMeta: metav1.ObjectMeta{ Namespace: req.Namespace, Name: req.Name, - }}) + }}, + ) return ctrl.Result{}, nil } // Error reading the object - requeue the request. @@ -88,6 +89,13 @@ func (r *gatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } + // Accept all GatewayClasses those ControllerName is flomesh.io/gateway-controller + r.setAcceptedStatus(gatewayClass) + result, err := r.updateStatus(ctx, gatewayClass, gwv1beta1.GatewayClassConditionStatusAccepted) + if err != nil { + return result, err + } + gatewayClassList, err := r.fctx.K8sAPI.GatewayAPIClient.GatewayV1beta1(). GatewayClasses(). List(ctx, metav1.ListOptions{}) @@ -96,13 +104,6 @@ func (r *gatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - // Accept all GatewayClasses those ControllerName is flomesh.io/gateway-controller - r.setAcceptedStatus(gatewayClass) - result, err := r.updateStatus(ctx, gatewayClass, gwv1beta1.GatewayClassConditionStatusAccepted) - if err != nil { - return result, err - } - // If there's multiple GatewayClasses, the oldest is set to active and the rest are set to inactive for _, class := range r.setActiveStatus(gatewayClassList) { result, err := r.updateStatus(ctx, class, gateway.GatewayClassConditionStatusActive) @@ -111,6 +112,14 @@ func (r *gatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request } } + // As status of all GatewayClasses have been updated, just send the event + r.fctx.EventHandler.OnAdd(&gwv1beta1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: req.Name, + }}, + ) + return ctrl.Result{}, nil } diff --git a/controllers/gateway/v1beta1/httproute_controller.go b/controllers/gateway/v1beta1/httproute_controller.go index 1b91b0d6..d418d9f7 100644 --- a/controllers/gateway/v1beta1/httproute_controller.go +++ b/controllers/gateway/v1beta1/httproute_controller.go @@ -33,8 +33,12 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -84,9 +88,19 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if len(activeGateways) > 0 { //activeGateway.Spec.GatewayClassName for _, gw := range activeGateways { + httpRoute.Status.Parents = nil + + for _, parentRef := range httpRoute.Spec.ParentRefs { + if utils.IsRefToGateway(parentRef, utils.ObjectKey(gw)) { + + for _, listener := range gw.Spec.Listeners { + + } + } + } //gw.Spec.Listeners[0].Hostname //httpRoute.Spec.Hostnames[] - httpRoute.Status.Parents = nil + for _, ref := range httpRoute.Spec.ParentRefs { //if ref.Group == @@ -107,5 +121,19 @@ func (r *httpRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( func (r *httpRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&gwv1beta1.HTTPRoute{}). + //Watches( + //&source.Kind{Type: &gwv1beta1.Gateway{}}, + //handler.EnqueueRequestsFromMapFunc(r.gatewayToRoutes), + //). Complete(r) } + +//func (r *httpRouteReconciler) gatewayToRoutes(obj client.Object) []reconcile.Request { +// gateway, ok := obj.(*gwv1beta1.Gateway) +// if !ok { +// klog.Errorf("unexpected object type: %T", obj) +// return nil +// } +// +// +//} diff --git a/docs/.gitkeep b/docs/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/gateway-config-json.md b/docs/gateway-config-json.md new file mode 100644 index 00000000..37a84c83 --- /dev/null +++ b/docs/gateway-config-json.md @@ -0,0 +1,4 @@ +| | Enum | | | +|------------------------------|----------------------------|---|---| +| .Listeners[].Protocol | HTTP, HTTPS, TLS, TCP, UDP | | | +| .Listeners[].TLS.TLSModeType | Terminate, Passthrough | | | diff --git a/pkg/event/handler/handler.go b/pkg/event/handler/handler.go index a2bab45d..a0d2ae6d 100644 --- a/pkg/event/handler/handler.go +++ b/pkg/event/handler/handler.go @@ -30,6 +30,7 @@ import ( gw "github.com/flomesh-io/fsm-classic/pkg/gateway" "github.com/google/go-cmp/cmp" "k8s.io/kubernetes/pkg/util/async" + "sigs.k8s.io/controller-runtime/pkg/client" "time" ) @@ -64,25 +65,25 @@ func NewEventHandler(config EventHandlerConfig) EventHandler { return handler } -func (e *FsmEventHandler) OnAdd(obj interface{}) { +func (e *FsmEventHandler) OnAdd(obj client.Object) { if e.onChange(nil, obj) { e.Sync() } } -func (e *FsmEventHandler) OnUpdate(oldObj, newObj interface{}) { +func (e *FsmEventHandler) OnUpdate(oldObj, newObj client.Object) { if e.onChange(oldObj, newObj) { e.Sync() } } -func (e *FsmEventHandler) OnDelete(obj interface{}) { +func (e *FsmEventHandler) OnDelete(obj client.Object) { if e.onChange(obj, nil) { e.Sync() } } -func (e *FsmEventHandler) onChange(oldObj, newObj interface{}) bool { +func (e *FsmEventHandler) onChange(oldObj, newObj client.Object) bool { if newObj == nil { return e.cache.Delete(oldObj) } else { diff --git a/pkg/gateway/cache/cache.go b/pkg/gateway/cache/cache.go index b51305d1..b9a4f407 100644 --- a/pkg/gateway/cache/cache.go +++ b/pkg/gateway/cache/cache.go @@ -265,15 +265,6 @@ func (c *GatewayCache) isEffectiveRoute(parentRefs []gwv1beta1.ParentReference) return false } -func objectKey(obj client.Object) client.ObjectKey { - ns := obj.GetNamespace() - if ns == "" { - ns = metav1.NamespaceDefault - } - - return client.ObjectKey{Namespace: ns, Name: obj.GetName()} -} - func (c *GatewayCache) BuildConfigs() { configs := make(map[string]*route.ConfigSpec) ctx := context.TODO() @@ -285,9 +276,11 @@ func (c *GatewayCache) BuildConfigs() { continue } + listeners := c.listeners(gw) + config := &route.ConfigSpec{ - Listeners: c.listeners(gw), - RouteRules: c.routeRules(gw), + Listeners: listeners, + RouteRules: c.routeRules(gw, listeners), Chains: chains(), } configs[ns] = config @@ -299,8 +292,8 @@ func (c *GatewayCache) listeners(gw *gwv1beta1.Gateway) []route.Listener { for _, l := range gw.Spec.Listeners { listener := route.Listener{ - Protocol: string(l.Protocol), - Port: int32(l.Port), + Protocol: l.Protocol, + Port: l.Port, } switch l.Protocol { @@ -310,7 +303,7 @@ func (c *GatewayCache) listeners(gw *gwv1beta1.Gateway) []route.Listener { switch *l.TLS.Mode { case gwv1beta1.TLSModeTerminate: listener.TLS = &route.TLS{ - TLSModeType: string(gwv1beta1.TLSModeTerminate), + TLSModeType: gwv1beta1.TLSModeTerminate, MTLS: false, // FIXME: source of mTLS Certificates: c.certificates(gw, l), } @@ -324,13 +317,13 @@ func (c *GatewayCache) listeners(gw *gwv1beta1.Gateway) []route.Listener { switch *l.TLS.Mode { case gwv1beta1.TLSModeTerminate: listener.TLS = &route.TLS{ - TLSModeType: string(gwv1beta1.TLSModeTerminate), + TLSModeType: gwv1beta1.TLSModeTerminate, MTLS: false, // FIXME: source of mTLS Certificates: c.certificates(gw, l), } case gwv1beta1.TLSModePassthrough: listener.TLS = &route.TLS{ - TLSModeType: string(gwv1beta1.TLSModePassthrough), + TLSModeType: gwv1beta1.TLSModePassthrough, MTLS: false, // FIXME: source of mTLS } } @@ -354,9 +347,9 @@ func (c *GatewayCache) certificates(gw *gwv1beta1.Gateway, l gwv1beta1.Listener) continue } certs = append(certs, route.Certificate{ - CertChain: string(secret.Data["tls.crt"]), - PrivateKey: string(secret.Data["tls.key"]), - IssuingCA: string(secret.Data["ca.crt"]), + CertChain: string(secret.Data[corev1.TLSCertKey]), + PrivateKey: string(secret.Data[corev1.TLSPrivateKeyKey]), + IssuingCA: string(secret.Data[corev1.ServiceAccountRootCAKey]), }) } } @@ -379,7 +372,7 @@ func secretKey(gw *gwv1beta1.Gateway, secretRef gwv1beta1.SecretObjectReference) } } -func (c *GatewayCache) routeRules(gw *gwv1beta1.Gateway) map[int32]route.RouteRule { +func (c *GatewayCache) routeRules(gw *gwv1beta1.Gateway, listeners []route.Listener) map[int32]route.RouteRule { rules := make(map[int32]route.RouteRule) for key := range c.httproutes { @@ -390,9 +383,90 @@ func (c *GatewayCache) routeRules(gw *gwv1beta1.Gateway) map[int32]route.RouteRu } for _, ref := range httpRoute.Spec.ParentRefs { - if string(*ref.Kind) == gw.Kind && string(*ref.Group) == gw.GroupVersionKind().Group { + if utils.IsRefToGateway(ref, utils.ObjectKey(gw)) { + for _, listener := range listeners { + switch listener.Protocol { + case gwv1beta1.HTTPProtocolType, gwv1beta1.HTTPSProtocolType: + + port := int32(listener.Port) + _, ok := rules[port] + if !ok { + rules[port] = route.L7RouteRule{} + } + + rule := rules[port] + switch r := rule.(type) { + case route.L7RouteRule: + r[httpRoute.Spec.Hostnames] = route.HTTPRouteRuleSpec{} + } + //rules[port]["xxx"] = route.HTTPRouteRuleSpec{} + + } + } + } + } + } + + for key := range c.grpcroutes { + grpcRoute := &gwv1alpha2.GRPCRoute{} + if err := c.client.Get(context.TODO(), key, grpcRoute); err != nil { + klog.Errorf("Failed to get GRPCRoute %s: %s", key, err) + continue + } + + for _, ref := range grpcRoute.Spec.ParentRefs { + if utils.IsRefToGateway(ref, utils.ObjectKey(gw)) { + for _, listener := range listeners { + switch listener.Protocol { + case gwv1beta1.HTTPSProtocolType: + + } + } + } + } + } + + for key := range c.tlsroutes { + tlsRoute := &gwv1alpha2.TLSRoute{} + if err := c.client.Get(context.TODO(), key, tlsRoute); err != nil { + klog.Errorf("Failed to get TLSRoute %s: %s", key, err) + continue + } + + for _, ref := range tlsRoute.Spec.ParentRefs { + if utils.IsRefToGateway(ref, utils.ObjectKey(gw)) { + for _, listener := range listeners { + switch listener.Protocol { + case gwv1beta1.TLSProtocolType: + if listener.TLS != nil && listener.TLS.TLSModeType == gwv1beta1.TLSModePassthrough { - //if *ref.Port == gw.Spec.Listeners {} + } + + } + } + } + } + } + + for key := range c.tcproutes { + tcpRoute := &gwv1alpha2.TCPRoute{} + if err := c.client.Get(context.TODO(), key, tcpRoute); err != nil { + klog.Errorf("Failed to get TCPRoute %s: %s", key, err) + continue + } + + for _, ref := range tcpRoute.Spec.ParentRefs { + if utils.IsRefToGateway(ref, utils.ObjectKey(gw)) { + for _, listener := range listeners { + switch listener.Protocol { + case gwv1beta1.TLSProtocolType: + if listener.TLS != nil && listener.TLS.TLSModeType == gwv1beta1.TLSModeTerminate { + + } + case gwv1beta1.TCPProtocolType: + + } + } } } } diff --git a/pkg/gateway/cache/endpoints_processor.go b/pkg/gateway/cache/endpoints_processor.go index 94360f6f..8cc374e8 100644 --- a/pkg/gateway/cache/endpoints_processor.go +++ b/pkg/gateway/cache/endpoints_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" ) @@ -39,7 +40,7 @@ func (p *EndpointsProcessor) Insert(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(ep) + key := utils.ObjectKey(ep) cache.endpoints[key] = true return cache.isRoutableService(key) @@ -52,7 +53,7 @@ func (p *EndpointsProcessor) Delete(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(ep) + key := utils.ObjectKey(ep) _, found := cache.endpoints[key] delete(cache.endpoints, key) diff --git a/pkg/gateway/cache/endpointslices_processor.go b/pkg/gateway/cache/endpointslices_processor.go index afe68470..65b4d8a5 100644 --- a/pkg/gateway/cache/endpointslices_processor.go +++ b/pkg/gateway/cache/endpointslices_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -51,7 +52,7 @@ func (p *EndpointSlicesProcessor) Insert(obj interface{}, cache *GatewayCache) b if !found { cache.endpointslices[svcKey] = make(map[client.ObjectKey]bool) } - cache.endpointslices[svcKey][objectKey(eps)] = true + cache.endpointslices[svcKey][utils.ObjectKey(eps)] = true return cache.isRoutableService(svcKey) } @@ -74,7 +75,7 @@ func (p *EndpointSlicesProcessor) Delete(obj interface{}, cache *GatewayCache) b return false } - sliceKey := objectKey(eps) + sliceKey := utils.ObjectKey(eps) _, found = slices[sliceKey] delete(cache.endpointslices[svcKey], sliceKey) diff --git a/pkg/gateway/cache/gatewayclasses_processor.go b/pkg/gateway/cache/gatewayclasses_processor.go index 0b5b72fb..5beab3be 100644 --- a/pkg/gateway/cache/gatewayclasses_processor.go +++ b/pkg/gateway/cache/gatewayclasses_processor.go @@ -25,6 +25,7 @@ package cache import ( + "context" "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -40,6 +41,12 @@ func (p *GatewayClassesProcessor) Insert(obj interface{}, cache *GatewayCache) b return false } + key := utils.ObjectKey(class) + if err := cache.client.Get(context.TODO(), key, class); err != nil { + klog.Errorf("Failed to get GatewayClass %s: %s", key, err) + return false + } + if utils.IsEffectiveGatewayClass(class) { cache.gatewayclass = class return true @@ -55,7 +62,11 @@ func (p *GatewayClassesProcessor) Delete(obj interface{}, cache *GatewayCache) b return false } - if utils.IsEffectiveGatewayClass(class) { + if cache.gatewayclass == nil { + return false + } + + if class.Name == cache.gatewayclass.Name { cache.gatewayclass = nil return true } diff --git a/pkg/gateway/cache/gateways_processor.go b/pkg/gateway/cache/gateways_processor.go index 1492ffe4..ce7ef79d 100644 --- a/pkg/gateway/cache/gateways_processor.go +++ b/pkg/gateway/cache/gateways_processor.go @@ -25,6 +25,7 @@ package cache import ( + "context" "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -40,8 +41,14 @@ func (p *GatewaysProcessor) Insert(obj interface{}, cache *GatewayCache) bool { return false } + key := utils.ObjectKey(gw) + if err := cache.client.Get(context.TODO(), key, gw); err != nil { + klog.Errorf("Failed to get Gateway %s: %s", key, err) + return false + } + if utils.IsAcceptedGateway(gw) { - cache.gateways[gw.Namespace] = objectKey(gw) + cache.gateways[gw.Namespace] = utils.ObjectKey(gw) return true } diff --git a/pkg/gateway/cache/grpcroutes_processor.go b/pkg/gateway/cache/grpcroutes_processor.go index 97dfcc47..77c3b4df 100644 --- a/pkg/gateway/cache/grpcroutes_processor.go +++ b/pkg/gateway/cache/grpcroutes_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -39,7 +40,7 @@ func (p *GRPCRoutesProcessor) Insert(obj interface{}, cache *GatewayCache) bool return false } - cache.grpcroutes[objectKey(route)] = true + cache.grpcroutes[utils.ObjectKey(route)] = true return cache.isEffectiveRoute(route.Spec.ParentRefs) } @@ -51,7 +52,7 @@ func (p *GRPCRoutesProcessor) Delete(obj interface{}, cache *GatewayCache) bool return false } - key := objectKey(route) + key := utils.ObjectKey(route) _, found := cache.grpcroutes[key] delete(cache.grpcroutes, key) diff --git a/pkg/gateway/cache/httproutes_processor.go b/pkg/gateway/cache/httproutes_processor.go index bd332ef4..94ab7277 100644 --- a/pkg/gateway/cache/httproutes_processor.go +++ b/pkg/gateway/cache/httproutes_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -39,7 +40,7 @@ func (p *HTTPRoutesProcessor) Insert(obj interface{}, cache *GatewayCache) bool return false } - cache.httproutes[objectKey(route)] = true + cache.httproutes[utils.ObjectKey(route)] = true return cache.isEffectiveRoute(route.Spec.ParentRefs) } @@ -51,7 +52,7 @@ func (p *HTTPRoutesProcessor) Delete(obj interface{}, cache *GatewayCache) bool return false } - key := objectKey(route) + key := utils.ObjectKey(route) _, found := cache.httproutes[key] delete(cache.httproutes, key) diff --git a/pkg/gateway/cache/service_processor.go b/pkg/gateway/cache/service_processor.go index 572aec5f..61e70288 100644 --- a/pkg/gateway/cache/service_processor.go +++ b/pkg/gateway/cache/service_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" ) @@ -39,7 +40,7 @@ func (p *ServicesProcessor) Insert(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(svc) + key := utils.ObjectKey(svc) cache.services[key] = true return cache.isRoutableService(key) @@ -52,7 +53,7 @@ func (p *ServicesProcessor) Delete(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(svc) + key := utils.ObjectKey(svc) _, found := cache.services[key] delete(cache.services, key) diff --git a/pkg/gateway/cache/serviceimports_processor.go b/pkg/gateway/cache/serviceimports_processor.go index 57ff683c..790f7ff3 100644 --- a/pkg/gateway/cache/serviceimports_processor.go +++ b/pkg/gateway/cache/serviceimports_processor.go @@ -26,6 +26,7 @@ package cache import ( svcimpv1alpha1 "github.com/flomesh-io/fsm-classic/apis/serviceimport/v1alpha1" + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" ) @@ -40,7 +41,7 @@ func (p *ServiceImportsProcessor) Insert(obj interface{}, cache *GatewayCache) b return false } - key := objectKey(svcimp) + key := utils.ObjectKey(svcimp) cache.serviceimports[key] = true return cache.isRoutableService(key) @@ -54,7 +55,7 @@ func (p *ServiceImportsProcessor) Delete(obj interface{}, cache *GatewayCache) b return false } - key := objectKey(svcimp) + key := utils.ObjectKey(svcimp) _, found := cache.serviceimports[key] delete(cache.serviceimports, key) diff --git a/pkg/gateway/cache/tcproutes_processor.go b/pkg/gateway/cache/tcproutes_processor.go index 683b0436..4cf18ebb 100644 --- a/pkg/gateway/cache/tcproutes_processor.go +++ b/pkg/gateway/cache/tcproutes_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -39,7 +40,7 @@ func (p *TCPRoutesProcessor) Insert(obj interface{}, cache *GatewayCache) bool { return false } - cache.tcproutes[objectKey(route)] = true + cache.tcproutes[utils.ObjectKey(route)] = true return cache.isEffectiveRoute(route.Spec.ParentRefs) } @@ -51,7 +52,7 @@ func (p *TCPRoutesProcessor) Delete(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(route) + key := utils.ObjectKey(route) _, found := cache.tcproutes[key] delete(cache.tcproutes, key) diff --git a/pkg/gateway/cache/tlsroutes_processor.go b/pkg/gateway/cache/tlsroutes_processor.go index f4e7890f..fb7bbb01 100644 --- a/pkg/gateway/cache/tlsroutes_processor.go +++ b/pkg/gateway/cache/tlsroutes_processor.go @@ -25,6 +25,7 @@ package cache import ( + "github.com/flomesh-io/fsm-classic/pkg/gateway/utils" "k8s.io/klog/v2" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -39,7 +40,7 @@ func (p *TLSRoutesProcessor) Insert(obj interface{}, cache *GatewayCache) bool { return false } - cache.tlsroutes[objectKey(route)] = true + cache.tlsroutes[utils.ObjectKey(route)] = true return cache.isEffectiveRoute(route.Spec.ParentRefs) } @@ -51,7 +52,7 @@ func (p *TLSRoutesProcessor) Delete(obj interface{}, cache *GatewayCache) bool { return false } - key := objectKey(route) + key := utils.ObjectKey(route) _, found := cache.tlsroutes[key] delete(cache.tlsroutes, key) diff --git a/pkg/gateway/route/types.go b/pkg/gateway/route/types.go index 788e3bf3..b1d2104a 100644 --- a/pkg/gateway/route/types.go +++ b/pkg/gateway/route/types.go @@ -23,6 +23,14 @@ func fmtPortName(in int32) string { return fmt.Sprintf(":%d", in) } +type MatchType string + +const ( + MatchTypeExact MatchType = "Exact" + MatchTypePrefix MatchType = "Prefix" + MatchTypeRegex MatchType = "Regex" +) + type ConfigSpec struct { Defaults Defaults `json:"Defaults"` Listeners []Listener `json:"Listeners"` @@ -37,15 +45,15 @@ type Defaults struct { } type Listener struct { - Protocol string `json:"Protocol"` - Port int32 `json:"Port"` - TLS *TLS `json:"TLS,omitempty"` + Protocol gwv1beta1.ProtocolType `json:"Protocol"` + Port gwv1beta1.PortNumber `json:"Port"` + TLS *TLS `json:"TLS,omitempty"` } type TLS struct { - TLSModeType string `json:"TLSModeType"` - MTLS bool `json:"mTLS"` - Certificates []Certificate `json:"Certificates,omitempty"` + TLSModeType gwv1beta1.TLSModeType `json:"TLSModeType"` + MTLS bool `json:"mTLS"` + Certificates []Certificate `json:"Certificates,omitempty"` } type Certificate struct { diff --git a/pkg/gateway/utils/utils.go b/pkg/gateway/utils/utils.go index 7b3a445d..870efe05 100644 --- a/pkg/gateway/utils/utils.go +++ b/pkg/gateway/utils/utils.go @@ -27,6 +27,7 @@ package utils import ( "github.com/flomesh-io/fsm-classic/apis/gateway" metautil "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -47,6 +48,10 @@ func IsAcceptedGateway(gateway *gwv1beta1.Gateway) bool { return metautil.IsStatusConditionTrue(gateway.Status.Conditions, string(gwv1beta1.GatewayConditionAccepted)) } +func IsActiveGateway(gateway *gwv1beta1.Gateway) bool { + return IsAcceptedGateway(gateway) +} + func IsRefToGateway(parentRef gwv1beta1.ParentReference, gateway client.ObjectKey) bool { if parentRef.Group != nil && string(*parentRef.Group) != gwv1beta1.GroupName { return false @@ -62,3 +67,18 @@ func IsRefToGateway(parentRef gwv1beta1.ParentReference, gateway client.ObjectKe return string(parentRef.Name) == gateway.Name } + +func ObjectKey(obj client.Object) client.ObjectKey { + ns := obj.GetNamespace() + if ns == "" { + ns = metav1.NamespaceDefault + } + + return client.ObjectKey{Namespace: ns, Name: obj.GetName()} +} + +func GroupPointer(group string) *gwv1beta1.Group { + result := gwv1beta1.Group(group) + + return &result +} diff --git a/pkg/webhooks/gateway/gateway_webhook.go b/pkg/webhooks/gateway/gateway_webhook.go index 6a3acc2f..c157ac19 100644 --- a/pkg/webhooks/gateway/gateway_webhook.go +++ b/pkg/webhooks/gateway/gateway_webhook.go @@ -34,6 +34,7 @@ import ( "github.com/flomesh-io/fsm-classic/pkg/util" "github.com/flomesh-io/fsm-classic/pkg/webhooks" admissionregv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" @@ -166,30 +167,80 @@ func (w *validator) validateCertificateSecret(gateway *gwv1beta1.Gateway) field. var errs field.ErrorList for i, c := range gateway.Spec.Listeners { - if *c.TLS.Mode == gwv1beta1.TLSModeTerminate { - for j, ref := range c.TLS.CertificateRefs { - if string(*ref.Kind) == "Secret" && string(*ref.Group) == "" { - ns := "" - if ref.Namespace == nil { - ns = gateway.Namespace - } else { - ns = string(*ref.Namespace) - } - name := string(ref.Name) - + switch c.Protocol { + case gwv1beta1.HTTPSProtocolType: + if c.TLS != nil && c.TLS.Mode != nil { + switch *c.TLS.Mode { + case gwv1beta1.TLSModeTerminate: + errs = append(errs, w.validateSecretsExistence(gateway, c, i)...) + case gwv1beta1.TLSModePassthrough: path := field.NewPath("spec"). Child("listeners").Index(i). Child("tls"). - Child("certificateRefs").Index(j) - _, err := w.k8sAPI.Client.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - errs = append(errs, field.NotFound(path, fmt.Sprintf("Failed to get Secret %s/%s: %s", ns, name, err))) + Child("mode") + errs = append(errs, field.Forbidden(path, fmt.Sprintf("TLSModeType %s is not supported when Protocol is %s, please use Protocol %s", gwv1beta1.TLSModePassthrough, gwv1beta1.HTTPSProtocolType, gwv1beta1.TLSProtocolType))) + } + } + case gwv1beta1.TLSProtocolType: + if c.TLS != nil && c.TLS.Mode != nil { + switch *c.TLS.Mode { + case gwv1beta1.TLSModeTerminate: + errs = append(errs, w.validateSecretsExistence(gateway, c, i)...) + case gwv1beta1.TLSModePassthrough: + if len(c.TLS.CertificateRefs) > 0 { + path := field.NewPath("spec"). + Child("listeners").Index(i). + Child("tls"). + Child("certificateRefs") + errs = append(errs, field.Forbidden(path, fmt.Sprintf("No need to provide certificates when Protocol is %s and TLSModeType is %s", gwv1beta1.TLSProtocolType, gwv1beta1.TLSModePassthrough))) } + } + } + } + } + + return errs +} + +func (w *validator) validateSecretsExistence(gateway *gwv1beta1.Gateway, c gwv1beta1.Listener, i int) field.ErrorList { + var errs field.ErrorList + + for j, ref := range c.TLS.CertificateRefs { + if string(*ref.Kind) == "Secret" && string(*ref.Group) == "" { + ns := "" + if ref.Namespace == nil { + ns = gateway.Namespace + } else { + ns = string(*ref.Namespace) + } + name := string(ref.Name) + + path := field.NewPath("spec"). + Child("listeners").Index(i). + Child("tls"). + Child("certificateRefs").Index(j) + secret, err := w.k8sAPI.Client.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + errs = append(errs, field.NotFound(path, fmt.Sprintf("Failed to get Secret %s/%s: %s", ns, name, err))) + continue + } + + v, ok := secret.Data[corev1.TLSCertKey] + if ok { + if string(v) == "" { + errs = append(errs, field.Invalid(path, string(v), fmt.Sprintf("The content of Secret %s/%s by key %s is empty", ns, name, corev1.TLSCertKey))) + } + } else { + errs = append(errs, field.NotFound(path, fmt.Sprintf("Secret %s/%s doesn't have required data by key %s", ns, name, corev1.TLSCertKey))) + } - //if secret.Type != corev1.SecretTypeTLS { - // errs = append(errs, field.Invalid(path, secret.Type, fmt.Sprintf("Invalid type %q of Secret %s/%s, only type 'kubernetes.io/tls' is supported.", secret.Type, ns, name))) - //} + v, ok = secret.Data[corev1.TLSPrivateKeyKey] + if ok { + if string(v) == "" { + errs = append(errs, field.Invalid(path, string(v), fmt.Sprintf("The content of Secret %s/%s by key %s is empty", ns, name, corev1.TLSPrivateKeyKey))) } + } else { + errs = append(errs, field.NotFound(path, fmt.Sprintf("Secret %s/%s doesn't have required data by key %s", ns, name, corev1.TLSPrivateKeyKey))) } } }