Skip to content
This repository has been archived by the owner on May 13, 2024. It is now read-only.

Commit

Permalink
feat: support service-lb (#106)
Browse files Browse the repository at this point in the history
* feat: support service-lb

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: register service-lb reconciler

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: a typo in ServiceReconciler

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: ServiceReconciler and NodeReconciler watching resources

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: grant permission to list, watch, get nodes

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: create/update DaemonSet

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: service-lb image

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: wrong owner reference

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: no enough permission to update/patch event

Signed-off-by: Lin Yang <reaver@flomesh.io>

* refactor: add log fo debugging

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: NodeReconciler

Signed-off-by: Lin Yang <reaver@flomesh.io>

* refactor: add log for debugging

Signed-off-by: Lin Yang <reaver@flomesh.io>

* fix: empty ProxyProfile spec hash

Signed-off-by: Lin Yang <reaver@flomesh.io>

* test: disable k3s service-lb

Signed-off-by: Lin Yang <reaver@flomesh.io>

* chore: regenerate deploy yaml

Signed-off-by: Lin Yang <reaver@flomesh.io>

Signed-off-by: Lin Yang <reaver@flomesh.io>
  • Loading branch information
reaver-flomesh authored Sep 28, 2022
1 parent 407983e commit dc14aab
Show file tree
Hide file tree
Showing 18 changed files with 1,026 additions and 194 deletions.
12 changes: 11 additions & 1 deletion charts/fsm/templates/fsm-role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@ rules:
resources: ["endpoints", "pods", "services", "secrets", "configmaps", "volumes"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"]

{{- if .Values.fsm.serviceLB.enabled }}
- apiGroups: [""]
resources: ["services/status"]
verbs: ["get", "list", "watch", "create", "update", "patch"]

- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
{{- end }}

- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "get", "watch"]
Expand All @@ -42,7 +52,7 @@ rules:

- apiGroups: [""]
resources: ["events"]
verbs: ["create", "watch"]
verbs: ["list", "get", "create", "watch", "patch", "update"]

- apiGroups: ["flomesh.io"]
resources: ["clusters", "proxyprofiles"]
Expand Down
6 changes: 5 additions & 1 deletion charts/fsm/templates/mesh-config-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ data:
"repository": "{{ .Values.fsm.image.repository }}",
"pipyImage": {{ include "fsm.pipy.image.wo-repo" . | quote }},
"proxyInitImage": {{ include "fsm.proxy-init.image.wo-repo" . | quote }},
"clusterConnectorImage": {{ include "fsm.cluster-connector.image.wo-repo" . | quote }}
"klipperLbImage": {{ include "fsm.service-lb.image.wo-repo" . | quote }}
},
"repo": {
Expand Down Expand Up @@ -60,5 +60,9 @@ data:
"zone": "default",
"group": "default",
"name": "local"
},
"serviceLB": {
"enabled": {{ .Values.fsm.serviceLB.enabled }}
}
}
16 changes: 16 additions & 0 deletions charts/fsm/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
"repo",
"ingress",
"gatewayApi",
"serviceLB",
"services",
"configmaps"
],
Expand Down Expand Up @@ -916,6 +917,21 @@
}
}
},
"serviceLB": {
"type": "object",
"default": {},
"title": "The serviceLB Schema",
"required": [
"enabled"
],
"properties": {
"enabled": {
"type": "boolean",
"default": false,
"title": "The enabled Schema"
}
}
},
"services": {
"type": "object",
"default": {},
Expand Down
5 changes: 5 additions & 0 deletions charts/fsm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,11 @@ fsm:
enabled: false
version: v0.4.3

serviceLB:
enabled: false
imageName: mirrored-klipper-lb
tag: v0.3.5

services:
repo:
name: fsm-repo-service
Expand Down
10 changes: 10 additions & 0 deletions charts/tpls/templates/_images.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,14 @@
{{/* curl image */}}
{{- define "fsm.curl.image" -}}
{{- printf "%s/%s" .Values.fsm.image.repository (include "fsm.curl.image.wo-repo" .) -}}
{{- end -}}

{{/* service-lb image without repository */}}
{{- define "fsm.service-lb.image.wo-repo" -}}
{{- printf "%s:%s" .Values.fsm.serviceLB.imageName .Values.fsm.serviceLB.tag -}}
{{- end -}}

{{/* service-lb image */}}
{{- define "fsm.service-lb.image" -}}
{{- printf "%s/%s" .Values.fsm.image.repository (include "fsm.service-lb.image.wo-repo" .) -}}
{{- end -}}
28 changes: 28 additions & 0 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
gatewayv1alpha2 "github.com/flomesh-io/fsm/controllers/gateway/v1alpha2"
nsigv1alpha1 "github.com/flomesh-io/fsm/controllers/namespacedingress/v1alpha1"
proxyprofilev1alpha1 "github.com/flomesh-io/fsm/controllers/proxyprofile/v1alpha1"
svclb "github.com/flomesh-io/fsm/controllers/servicelb"
flomeshadmission "github.com/flomesh-io/fsm/pkg/admission"
"github.com/flomesh-io/fsm/pkg/certificate"
certificateconfig "github.com/flomesh-io/fsm/pkg/certificate/config"
Expand Down Expand Up @@ -327,6 +328,10 @@ func registerCRDs(mgr manager.Manager, api *kube.K8sAPI, controlPlaneConfigStore
if mc.Ingress.Namespaced {
registerNamespacedIngressCRD(mgr, api, controlPlaneConfigStore, certMgr)
}

if mc.ServiceLB.Enabled {
registerServiceLB(mgr, api, controlPlaneConfigStore)
}
}

func registerProxyProfileCRD(mgr manager.Manager, api *kube.K8sAPI, controlPlaneConfigStore *config.Store) {
Expand Down Expand Up @@ -441,6 +446,29 @@ func registerGatewayAPICRDs(mgr manager.Manager, api *kube.K8sAPI, controlPlaneC
}
}

func registerServiceLB(mgr manager.Manager, api *kube.K8sAPI, store *cfghandler.Store) {
if err := (&svclb.ServiceReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("ServiceLB"),
K8sAPI: api,
ControlPlaneConfigStore: store,
}).SetupWithManager(mgr); err != nil {
klog.Fatal(err, "unable to create controller", "controller", "ServiceLB(Service)")
os.Exit(1)
}
if err := (&svclb.NodeReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("ServiceLB"),
K8sAPI: api,
ControlPlaneConfigStore: store,
}).SetupWithManager(mgr); err != nil {
klog.Fatal(err, "unable to create controller", "controller", "ServiceLB(Node)")
os.Exit(1)
}
}

func registerToWebhookServer(mgr manager.Manager, api *kube.K8sAPI, controlPlaneConfigStore *cfghandler.Store) {
hookServer := mgr.GetWebhookServer()
mc := controlPlaneConfigStore.MeshConfig.GetConfig()
Expand Down
8 changes: 8 additions & 0 deletions controllers/proxyprofile/v1alpha1/proxyprofile_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,14 @@ func (r *ProxyProfileReconciler) reconcileRemoteMode(ctx context.Context, pf *pf
// check if the spec is changed, only changed ProxyProfile triggers the restart
oldHash := hashStore[pf.Name]
hash := pf.Annotations[commons.SpecHashAnnotation]
if hash == "" {
// It should not be empty, if it's empty, recalculate and update
hash = pf.SpecHash()
pf.Annotations[commons.SpecHashAnnotation] = hash
if err := r.Update(ctx, pf); err != nil {
return ctrl.Result{}, err
}
}
klog.V(5).Infof("Old Hash=%q, New Hash=%q.", oldHash, hash)
if oldHash == hash {
klog.V(5).Infof("Hash of ProxyProfile %q doesn't change, skipping...", pf.Name)
Expand Down
124 changes: 124 additions & 0 deletions controllers/servicelb/node_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
* MIT License
*
* Copyright (c) since 2021, flomesh.io Authors.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package servicelb

import (
"context"
_ "embed"
"github.com/flomesh-io/fsm/pkg/config"
"github.com/flomesh-io/fsm/pkg/kube"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// NodeReconciler reconciles a Node object
type NodeReconciler struct {
client.Client
K8sAPI *kube.K8sAPI
Scheme *runtime.Scheme
Recorder record.EventRecorder
ControlPlaneConfigStore *config.Store
}

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Node object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.10.0/pkg/reconcile
func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Fetch the Node instance
node := &corev1.Node{}
if err := r.Get(
ctx,
req.NamespacedName,
node,
); err != nil {
if errors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
// Return and don't requeue
klog.V(3).Info("Node resource not found. Ignoring since object must be deleted")
return ctrl.Result{}, nil
}
// Error reading the object - requeue the request.
klog.Errorf("Failed to get Node, %#v", err)
return ctrl.Result{}, err
}

if _, ok := node.Labels[daemonsetNodeLabel]; !ok {
return ctrl.Result{}, nil
}

if err := r.updateDaemonSets(ctx); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

func (r *NodeReconciler) updateDaemonSets(ctx context.Context) error {
klog.V(5).Infof("Updating DaemonSets due to node labels change ...")

daemonsets := &appv1.DaemonSetList{}
if err := r.List(
ctx,
daemonsets,
client.InNamespace(corev1.NamespaceAll),
client.MatchingLabels{
nodeSelectorLabel: "false",
},
); err != nil {
return err
}

for _, ds := range daemonsets.Items {
ds.Spec.Template.Spec.NodeSelector = map[string]string{
daemonsetNodeLabel: "true",
}
ds.Labels[nodeSelectorLabel] = "true"
if err := r.Update(ctx, &ds); err != nil {
return err
}
}

return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&corev1.Node{}).
Complete(r)
}
Loading

0 comments on commit dc14aab

Please sign in to comment.