From 11fe147b15b905a759419c1dbea90676a6a710df Mon Sep 17 00:00:00 2001 From: jiahui Date: Fri, 3 Nov 2023 17:13:14 +0800 Subject: [PATCH] add webhook handler --- controllers/admission/api/v1/pvc_webhook.go | 114 ++++++++++++++---- controllers/admission/cmd/main.go | 3 + controllers/pkg/prometheus/prometheus_test.go | 1 - 3 files changed, 92 insertions(+), 26 deletions(-) diff --git a/controllers/admission/api/v1/pvc_webhook.go b/controllers/admission/api/v1/pvc_webhook.go index 0e9dccf7f58..9d9a968e04a 100644 --- a/controllers/admission/api/v1/pvc_webhook.go +++ b/controllers/admission/api/v1/pvc_webhook.go @@ -3,6 +3,11 @@ package v1 import ( "context" "fmt" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "k8s.io/api/apps/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -16,11 +21,47 @@ import ( var pvcLog = logf.Log.WithName("pvc-validating-webhook") +// +kubebuilder:webhook:path=/validate-opsrequest-sts-pvc,mutating=false,failurePolicy=fail,groups=apps.kubeblocks.io;apps,resources=opsrequests;statefulsets,verbs=create;update;delete,versions=v1alpha1;v1,name=vresources.kb.io,sideEffects=None,admissionReviewVersions={v1,v1beta1} + type PvcValidator struct { client.Client + PromoURL string +} + +func (v *PvcValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + var obj runtime.Object + var oldObj runtime.Object + var err error + + switch req.Kind.Kind { + case "OpsRequest": + obj = &kbv1alpha1.OpsRequest{} + if req.Operation == admissionv1.Update { + oldObj = &kbv1alpha1.OpsRequest{} + } + case "StatefulSet": + obj = &v1beta2.StatefulSet{} + if req.Operation == admissionv1.Update { + oldObj = &v1beta2.StatefulSet{} + } + default: + return admission.Errored(http.StatusBadRequest, fmt.Errorf("unhandled kind: %s", req.Kind.Kind)) + } + switch req.Operation { + case admissionv1.Create: + err = v.ValidateCreate(ctx, obj) + case admissionv1.Update: + err = v.ValidateUpdate(ctx, oldObj, obj) + } + + if err != nil { + return admission.Denied(err.Error()) + } + + return admission.Allowed("allowed to commit the request") } -func (v *PvcValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (v *PvcValidator) ValidateCreate(_ context.Context, obj runtime.Object) error { ops, isKBOps := obj.(*kbv1alpha1.OpsRequest) if isKBOps && ops.Spec.Type == kbv1alpha1.VolumeExpansionType { return v.validateKBOpsRequest(ops) @@ -28,7 +69,7 @@ func (v *PvcValidator) ValidateCreate(ctx context.Context, obj runtime.Object) e return nil } -func (v *PvcValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (v *PvcValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) error { oldSts, isSts := oldObj.(*v1beta2.StatefulSet) if isSts { return v.validateStatefulSet(oldSts, newObj.(*v1beta2.StatefulSet)) @@ -44,19 +85,18 @@ func (v *PvcValidator) validateKBOpsRequest(opsRequest *kbv1alpha1.OpsRequest) e if opsRequest.Spec.VolumeExpansionList == nil { return fmt.Errorf("volume expansion list is nil") } - stsName := opsRequest.Spec.ClusterRef + "-" + opsRequest.Spec.VolumeExpansionList[0].ComponentName - sts := &v1beta2.StatefulSet{} - if err := v.Client.Get(context.Background(), client.ObjectKey{ - Namespace: opsRequest.Namespace, - Name: stsName, - }, sts); err != nil { - return fmt.Errorf("failed to get sts: %w", err) - } - nodeNames, err := v.getStsPodNodeName(sts) + + //app.kubernetes.io/instance=test-name,app.kubernetes.io/managed-by=kubeblocks + nodeNames, err := v.getPodNodeName(opsRequest.Namespace, client.MatchingLabels{"app.kubernetes.io/instance": opsRequest.Spec.ClusterRef, "app.kubernetes.io/managed-by": "kubeblocks"}) if err != nil { return fmt.Errorf("failed to get sts pod node name: %w", err) } - err = v.checkStorageCapacity(nodeNames, opsRequest.Spec.VolumeExpansionList[0].VolumeClaimTemplates[0].Storage.Value(), opsRequest.Namespace, opsRequest.Name) + expansionSize, err := v.getResizeStorageWithOpsRequest(opsRequest) + if err != nil { + return fmt.Errorf("failed to get storage with ops request: %w", err) + } + + err = v.checkStorageCapacity(nodeNames, expansionSize, opsRequest.Namespace, opsRequest.Name) if err != nil { return err } @@ -64,12 +104,14 @@ func (v *PvcValidator) validateKBOpsRequest(opsRequest *kbv1alpha1.OpsRequest) e } func (v *PvcValidator) validateStatefulSet(oldSts, newSts *v1beta2.StatefulSet) error { - podList, err := v.getStsPodNodeName(newSts) + podList, err := v.getPodNodeName(newSts.Namespace, newSts.Spec.Selector.MatchLabels) if err != nil { pvcLog.Error(err, "failed to get sts pod node name") return nil } - err = v.checkStorageCapacity(podList, newSts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Value(), newSts.Namespace, newSts.Name) + err = v.checkStorageCapacity(podList, + newSts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Value()-oldSts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Value(), + newSts.Namespace, newSts.Name) if err != nil { return err } @@ -83,23 +125,20 @@ func (v *PvcValidator) checkStorageCapacity(nodeNames []string, requestedStorage pvcLog.Error(err, "failed to get lvm vgs total free") return nil } - scanUpSize := residualStorage - requestedStorage - if scanUpSize < 0 { - return fmt.Errorf("pvc %s/%s can not be scaled down", namespace, name) - } if residualStorage < requestedStorage { - return fmt.Errorf("pvc %s/%s can not be scaled up, residual storage is not enough on node %s: %d left", namespace, name, nodeName, residualStorage) + pvcLog.Error(fmt.Errorf("failed to scaled down pvc"), "pvc can not be scaled up", "namespace", namespace, "pvc name", name, "nodeName", nodeName, "residualStorage", residualStorage, "requestedStorage", requestedStorage) + return fmt.Errorf("pvc %s/%s can not be scaled down", namespace, name) } } return nil } -func (v *PvcValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error { - return nil -} +//func (v *PvcValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error { +// return nil +//} func (v *PvcValidator) newLVMVgTotalFreeQuery(node string) (int64, error) { - prom, err := prometheus.NewPrometheus("") + prom, err := prometheus.NewPrometheus(v.PromoURL) if err != nil { return 0, err } @@ -112,9 +151,9 @@ func (v *PvcValidator) newLVMVgTotalFreeQuery(node string) (int64, error) { return int64(residualSize), nil } -func (v *PvcValidator) getStsPodNodeName(sts *v1beta2.StatefulSet) ([]string, error) { +func (v *PvcValidator) getPodNodeName(namespace string, matchLabels client.MatchingLabels) ([]string, error) { podList := &corev1.PodList{} - err := v.Client.List(context.Background(), podList, client.InNamespace(sts.Namespace), client.MatchingLabels(sts.Spec.Selector.MatchLabels)) + err := v.Client.List(context.Background(), podList, client.InNamespace(namespace), matchLabels) if err != nil { return nil, fmt.Errorf("failed to list pods: %w", err) } @@ -124,3 +163,28 @@ func (v *PvcValidator) getStsPodNodeName(sts *v1beta2.StatefulSet) ([]string, er } return nodeNames, nil } + +func (v *PvcValidator) getResizeStorageWithOpsRequest(opsRequest *kbv1alpha1.OpsRequest) (int64, error) { + opsVC := opsRequest.Spec.VolumeExpansionList[0] + opsVCVolumeClaimTemplates := opsVC.VolumeClaimTemplates[0] + cluster := &kbv1alpha1.Cluster{} + if err := v.Client.Get(context.Background(), client.ObjectKey{ + Namespace: opsRequest.Namespace, + Name: opsRequest.Spec.ClusterRef, + }, cluster); err != nil { + return 0, fmt.Errorf("failed to get cluster: %w", err) + } + + for _, cp := range cluster.Spec.ComponentSpecs { + if cp.Name != opsVC.ComponentName { + continue + } + for _, vc := range cp.VolumeClaimTemplates { + if vc.Name != opsVCVolumeClaimTemplates.Name { + continue + } + return opsVCVolumeClaimTemplates.Storage.Value() - vc.Spec.Resources.Requests.Storage().Value(), nil + } + } + return 0, fmt.Errorf("not found volume claim template: %s", opsRequest.Spec.VolumeExpansionList[0].VolumeClaimTemplates[0].Name) +} diff --git a/controllers/admission/cmd/main.go b/controllers/admission/cmd/main.go index 1ec984ad9d7..e41cbe0690e 100644 --- a/controllers/admission/cmd/main.go +++ b/controllers/admission/cmd/main.go @@ -19,6 +19,8 @@ package main import ( "flag" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "os" v1 "github.com/labring/sealos/controllers/admission/api/v1" @@ -108,6 +110,7 @@ func main() { setupLog.Error(err, "unable to create namespace webhook") os.Exit(1) } + mgr.GetWebhookServer().Register("/validate-opsrequest-sts-pvc", &webhook.Admission{Handler: &v1.PvcValidator{Client: mgr.GetClient(), PromoURL: os.Getenv("PROMO_URL")}}) //+kubebuilder:scaffold:builder diff --git a/controllers/pkg/prometheus/prometheus_test.go b/controllers/pkg/prometheus/prometheus_test.go index e5340cd1649..2a152e5649d 100644 --- a/controllers/pkg/prometheus/prometheus_test.go +++ b/controllers/pkg/prometheus/prometheus_test.go @@ -21,7 +21,6 @@ func Test_prometheus_QueryLvmVgsTotalFree(t *testing.T) { fmt.Println("value: ", value) fmt.Println(formatBytes(value)) - } func formatBytes(bytes float64) string {