From 40babea9d5fe17c08783e36931e0fa5632ac5a1d Mon Sep 17 00:00:00 2001 From: Jiahui <4543bxy@gmail.com> Date: Mon, 31 Jul 2023 18:25:16 +0800 Subject: [PATCH] feat: add default limitrange (#3603) * add default limit range * add ignore sync anno * fix duplication reconcile * add role * init anno --- .../default/manager_auth_proxy_patch.yaml | 1 + controllers/account/config/rbac/role.yaml | 12 +++ .../account/controllers/account_controller.go | 75 ++++++++++--------- .../account/controllers/billing_controller.go | 16 ---- .../account/deploy/manifests/deploy.yaml | 12 +++ controllers/pkg/common/resources.go | 53 +++++++++++++ 6 files changed, 116 insertions(+), 53 deletions(-) diff --git a/controllers/account/config/default/manager_auth_proxy_patch.yaml b/controllers/account/config/default/manager_auth_proxy_patch.yaml index 753ec18fe45..cd07143a818 100644 --- a/controllers/account/config/default/manager_auth_proxy_patch.yaml +++ b/controllers/account/config/default/manager_auth_proxy_patch.yaml @@ -35,6 +35,7 @@ spec: envFrom: - secretRef: name: payment-secret + optional: true securityContext: runAsNonRoot: true allowPrivilegeEscalation: false diff --git a/controllers/account/config/rbac/role.yaml b/controllers/account/config/rbac/role.yaml index 60c758c0f03..527ee8e0d42 100644 --- a/controllers/account/config/rbac/role.yaml +++ b/controllers/account/config/rbac/role.yaml @@ -245,6 +245,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - limitranges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/account/controllers/account_controller.go b/controllers/account/controllers/account_controller.go index 9e50a9ecd75..a2b86b6a88c 100644 --- a/controllers/account/controllers/account_controller.go +++ b/controllers/account/controllers/account_controller.go @@ -24,12 +24,17 @@ import ( "strings" "time" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" + + "sigs.k8s.io/controller-runtime/pkg/builder" + + "github.com/labring/sealos/controllers/pkg/common" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/builder" - "github.com/labring/sealos/controllers/pkg/crypto" retry2 "k8s.io/client-go/util/retry" @@ -53,23 +58,21 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/labring/sealos/pkg/pay" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source" - accountv1 "github.com/labring/sealos/controllers/account/api/v1" + "github.com/labring/sealos/pkg/pay" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - ACCOUNTNAMESPACEENV = "ACCOUNT_NAMESPACE" - DEFAULTACCOUNTNAMESPACE = "sealos-system" - AccountAnnotationNewAccount = "account.sealos.io/new-account" - NEWACCOUNTAMOUNTENV = "NEW_ACCOUNT_AMOUNT" - RECHARGEGIFT = "recharge-gift" - SEALOS = "sealos" + ACCOUNTNAMESPACEENV = "ACCOUNT_NAMESPACE" + DEFAULTACCOUNTNAMESPACE = "sealos-system" + AccountAnnotationNewAccount = "account.sealos.io/new-account" + AccountAnnotationIgnoreQuota = "account.sealos.io/ignore-quota" + NEWACCOUNTAMOUNTENV = "NEW_ACCOUNT_AMOUNT" + RECHARGEGIFT = "recharge-gift" + SEALOS = "sealos" ) // AccountReconciler reconciles a Account object @@ -84,6 +87,8 @@ type AccountReconciler struct { //+kubebuilder:rbac:groups=account.sealos.io,resources=accounts,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=account.sealos.io,resources=accounts/status,verbs=get;update;patch //+kubebuilder:rbac:groups=account.sealos.io,resources=accounts/finalizers,verbs=update +//+kubebuilder:rbac:groups=core,resources=resourcequotas,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=limitranges,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=user.sealos.io,resources=users,verbs=get;list;watch //+kubebuilder:rbac:groups=account.sealos.io,resources=accountbalances,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=account.sealos.io,resources=accountbalances/status,verbs=get;list;watch;create;update;patch;delete @@ -217,6 +222,7 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp Namespace: accountNamespace, }, } + account.Annotations = make(map[string]string) if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &account, func() error { return nil }); err != nil { @@ -230,6 +236,11 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp if err != nil { return nil, fmt.Errorf("sync init balance failed: %v", err) } + if account.GetAnnotations()[AccountAnnotationIgnoreQuota] != "true" { + if err := r.syncResourceQuotaAndLimitRange(ctx, userNamespace); err != nil { + return nil, fmt.Errorf("sync resource resourceQuota and limitRange failed: %v", err) + } + } // add account balance when account is new user stringAmount := os.Getenv(NEWACCOUNTAMOUNTENV) if stringAmount == "" { @@ -237,7 +248,7 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp return &account, nil } - if newAccountFlag := account.Annotations[AccountAnnotationNewAccount]; newAccountFlag == "false" { + if account.Annotations[AccountAnnotationNewAccount] == "false" { r.Logger.V(1).Info("account is not a new user ", "account", account) return &account, nil } @@ -248,9 +259,6 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp return nil, fmt.Errorf("convert %s to int failed: %v", stringAmount, err) } if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &account, func() error { - if account.Annotations == nil { - account.Annotations = make(map[string]string, 0) - } account.Annotations[AccountAnnotationNewAccount] = "false" return nil }); err != nil { @@ -269,29 +277,23 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp } r.Logger.Info("account created,will charge new account some money", "account", account, "stringAmount", stringAmount) - if err := r.syncResourceQuota(ctx, userNamespace); err != nil { - return nil, fmt.Errorf("sync resource quota failed: %v", err) - } return &account, nil } -func (r *AccountReconciler) syncResourceQuota(ctx context.Context, nsName string) error { - quota := &corev1.ResourceQuota{ - ObjectMeta: metav1.ObjectMeta{ - Name: ResourceQuotaPrefix + nsName, - Namespace: nsName, - }, - } - - return retry.Retry(10, 1*time.Second, func() error { - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, quota, func() error { - quota.Spec.Hard = DefaultResourceQuota() - return nil - }); err != nil { - return fmt.Errorf("sync resource quota failed: %v", err) +func (r *AccountReconciler) syncResourceQuotaAndLimitRange(ctx context.Context, nsName string) error { + objs := []client.Object{client.Object(common.GetDefaultLimitRange(nsName, nsName)), client.Object(common.GetDefaultResourceQuota(nsName, ResourceQuotaPrefix+nsName))} + for i := range objs { + err := retry.Retry(10, 1*time.Second, func() error { + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, objs[i], func() error { + return nil + }) + return err + }) + if err != nil { + return fmt.Errorf("sync resource %T failed: %v", objs[i], err) } - return nil - }) + } + return nil } func (r *AccountReconciler) syncRoleAndRoleBinding(ctx context.Context, name, namespace string) error { @@ -478,10 +480,9 @@ func (r *AccountReconciler) SetupWithManager(mgr ctrl.Manager, rateOpts controll return fmt.Errorf("mongo url is empty") } return ctrl.NewControllerManagedBy(mgr). - For(&accountv1.Account{}). + For(&userV1.User{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}))). Watches(&source.Kind{Type: &accountv1.Payment{}}, &handler.EnqueueRequestForObject{}). Watches(&source.Kind{Type: &accountv1.AccountBalance{}}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(&NamespaceFilterPredicate{Namespace: r.AccountSystemNamespace})). - Watches(&source.Kind{Type: &userV1.User{}}, &handler.EnqueueRequestForObject{}). WithOptions(rateOpts). Complete(r) } diff --git a/controllers/account/controllers/billing_controller.go b/controllers/account/controllers/billing_controller.go index d482e5fc2e1..aa30cf87ba3 100644 --- a/controllers/account/controllers/billing_controller.go +++ b/controllers/account/controllers/billing_controller.go @@ -31,7 +31,6 @@ import ( v1 "github.com/labring/sealos/controllers/user/api/v1" gonanoid "github.com/matoous/go-nanoid/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/event" @@ -290,18 +289,3 @@ func (r *BillingReconciler) SetupWithManager(mgr ctrl.Manager, rateOpts controll func getUsername(namespace string) string { return strings.TrimPrefix(namespace, UserNamespacePrefix) } - -func DefaultResourceQuota() corev1.ResourceList { - return corev1.ResourceList{ - //corev1.ResourceRequestsCPU: resource.MustParse("100"), - corev1.ResourceLimitsCPU: resource.MustParse("16"), - //corev1.ResourceRequestsMemory: resource.MustParse("100"), - corev1.ResourceLimitsMemory: resource.MustParse("64Gi"), - //For all PVCs, the total demand for storage resources cannot exceed this value - corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), - //"limit.storage": resource.MustParse("100Gi"), - //Local ephemeral storage - corev1.ResourceLimitsEphemeralStorage: resource.MustParse("100Gi"), - //corev1.ResourceRequestsEphemeralStorage: resource.MustParse("100Gi"), - } -} diff --git a/controllers/account/deploy/manifests/deploy.yaml b/controllers/account/deploy/manifests/deploy.yaml index 6e56cb4738f..4b651a47c04 100644 --- a/controllers/account/deploy/manifests/deploy.yaml +++ b/controllers/account/deploy/manifests/deploy.yaml @@ -875,6 +875,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - limitranges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/pkg/common/resources.go b/controllers/pkg/common/resources.go index 8609c04fbe5..26a17aa2a64 100644 --- a/controllers/pkg/common/resources.go +++ b/controllers/pkg/common/resources.go @@ -6,6 +6,8 @@ import ( "math" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/labring/sealos/controllers/pkg/common/gpu" "go.mongodb.org/mongo-driver/bson" @@ -167,6 +169,57 @@ var infraMemoryMap = map[string]int{ "ecs.g7.xlarge": 16, } +func GetDefaultResourceQuota(ns, name string) *corev1.ResourceQuota { + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: DefaultResourceQuotaHard(), + }, + } +} + +func GetDefaultLimitRange(ns, name string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: corev1.LimitRangeSpec{ + Limits: DefaultLimitRangeLimits(), + }, + } +} + +func DefaultResourceQuotaHard() corev1.ResourceList { + return corev1.ResourceList{ + //corev1.ResourceRequestsCPU: resource.MustParse("100"), + corev1.ResourceLimitsCPU: resource.MustParse("16"), + //corev1.ResourceRequestsMemory: resource.MustParse("100"), + corev1.ResourceLimitsMemory: resource.MustParse("64Gi"), + //For all PVCs, the total demand for storage resources cannot exceed this value + corev1.ResourceRequestsStorage: resource.MustParse("100Gi"), + //"limit.storage": resource.MustParse("100Gi"), + //Local ephemeral storage + corev1.ResourceLimitsEphemeralStorage: resource.MustParse("100Gi"), + //corev1.ResourceRequestsEphemeralStorage: resource.MustParse("100Gi"), + } +} + +func DefaultLimitRangeLimits() []corev1.LimitRangeItem { + return []corev1.LimitRangeItem{ + { + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + }, + } +} + // MiB func GetInfraCPUQuantity(flavor string, count int) *resource.Quantity { if v, ok := infraCPUMap[flavor]; ok {