Added a cloud-based module, responsible for message notifications, li…

…cense issuance and validation, information collection, and monitoring functions. (labring#3409)

* add a notification controller

* fix notification controller bugs, add RWMutex

* Add cloudclient reconcile

* add license controller and cloudsync controller

* add cloudsync controller and fix bugs

* fix go mod

* fix gitignore bugs

* Enhanced the coding standards

* add cloud collector controller, fix some bugs

* add license monitor

* add license monitor

* init encrypt account

* fix bugs

* fix bugs

* fix bugs

* Simplified the code logic and unified the abstract logic under two different scenarios.

* fix bugs

* fix bugs

* fix bugs

* modify the name and fix bugs

---------

Co-authored-by: jiahui <[email protected]>

.github/workflows/controllers.yml

@@ -86,6 +86,7 @@ jobs:
           - { name: db-adminer, path: db/adminer }
           - { name: resources, path: resources }
           - { name: resources-metering, path: resources/metering }
+          - { name: cloud, path: cloud }
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -190,6 +191,7 @@ jobs:
           - { name: db-adminer, path: db/adminer }
           - { name: resources, path: resources }
           - { name: resources-metering, path: resources/metering }
+          - { name: cloud, path: cloud }
     steps:
       - name: Checkout
         uses: actions/checkout@v3

controllers/account/api/v1/account_types.go

@@ -62,11 +62,14 @@ type AccountSpec struct{}
 
 // AccountStatus defines the observed state of Account
 type AccountStatus struct {
+	// EncryptBalance is to encrypt balance
+	EncryptBalance string `json:"encryptBalance,omitempty"`
 	// Recharge amount
 	Balance int64 `json:"balance,omitempty"`
-
 	//Deduction amount
 	DeductionBalance int64 `json:"deductionBalance,omitempty"`
+	// EncryptDeductionBalance is to encrypt DeductionBalance
+	EncryptDeductionBalance string `json:"encryptDeductionBalance,omitempty"`
 	// delete in the future
 	ChargeList []Charge `json:"chargeList,omitempty"`
 }

controllers/account/controllers/account_controller.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/labring/sealos/controllers/pkg/crypto"
+
 	retry2 "k8s.io/client-go/util/retry"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -56,13 +58,15 @@ import (
 
 const (
 	ACCOUNTNAMESPACEENV         = "ACCOUNT_NAMESPACE"
+	PrivateDeployEnv            = "PRIVATE_DEPLOY"
 	DEFAULTACCOUNTNAMESPACE     = "sealos-system"
 	AccountAnnotationNewAccount = "account.sealos.io/new-account"
 	NEWACCOUNTAMOUNTENV         = "NEW_ACCOUNT_AMOUNT"
 )
 
 // AccountReconciler reconciles a Account object
 type AccountReconciler struct {
+	PrivateDeploy bool
 	client.Client
 	Scheme                 *runtime.Scheme
 	Logger                 logr.Logger
@@ -122,6 +126,21 @@ func (r *AccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, fmt.Errorf("get account failed: %v", err)
 	}
 
+	if r.PrivateDeploy {
+		encryptBalance := account.Status.EncryptBalance
+		encryptDeductionBalance := account.Status.EncryptDeductionBalance
+		balance, err := crypto.DecryptInt64(encryptBalance)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("decrypt balance failed: %v", err)
+		}
+		account.Status.Balance = balance
+		deductionBalance, err := crypto.DecryptInt64(encryptDeductionBalance)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("decrypt deduction balance failed: %v", err)
+		}
+		account.Status.DeductionBalance = deductionBalance
+	}
+
 	orderResp, err := pay.QueryOrder(payment.Status.TradeNO)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("query order failed: %v", err)
@@ -145,6 +164,14 @@ func (r *AccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		payAmount := *orderResp.Amount.Total * 10000
 		//1¥ = 100WechatPayAmount; 1 WechatPayAmount = 10000 SealosAmount
 		var gift = giveGift(payAmount)
+		if r.PrivateDeploy {
+			encryptBalance := account.Status.EncryptBalance
+			encryptBalance, err = crypto.RechargeBalance(encryptBalance, payAmount+gift)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("recharge encrypt balance failed: %v", err)
+			}
+			account.Status.EncryptBalance = encryptBalance
+		}
 		account.Status.Balance += payAmount + gift
 		if err := r.Status().Update(ctx, account); err != nil {
 			return ctrl.Result{}, fmt.Errorf("update account failed: %v", err)
@@ -228,6 +255,14 @@ func (r *AccountReconciler) syncAccount(ctx context.Context, name, accountNamesp
 	}); err != nil {
 		return nil, err
 	}
+	if r.PrivateDeploy {
+		encryptBalance := account.Status.EncryptBalance
+		encryptBalance, err = crypto.RechargeBalance(encryptBalance, int64(amount))
+		if err != nil {
+			return nil, fmt.Errorf("recharge balance failed: %v", err)
+		}
+		account.Status.EncryptBalance = encryptBalance
+	}
 	account.Status.Balance += int64(amount)
 	if err := r.Status().Update(ctx, &account); err != nil {
 		return nil, err
@@ -347,6 +382,25 @@ func (r *AccountReconciler) updateDeductionBalance(ctx context.Context, accountB
 	} else {
 		account.Status.DeductionBalance += accountBalance.Spec.Amount
 	}
+	if r.PrivateDeploy {
+		if accountBalance.Spec.Type == accountv1.TransferIn {
+			encryptBalance := account.Status.EncryptBalance
+			encryptBalance, err = crypto.RechargeBalance(encryptBalance, accountBalance.Spec.Amount)
+			if err != nil {
+				r.Logger.Error(err, err.Error())
+				return err
+			}
+			account.Status.EncryptBalance = encryptBalance
+		} else {
+			encryptDeductionBalance := account.Status.EncryptDeductionBalance
+			encryptDeductionBalance, err = crypto.RechargeBalance(encryptDeductionBalance, accountBalance.Spec.Amount)
+			if err != nil {
+				r.Logger.Error(err, err.Error())
+				return err
+			}
+			account.Status.EncryptDeductionBalance = encryptDeductionBalance
+		}
+	}
 
 	if err := r.Status().Update(ctx, account); err != nil {
 		r.Logger.Error(err, err.Error())

controllers/cloud/.dockerignore

@@ -0,0 +1,3 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore build and test binaries.
+testbin/

controllers/cloud/.gitignore

@@ -0,0 +1,26 @@
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin
+testbin/*
+Dockerfile.cross
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+*.swp
+*.swo
+*~

controllers/cloud/Dockerfile

@@ -0,0 +1,8 @@
+FROM gcr.io/distroless/static:nonroot
+ARG TARGETARCH
+
+WORKDIR /
+USER 65532:65532
+
+COPY bin/controller-cloud-$TARGETARCH /manager
+ENTRYPOINT ["/manager"]

controllers/cloud/Makefile

@@ -0,0 +1,162 @@
+
+# Image URL to use all building/pushing image targets
+IMG ?= ghcr.io/labring/sealos-cloud-controller:latest
+# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
+ENVTEST_K8S_VERSION = 1.26.1
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+.PHONY: all
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Development
+
+.PHONY: manifests
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+.PHONY: generate
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+.PHONY: fmt
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+.PHONY: vet
+vet: ## Run go vet against code.
+	go vet ./...
+
+.PHONY: test
+test: manifests generate fmt vet envtest ## Run tests.
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
+
+##@ Build
+
+.PHONY: build
+build: manifests generate fmt vet ## Build manager binary.
+	CGO_ENABLED=0 GOOS=linux  go build -o bin/manager cmd/main.go
+
+.PHONY: run
+run: manifests generate fmt vet ## Run a controller from your host.
+	go run ./cmd/main.go
+
+# If you wish built the manager image targeting other platforms you can use the --platform flag.
+# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
+# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: docker-build
+docker-build: test ## Build docker image with the manager.
+	docker build -t ${IMG} .
+
+.PHONY: docker-push
+docker-push: ## Push docker image with the manager.
+	docker push ${IMG}
+
+# PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/
+# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
+# To properly provided solutions that supports more than one platform you should use this option.
+PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+.PHONY: docker-buildx
+docker-buildx: test ## Build and push docker image for the manager for cross-platform support
+	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	- docker buildx create --name project-v3-builder
+	docker buildx use project-v3-builder
+	- docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- docker buildx rm project-v3-builder
+	rm Dockerfile.cross
+
+##@ Deployment
+
+ifndef ignore-not-found
+  ignore-not-found = false
+endif
+
+.PHONY: install
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+.PHONY: uninstall
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+
+.PHONY: pre-deploy
+pre-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default  > deploy/manifests/deploy.yaml
+
+.PHONY: deploy
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+
+.PHONY: undeploy
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+
+##@ Build Dependencies
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v4.2.0
+CONTROLLER_TOOLS_VERSION ?= v0.8.0
+
+KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.
+$(KUSTOMIZE): $(LOCALBIN)
+	@if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \
+		echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \
+		rm -rf $(LOCALBIN)/kustomize; \
+	fi
+	test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) --output install_kustomize.sh && bash install_kustomize.sh $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); rm install_kustomize.sh; }
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest

controllers/cloud/PROJECT

@@ -0,0 +1,44 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
+domain: sealos.io
+layout:
+- go.kubebuilder.io/v4
+projectName: cloud-controller
+repo: github.com/labring/sealos/controllers/cloud
+resources:
+- controller: true
+  domain: sealos.io
+  group: cloud
+  kind: Notification
+  version: v1
+- controller: true
+  domain: sealos.io
+  group: cloud
+  kind: Collector
+  version: v1
+- controller: true
+  domain: sealos.io
+  group: cloud
+  kind: CloudSync
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: sealos.io
+  group: cloud
+  kind: License
+  path: github.com/labring/sealos/controllers/cloud/api/v1
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: sealos.io
+  group: cloud
+  kind: Launcher
+  path: github.com/labring/sealos/controllers/cloud/api/v1
+  version: v1
+version: "3"