feat: application set handler (#241)

* application set handler

1. added appset watcher
2. change: cluster role (applicationset permisssion), ability to read/get/list secrets
3. change: updated the helm chart version
4. change: tiltfile for appset, Tile will no longer install appset yaml to the local k8s, instead, copies the template appsets yaml with replaced repo_url. Terraform will copy the file to the appropriate repo.
5. new: kubernetes client for EKS.
6. new: in EKS Client, STS authentication is automated with a custom http transport (to auto refresh token)
7. new: extended the cobra flag, and serverConfig to handle kubernetes-type (e.g. eks, local), kubernetes-clusterid, kubernetes-cluster-region
8. change: moved appwatcher's kubeclient init to container and instead pass the rest.Config as parameter to appwatcher.
9. new: forked generator code from argo-cd (excluding pull_request generator as it is not compatible with go-gitlab v0.105)

* ci fix

1. lint fix
2. remove commented code from app_directory
3. app and appset watcher to check kubeCfg is not nil
4. add comments to the docs

* role based access control chart

1. add new helm chart to help install role/rolebinding to grant access to the kubernetes cluster remotely.

---------

Signed-off-by: James Hong <[email protected]>

.mockery.yaml

@@ -5,3 +5,11 @@ packages:
     # place your package-specific config here
     config:
       all: true
+  github.com/zapier/kubechecks/pkg/generator:
+    # place your package-specific config here
+    config:
+      all: true
+  github.com/zapier/kubechecks/pkg/affected_apps:
+    # place your package-specific config here
+    config:
+      all: true

Tiltfile

@@ -5,6 +5,7 @@ load('ext://tests/golang', 'test_go')
 load('ext://namespace', 'namespace_create')
 load('ext://uibutton', 'cmd_button')
 load('ext://helm_resource', 'helm_resource')
+load('ext://local_output', 'local_output')
 load('./.tilt/terraform/Tiltfile', 'local_terraform_resource')
 load('./.tilt/utils/Tiltfile', 'check_env_set')
 
@@ -144,10 +145,6 @@ test_go(
 )
 
 
-# get the git commit ref
-def get_git_head():
-    result = local('git rev-parse --short HEAD')
-    return result
 
 # read .tool-versions file and return a dictionary of tools and their versions
 def parse_tool_versions(fn):
@@ -174,7 +171,9 @@ def parse_tool_versions(fn):
     return tools
 
 tool_versions = parse_tool_versions(".tool-versions")
-git_commit = str(get_git_head()).strip()
+
+# get the git commit ref
+git_commit = local_output('git rev-parse --short HEAD')
 
 earthly_build(
     context='.',
@@ -260,8 +259,8 @@ k8s_resource(
 load("localdev/test_apps/Tiltfile", "install_test_apps")
 install_test_apps(cfg)
 
-load("localdev/test_appsets/Tiltfile", "install_test_appsets")
-install_test_appsets(cfg)
+load("localdev/test_appsets/Tiltfile", "copy_test_appsets")
+copy_test_appsets(cfg)
 
 
 force_argocd_cleanup_on_tilt_down()

charts/kubechecks-rbac/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: kubechecks-rbac
+description: A Helm chart for kubechecks Role and RoleBinding
+version: 0.4.5
+type: application
+maintainers:
+  - name: zapier

charts/kubechecks-rbac/README.md

@@ -0,0 +1,6 @@
+# kubechecks-rbac
+
+This chart deploys the Cluster Role and Cluster Role binding for the kubechecks running outside of existing cluster. 
+
+It is not required if you're operating all within the same cluster.
+

charts/kubechecks-rbac/templates/role.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Values.clusterRoleName | default "kubechecks-remote-clusterrole" }}
+rules:
+  - apiGroups: ['argoproj.io']
+    resources: ['applications', 'appprojects', 'applicationsets', 'services']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: [''] # The core API group, which is indicated by an empty string
+    resources: ['secrets']
+    verbs: ['get', 'list', 'watch']

charts/kubechecks-rbac/templates/rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Values.clusterRoleBindingName | default "kubechecks-remote-role-binding" }}
+  namespace: {{ .Values.namespace | default "argocd" }}
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: {{ .Values.clusterRoleBindingGroup | default "kubechecks-remote-group" }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Values.clusterRoleName | default "kubechecks-remote-role" }}
+  apiGroup: rbac.authorization.k8s.io

charts/kubechecks-rbac/tests/role_test.yaml

@@ -0,0 +1,15 @@
+suite: role tests
+
+templates:
+  - role.yaml
+
+tests:
+  - it: should create a Role with the correct name
+    set:
+      clusterRoleName: "kubechecks-test-role"
+    asserts:
+      - isKind:
+          of: ClusterRole
+      - equal:
+          path: metadata.name
+          value: kubechecks-test-role

charts/kubechecks-rbac/tests/rolebinding_test.yaml

@@ -0,0 +1,16 @@
+suite: role binding tests
+
+templates:
+  - rolebinding.yaml
+
+tests:
+  - it: should create a RoleBinding with the correct name with EKS IAM role
+    set:
+      clusterRoleBindingName: "kubechecks-test-rolebinding-rbac"
+      clusterRoleBindingGroup: "kubechecks-remote-group"
+    asserts:
+      - isKind:
+          of: ClusterRoleBinding
+      - equal:
+          path: metadata.name
+          value: kubechecks-test-rolebinding-rbac

charts/kubechecks-rbac/values.schema.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Kubechecks Values Schema",
+  "type": "object",
+  "properties": {
+    "clusterRoleName": {
+      "type": "string",
+      "description": "The name of the Cluster Role to be created.",
+      "default": "kubechecks-remote-role"
+    },
+    "clusterRoleBindingName": {
+      "type": "string",
+      "description": "The name of the ClusterRoleBinding to be created.",
+      "default": "kubechecks-remote-role-binding"
+    },
+    "clusterRoleBindingGroup": {
+      "type": "string",
+      "description": "The name of the Group to be created.",
+      "default": "kubechecks-remote-group"
+    },
+    "namespace": {
+      "type": "string",
+      "description": "The namespace where the Role and RoleBinding will be created.",
+      "default": "argocd"
+    }
+  },
+  "required": ["clusterRoleName", "clusterRoleBindingName", "clusterRoleBindingGroup", "namespace"],
+  "additionalProperties": false
+}

charts/kubechecks-rbac/values.yaml

@@ -0,0 +1,6 @@
+clusterRoleName: "kubechecks-remote-role"
+clusterRoleBindingName: "kubechecks-remote-role-binding"
+clusterRoleBindingGroup: "kubechecks-remote-group"
+
+# namespace to create the ClusterRole and RoleBinding, this has to match the argocd is operating.
+namespace: "argocd"

charts/kubechecks/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kubechecks
 description: A Helm chart for kubechecks
-version: 0.4.4
+version: 0.4.5
 type: application
 maintainers:
   - name: zapier

charts/kubechecks/templates/clusterrole.yaml

@@ -4,5 +4,8 @@ metadata:
   name: {{ include "kubechecks.fullname" . }}
 rules:
   - apiGroups: ['argoproj.io']
-    resources: ['applications', 'appprojects', 'services']
+    resources: ['applications', 'appprojects', 'applicationsets', 'services']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: [''] # The core API group, which is indicated by an empty string
+    resources: ['secrets']
     verbs: ['get', 'list', 'watch']

charts/kubechecks/values.yaml

@@ -6,6 +6,7 @@ configMap:
   env: {}
   # KUBECHECKS_ARGOCD_API_INSECURE: "false"
   # KUBECHECKS_ARGOCD_API_PATH_PREFIX: /
+  # KUBECHECKS_ARGOCD_API_NAMESPACE: argocd
   # KUBECHECKS_ARGOCD_WEBHOOK_URL: https://argocd.<domain.com>/api/webhook
   # KUBECHECKS_FALLBACK_K8S_VERSION: "1.22.0"
   # KUBECHECKS_LOG_LEVEL: debug

cmd/container.go

@@ -12,6 +12,7 @@ import (
 	"github.com/zapier/kubechecks/pkg/config"
 	"github.com/zapier/kubechecks/pkg/container"
 	"github.com/zapier/kubechecks/pkg/git"
+	client "github.com/zapier/kubechecks/pkg/kubernetes"
 	"github.com/zapier/kubechecks/pkg/vcs/github_client"
 	"github.com/zapier/kubechecks/pkg/vcs/gitlab_client"
 )
@@ -36,7 +37,30 @@ func newContainer(ctx context.Context, cfg config.ServerConfig, watchApps bool)
 	if err != nil {
 		return ctr, errors.Wrap(err, "failed to create vcs client")
 	}
+	var kubeClient client.Interface
 
+	switch cfg.KubernetesType {
+	// TODO: expand with other cluster types
+	case client.ClusterTypeLOCAL:
+		kubeClient, err = client.New(&client.NewClientInput{
+			KubernetesConfigPath: cfg.KubernetesConfig,
+			ClusterType:          cfg.KubernetesType,
+		})
+		if err != nil {
+			return ctr, errors.Wrap(err, "failed to create kube client")
+		}
+	case client.ClusterTypeEKS:
+		kubeClient, err = client.New(&client.NewClientInput{
+			KubernetesConfigPath: cfg.KubernetesConfig,
+			ClusterType:          cfg.KubernetesType,
+		},
+			client.EKSClientOption(ctx, cfg.KubernetesClusterID),
+		)
+		if err != nil {
+			return ctr, errors.Wrap(err, "failed to create kube client")
+		}
+	}
+	ctr.KubeClientSet = kubeClient
 	// create argo client
 	if ctr.ArgoClient, err = argo_client.NewArgoClient(cfg); err != nil {
 		return ctr, errors.Wrap(err, "failed to create argo client")
@@ -52,13 +76,22 @@ func newContainer(ctx context.Context, cfg config.ServerConfig, watchApps bool)
 			return ctr, errors.Wrap(err, "failed to build apps map")
 		}
 
+		if err = buildAppSetsMap(ctx, ctr.ArgoClient, ctr.VcsToArgoMap); err != nil {
+			return ctr, errors.Wrap(err, "failed to build appsets map")
+		}
+
 		if watchApps {
-			ctr.ApplicationWatcher, err = app_watcher.NewApplicationWatcher(vcsToArgoMap, cfg)
+			ctr.ApplicationWatcher, err = app_watcher.NewApplicationWatcher(kubeClient.Config(), vcsToArgoMap, cfg)
 			if err != nil {
 				return ctr, errors.Wrap(err, "failed to create watch applications")
 			}
+			ctr.ApplicationSetWatcher, err = app_watcher.NewApplicationSetWatcher(kubeClient.Config(), vcsToArgoMap, cfg)
+			if err != nil {
+				return ctr, errors.Wrap(err, "failed to create watch application sets")
+			}
 
 			go ctr.ApplicationWatcher.Run(ctx, 1)
+			go ctr.ApplicationSetWatcher.Run(ctx)
 		}
 	} else {
 		log.Info().Msgf("not monitoring applications, MonitorAllApplications: %+v", cfg.MonitorAllApplications)
@@ -75,6 +108,16 @@ func buildAppsMap(ctx context.Context, argoClient *argo_client.ArgoClient, resul
 	for _, app := range apps.Items {
 		result.AddApp(&app)
 	}
+	return nil
+}
 
+func buildAppSetsMap(ctx context.Context, argoClient *argo_client.ArgoClient, result container.VcsToArgoMap) error {
+	appSets, err := argoClient.GetApplicationSets(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to list application sets")
+	}
+	for _, appSet := range appSets.Items {
+		result.AddAppSet(&appSet)
+	}
 	return nil
 }

cmd/root.go

@@ -63,6 +63,11 @@ func init() {
 	stringFlag(flags, "argocd-api-namespace", "ArgoCD namespace where the application watcher will read Custom Resource Definitions (CRD) for Application and ApplicationSet resources.",
 		newStringOpts().
 			withDefault("argocd"))
+	stringFlag(flags, "kubernetes-type", "Kubernetes Type One of eks, or local. Defaults to local.",
+		newStringOpts().
+			withChoices("eks", "local").
+			withDefault("local"))
+	stringFlag(flags, "kubernetes-clusterid", "Kubernetes Cluster ID, must be specified if kubernetes-type is eks.")
 	stringFlag(flags, "kubernetes-config", "Path to your kubernetes config file, used to monitor applications.")
 
 	stringFlag(flags, "otel-collector-port", "The OpenTelemetry collector port.")
@@ -73,10 +78,7 @@ func init() {
 		newStringOpts().
 			withChoices("hide", "delete").
 			withDefault("hide"))
-	stringSliceFlag(flags, "schemas-location", "Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.",
-		newStringSliceOpts().
-			withDefault([]string{"./schemas"}))
-
+	stringSliceFlag(flags, "schemas-location", "Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.")
 	boolFlag(flags, "enable-conftest", "Set to true to enable conftest policy checking of manifests.")
 	stringSliceFlag(flags, "policies-location", "Sets rego policy locations to be used for every check request. Can be common path inside the repos being checked or git urls in either git or http(s) format.",
 		newStringSliceOpts().

docs/architecture.md

@@ -36,3 +36,10 @@ By abstracting the PR/MR in this way, `kubechecks` remains VCS provider agnostic
 ![Check Event and Repo type diagrams](./img/checkevent.png){: style="height:350px;display:block;margin:0 auto;"}
 
 The final piece of the puzzle is the `CheckEvent`; an internal structure that takes a `Client` and a `Repo` and begins running all configured checks. A `CheckEvent` first determines what applications within the repository have been affected by the PR/MR, and begins concurrently running the check suite against each affected application to generate a report for that app. As each application updates its report, the `CheckEvent` compiles all reports together and instructs the `Client` to update the PR/MR with a comment detailing the current progress; resulting in one comment per run of `kubechecks` with the latest information about that particular run. Whenever a new run of `kubechecks` is initiated, all previous comments are deleted to reduce clutter.
+
+### Event Flow Diagram
+
+![Event Flow Diagram](./img/eventflowdiagram.png){: style="height:350px;display:block;margin:0 auto;"}
+
+This diagram illustrates the flow of events from the initial webhook trigger to the final report generation and comment update process.
+

docs/usage.md

@@ -46,7 +46,9 @@ The full list of supported environment variables is described below:
 |`KUBECHECKS_ENABLE_PREUPGRADE`|Enable preupgrade checks.|`true`|
 |`KUBECHECKS_ENSURE_WEBHOOKS`|Ensure that webhooks are created in repositories referenced by argo.|`false`|
 |`KUBECHECKS_FALLBACK_K8S_VERSION`|Fallback target Kubernetes version for schema / upgrade checks.|`1.23.0`|
+|`KUBECHECKS_KUBERNETES_CLUSTERID`|Kubernetes Cluster ID, must be specified if kubernetes-type is eks.||
 |`KUBECHECKS_KUBERNETES_CONFIG`|Path to your kubernetes config file, used to monitor applications.||
+|`KUBECHECKS_KUBERNETES_TYPE`|Kubernetes Type One of eks, or local.|`local`|
 |`KUBECHECKS_LABEL_FILTER`|(Optional) If set, The label that must be set on an MR (as "kubechecks:<value>") for kubechecks to process the merge request webhook.||
 |`KUBECHECKS_LOG_LEVEL`|Set the log output level. One of error, warn, info, debug, trace.|`info`|
 |`KUBECHECKS_MAX_CONCURRENCT_CHECKS`|Number of concurrent checks to run.|`32`|
@@ -59,7 +61,7 @@ The full list of supported environment variables is described below:
 |`KUBECHECKS_PERSIST_LOG_LEVEL`|Persists the set log level down to other module loggers.|`false`|
 |`KUBECHECKS_POLICIES_LOCATION`|Sets rego policy locations to be used for every check request. Can be common path inside the repos being checked or git urls in either git or http(s) format.|`[./policies]`|
 |`KUBECHECKS_REPO_REFRESH_INTERVAL`|Interval between static repo refreshes (for schemas and policies).|`5m`|
-|`KUBECHECKS_SCHEMAS_LOCATION`|Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.|`[./schemas]`|
+|`KUBECHECKS_SCHEMAS_LOCATION`|Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.|`[]`|
 |`KUBECHECKS_SHOW_DEBUG_INFO`|Set to true to print debug info to the footer of MR comments.|`false`|
 |`KUBECHECKS_TIDY_OUTDATED_COMMENTS_MODE`|Sets the mode to use when tidying outdated comments. One of hide, delete.|`hide`|
 |`KUBECHECKS_VCS_BASE_URL`|VCS base url, useful if self hosting gitlab, enterprise github, etc.||