Merge pull request #247 from zach-source/feat/addCelReadiness

feat: adds derive from cel query readiness policy

apis/object/v1alpha2/types.go

@@ -120,17 +120,31 @@ const (
 	// ReadinessPolicyAllTrue means that all conditions have status true on the object.
 	// There must be at least one condition.
 	ReadinessPolicyAllTrue ReadinessPolicy = "AllTrue"
+	// ReadinessPolicyDeriveFromCelQuery means that a cel expression will be used to calculate the overall status.
+	// The cel expression must be provided on the readiness struct.
+	ReadinessPolicyDeriveFromCelQuery ReadinessPolicy = "DeriveFromCelQuery"
 )
 
 // Readiness defines how the object's readiness condition should be computed,
 // if not specified it will be considered ready as soon as the underlying external
 // resource is considered up-to-date.
+// +kubebuilder:validation:XValidation:rule="self.policy != 'DeriveFromCelQuery' || (self.policy == 'DeriveFromCelQuery' && size(self.celQuery) > 0)",message="celQuery must be set if policy is DeriveFromCelQuery"
 type Readiness struct {
 	// Policy defines how the Object's readiness condition should be computed.
 	// +optional
-	// +kubebuilder:validation:Enum=SuccessfulCreate;DeriveFromObject;AllTrue
+	// +kubebuilder:validation:Enum=SuccessfulCreate;DeriveFromObject;AllTrue;DeriveFromCelQuery
 	// +kubebuilder:default=SuccessfulCreate
 	Policy ReadinessPolicy `json:"policy,omitempty"`
+
+	// CelQuery defines a cel query to evaluate the readiness. The
+	// observed object is passed to the cel query with the word `object`.
+	// Cel macros are available to be used, see https://github.com/google/cel-spec/blob/master/doc/langdef.md#macros
+	// for more information.
+	// Examples:
+	//  `object.status.isReady == true`: checks for a boolean field called isReady on status.
+	//  `object.status.conditions.all(x, x.status == "True")` mimics the behavior of the AllTrue readiness policy
+	//  `object.status.conditions.exists(c, c.type == "condition1" && c.status == "True" )` checks just one condition
+	CelQuery string `json:"celQuery,omitempty"`
 }
 
 // ConnectionDetail represents an entry in the connection secret for an Object

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60
 	github.com/crossplane/crossplane-tools v0.0.0-20240522174801-1ad3d4c87f21
+	github.com/google/cel-go v0.17.7
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/pkg/errors v0.9.1
@@ -38,6 +39,7 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -79,6 +81,7 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
@@ -91,6 +94,8 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.18.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/retry.v1 v1.0.3 // indirect

go.sum

@@ -32,6 +32,8 @@ github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjH
 github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -88,6 +90,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ=
+github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU=
 github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M=
 github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -167,11 +171,14 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -267,6 +274,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo=
+google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/controller/object/object.go

@@ -21,9 +21,12 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/rand"
+	"reflect"
 	"strings"
 	"time"
 
+	"github.com/google/cel-go/cel"
+	celtypes "github.com/google/cel-go/common/types"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -96,6 +99,14 @@ const (
 	errGetValueAtFieldPath  = "cannot get value at fieldPath"
 	errDecodeSecretData     = "cannot decode secret data"
 	errSanitizeSecretData   = "cannot sanitize secret data"
+
+	errCelQueryFailedToCompile           = "failed to compile query"
+	errCelQueryReturnTypeNotBool         = "celQuery does not return a bool type"
+	errCelQueryFailedToCreateProgram     = "failed to create program from the cel query"
+	errCelQueryFailedToEvalProgram       = "failed to eval the program"
+	errCelQueryCannotBeEmpty             = "cel query cannot be empty"
+	errCelQueryFailedToCreateEnvironment = "cel query failed to create environment"
+	errCelQueryJSON                      = "failed to marshal or unmarshal the obj for cel query"
 )
 
 // KindObserver tracks kinds of referenced composed resources in order to start
@@ -418,48 +429,36 @@ func (c *external) setObserved(obj *v1alpha2.Object, observed *unstructured.Unst
 }
 
 func (c *external) updateConditionFromObserved(obj *v1alpha2.Object, observed *unstructured.Unstructured) error {
+	var ready bool
+	var err error
+
 	switch obj.Spec.Readiness.Policy {
 	case v1alpha2.ReadinessPolicyDeriveFromObject:
-		conditioned := xpv1.ConditionedStatus{}
-		err := fieldpath.Pave(observed.Object).GetValueInto("status", &conditioned)
-		if err != nil {
-			c.logger.Debug("Got error while getting conditions from observed object, setting it as Unavailable", "error", err, "observed", observed)
-			obj.SetConditions(xpv1.Unavailable())
-			return nil
-		}
-		if status := conditioned.GetCondition(xpv1.TypeReady).Status; status != v1.ConditionTrue {
-			c.logger.Debug("Observed object is not ready, setting it as Unavailable", "status", status, "observed", observed)
-			obj.SetConditions(xpv1.Unavailable())
-			return nil
-		}
-		obj.SetConditions(xpv1.Available())
+		ready = c.checkDeriveFromObject(observed)
 	case v1alpha2.ReadinessPolicyAllTrue:
-		conditioned := xpv1.ConditionedStatus{}
-		err := fieldpath.Pave(observed.Object).GetValueInto("status", &conditioned)
-		if err != nil {
-			c.logger.Debug("Got error while getting conditions from observed object, setting it as Unavailable", "error", err, "observed", observed)
-			obj.SetConditions(xpv1.Unavailable())
-			return nil
-		}
-		allTrue := len(conditioned.Conditions) > 0
-		for _, condition := range conditioned.Conditions {
-			if condition.Status != v1.ConditionTrue {
-				allTrue = false
-				break
-			}
-		}
-		if allTrue {
-			obj.SetConditions(xpv1.Available())
-		} else {
-			obj.SetConditions(xpv1.Unavailable())
-		}
+		ready = c.checkAllConditions(observed)
+	case v1alpha2.ReadinessPolicyDeriveFromCelQuery:
+		ready, err = c.checkDeriveFromCelQuery(obj, observed)
 	case v1alpha2.ReadinessPolicySuccessfulCreate, "":
 		// do nothing, will be handled by c.handleLastApplied method
 		// "" should never happen, but just in case we will treat it as SuccessfulCreate for backward compatibility
+		return nil
 	default:
 		// should never happen
 		return errors.Errorf("unknown readiness policy %q", obj.Spec.Readiness.Policy)
 	}
+
+	if err != nil {
+		obj.SetConditions(xpv1.Unavailable().WithMessage(err.Error()))
+		return nil
+	}
+
+	if !ready {
+		obj.SetConditions(xpv1.Unavailable())
+		return nil
+	}
+
+	obj.SetConditions(xpv1.Available())
 	return nil
 }
 
@@ -483,6 +482,103 @@ func getReferenceInfo(ref v1alpha2.Reference) (string, string, string, string) {
 	return apiVersion, kind, namespace, name
 }
 
+func (c *external) checkDeriveFromObject(observed *unstructured.Unstructured) bool {
+	conditioned := xpv1.ConditionedStatus{}
+	if err := fieldpath.Pave(observed.Object).GetValueInto("status", &conditioned); err != nil {
+		c.logger.Debug("Got error while getting conditions from observed object, setting it as Unavailable", "error", err, "observed", observed)
+		return false
+	}
+	if status := conditioned.GetCondition(xpv1.TypeReady).Status; status != v1.ConditionTrue {
+		c.logger.Debug("Observed object is not ready, setting it as Unavailable", "status", status, "observed", observed)
+		return false
+	}
+	return true
+}
+
+func (c *external) checkAllConditions(observed *unstructured.Unstructured) (allTrue bool) {
+	conditioned := xpv1.ConditionedStatus{}
+	err := fieldpath.Pave(observed.Object).GetValueInto("status", &conditioned)
+	if err != nil {
+		c.logger.Debug("Got error while getting conditions from observed object, setting it as Unavailable", "error", err, "observed", observed)
+		return false
+	}
+	allTrue = len(conditioned.Conditions) > 0
+	for _, condition := range conditioned.Conditions {
+		if condition.Status != v1.ConditionTrue {
+			allTrue = false
+			return allTrue
+		}
+	}
+	return allTrue
+}
+
+// checkDeriveFromCelQuery will look at the celQuery field and run it as a program, using the observed object as input to
+// evaluate if the object is ready or not
+func (c *external) checkDeriveFromCelQuery(obj *v1alpha2.Object, observed *unstructured.Unstructured) (ready bool, err error) {
+	// There is a validation on it but this can still happen before 1.29
+	if obj.Spec.Readiness.CelQuery == "" {
+		c.logger.Debug("cel query is empty")
+		err = errors.New(errCelQueryCannotBeEmpty)
+		return ready, err
+	}
+
+	env, err := cel.NewEnv(
+		cel.Variable("object", cel.AnyType),
+	)
+	if err != nil {
+		c.logger.Debug("failed to create cel env", "err", err)
+		err = errors.Wrap(err, errCelQueryFailedToCreateEnvironment)
+		return ready, err
+	}
+
+	ast, iss := env.Compile(obj.Spec.Readiness.CelQuery)
+	if iss.Err() != nil {
+		c.logger.Debug("failed to compile query", "err", iss.Err())
+		err = errors.Wrap(err, errCelQueryFailedToCompile)
+		return ready, err
+	}
+	if !reflect.DeepEqual(ast.OutputType(), cel.BoolType) {
+		c.logger.Debug(errCelQueryReturnTypeNotBool, "err", iss.Err())
+		err = errors.Wrap(err, errCelQueryReturnTypeNotBool)
+		return ready, err
+	}
+
+	program, err := env.Program(ast)
+	if err != nil {
+		c.logger.Debug("failed to create program from the cel query", "err", err)
+		err = errors.Wrap(err, errCelQueryFailedToCreateProgram)
+		return ready, err
+	}
+
+	data, err := json.Marshal(observed.Object)
+	if err != nil {
+		// this should not happen, but just in case
+		c.logger.Debug("failed to marshal the object", "err", err)
+		err = errors.Wrap(err, errCelQueryJSON)
+		return ready, err
+	}
+	objMap := map[string]any{}
+	err = json.Unmarshal(data, &objMap)
+	if err != nil {
+		// this should not happen, but just in case
+		c.logger.Debug("failed to unmarshal the object", "err", err)
+		err = errors.Wrap(err, errCelQueryJSON)
+		return ready, err
+	}
+
+	val, _, err := program.Eval(map[string]any{
+		"object": objMap,
+	})
+	if err != nil {
+		c.logger.Debug("failed to eval the program", "err", err)
+		err = errors.Wrap(err, errCelQueryFailedToEvalProgram)
+		return ready, err
+	}
+
+	ready = (val == celtypes.True)
+	return ready, err
+}
+
 // resolveReferencies resolves references for the current Object. If it fails to
 // resolve some reference, e.g.: due to reference not ready, it will then return
 // error and requeue to wait for resolving it next time.