feat: policy optimizer

Cargo.toml

@@ -1,40 +1,14 @@
-[package]
-name = "policy-server"
-version = "1.7.0-rc1"
-authors = [
-  "Kubewarden Developers <[email protected]>",
-  "Flavio Castelli <[email protected]>",
-  "Rafael Fernández López <[email protected]>",
-  "Víctor Cuadrado Juan <[email protected]>",
-  "José Guilherme Vanz <[email protected]>"
-]
-edition = "2021"
+[workspace]
+members = ["crates/policy-server", "crates/policy-optimizer"]
+resolver = "2"
 
-[dependencies]
+[workspace.dependencies]
 anyhow = "1.0"
-clap = { version = "4.2", features = [ "cargo", "env" ] }
-daemonize = "0.5"
-humansize = "2.1"
-itertools = "0.11.0"
-k8s-openapi = { version = "0.18.0", default-features = false, features = ["v1_26"] }
-lazy_static = "1.4.0"
-num_cpus = "1.16.0"
-opentelemetry-otlp = { version = "0.10.0", features = ["metrics", "tonic"] }
-opentelemetry = { version = "0.17", default-features = false, features = ["metrics", "trace", "rt-tokio", "serialize"] }
-procfs = "0.15"
-policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.11.1" }
-rayon = "1.6"
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_yaml = "0.9.25"
+clap = { version = "4.2", features = ["env"] }
+k8s-openapi = { version = "0.19.0", default-features = false, features = [
+  "v1_26",
+] }
 tokio = { version = "^1", features = ["full"] }
 tracing = "0.1"
 tracing-futures = "0.2"
-tracing-opentelemetry = "0.17.4"
 tracing-subscriber = { version = "0.3", features = ["ansi", "fmt", "json"] }
-warp = { version = "0.3.5", default_features = false, features = [ "multipart", "tls"] }
-semver = { version = "1.0.18", features = ["serde"] }
-
-[dev-dependencies]
-rstest = "0.18"
-tempfile = "3.7.0"

Dockerfile

@@ -1,3 +1,9 @@
+# This is a Dockerfile that can be used to build the policy-server
+# image on a local developer workstation.
+# It could be used to build the official containers inside of GitHub actions,
+# but this would be impractical because the ARM64 build would take ~ 3 hours
+# and half (no kidding!).
+
 FROM rust:1.70 AS build
 WORKDIR /usr/src
 
@@ -10,18 +16,41 @@ ENV CC="clang"
 
 RUN mkdir /usr/src/policy-server
 WORKDIR /usr/src/policy-server
-COPY ./ ./
-RUN cargo install --target $(arch)-unknown-linux-musl --path .
+
+# Building policy-server takes ~8 minutes on a fast machine,
+# we don't want to rebuild that unless something changed inside of its codebase.
+# Because of that we have to make a smart use of Docker cache system.
+
+# Copy files shared by policy-server and policy-optimizer
+COPY Cargo.toml Cargo.lock ./
+RUN mkdir -p crates
+
+# Build policy-server first
+COPY crates/policy-server crates/policy-server
+# Create an empty rust project for policy-optimizer. This is required because the
+# top-level Cargo.toml references the project as part of the workspace. There must
+# be some valid Cargo.toml in there, otherwise building policy-server will fail
+RUN cd crates/ && cargo new --bin policy-optimizer
+RUN cargo build --release --target $(arch)-unknown-linux-musl --bin policy-server
+RUN cp target/$(arch)-unknown-linux-musl/release/policy-server policy-server
+
+# Build policy-optimizer, start by removing the fake project that was created above
+RUN rm -rf crates/policy-optimizer
+COPY crates/policy-optimizer crates/policy-optimizer
+RUN cargo build --release --target $(arch)-unknown-linux-musl --bin policy-optimizer
+RUN cp target/$(arch)-unknown-linux-musl/release/policy-optimizer policy-optimizer
 
 FROM alpine AS cfg
 RUN echo "policy-server:x:65533:65533::/tmp:/sbin/nologin" >> /etc/passwd
 RUN echo "policy-server:x:65533:policy-server" >> /etc/group
 
-# Copy the statically-linked binary into a scratch container.
+# Copy the statically-linked binaries into a scratch container.
 FROM scratch
 COPY --from=cfg /etc/passwd /etc/passwd
 COPY --from=cfg /etc/group /etc/group
-COPY --from=build --chmod=0755 /usr/local/cargo/bin/policy-server /policy-server
+COPY --from=build --chmod=0755 /usr/src/policy-server/policy-server /policy-server
+COPY --from=build --chmod=0755 /usr/src/policy-server/policy-optimizer /policy-optimizer
+# The Cargo.lock contains both the dependencies of policy-server and policy-optimizer
 ADD Cargo.lock /Cargo.lock
 USER 65533:65533
 EXPOSE 3000

Makefile

@@ -3,10 +3,10 @@ IMG ?= policy-server:latest
 BINDIR ?= bin
 SBOM_GENERATOR_TOOL_VERSION ?= v0.0.15
 
-SOURCE_FILES := $(shell test -e src/ && find src -type f)
+SOURCE_FILES_POLICY_SERVER := $(shell test -e crates/policy-server/src/ && find crates/policy-server/src -type f)
 
-target/release/policy-server: $(SOURCE_FILES) Cargo.*
-	cargo build --release
+target/release/policy-server: $(SOURCE_FILES_POLICY_SERVER) crates/policy-server/Cargo.*
+	cargo build -p policy-server --release
 
 .PHONY: build
 build: target/release/policy-server
@@ -17,7 +17,7 @@ fmt:
 
 .PHONY: lint
 lint:
-	cargo clippy -- -D warnings
+	cargo clippy --workspace  -- -D warnings
 
 .PHONY: test
 test: fmt lint

crates/policy-optimizer/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "policy-optimizer"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
+k8s-openapi = { workspace = true }
+kube = { version = "0.85.0", default-features = false, features = [
+  "client",
+  "rustls-tls",
+] }
+# We have to wait to for the next release of the crate that includes this fix
+kubert = { version = "0.18.0", default-features = false, features = ["lease"] }
+tokio = { workspace = true }
+tracing-futures = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tracing = { workspace = true }

crates/policy-optimizer/src/cli.rs

@@ -0,0 +1,34 @@
+use clap::builder::TypedValueParser;
+use clap::Parser;
+use tracing_subscriber::filter::LevelFilter;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+pub struct Cli {
+    // TODO: use the same logging flags as policy-server
+    /// Log level
+    #[arg(
+        long,
+        env = "LOG_LEVEL",
+        default_value_t = LevelFilter::INFO,
+        value_parser = clap::builder::PossibleValuesParser::new(["trace", "debug", "info", "warn", "error"])
+            .map(|s| s.parse::<LevelFilter>().unwrap()),
+    )]
+    pub log_level: LevelFilter,
+
+    /// Policy Server Deployment name
+    #[clap(long, env = "KUBEWARDEN_POLICY_SERVER_DEPLOYMENT_NAME")]
+    pub policy_server_deployment_name: String,
+
+    /// Kubernetes Namespace where the container is running
+    #[clap(long, env = "NAMESPACE")]
+    pub namespace: String,
+
+    /// YAML file holding the policies to be loaded and their settings
+    #[clap(long, env = "KUBEWARDEN_POLICIES")]
+    pub policies: String,
+
+    /// Download path for the policies
+    #[clap(long, env = "KUBEWARDEN_POLICIES_DOWNLOAD_DIR")]
+    pub policies_download_dir: String,
+}

crates/policy-optimizer/src/main.rs

@@ -0,0 +1,133 @@
+// The code dealing with the Lease is based on https://github.com/linkerd/linkerd2/blob/3d601c2ed4382802340c92bcd97a9bae41747958/policy-controller/src/main.rs#L318
+
+use anyhow::{anyhow, Result};
+use clap::Parser;
+use k8s_openapi::api::apps::v1::Deployment;
+use k8s_openapi::api::coordination::v1 as coordv1;
+use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
+use kube::Resource;
+use kube::{api::PatchParams, Client};
+use kubert::lease::LeaseManager;
+use tokio::time::Duration;
+use tracing_subscriber::prelude::*;
+use tracing_subscriber::{filter::EnvFilter, fmt};
+
+const LEASE_NAME_PREFIX: &str = "policy-optimizer";
+const LEASE_DURATION: Duration = Duration::from_secs(30);
+const RENEW_GRACE_PERIOD: Duration = Duration::from_secs(1);
+
+mod cli;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let claimant = std::env::var("HOSTNAME")
+        .map_err(|e| anyhow!("Can access `HOSTNAME` environment variable: {e:?}"))?;
+
+    let cli = cli::Cli::parse();
+    // setup logging
+    let level_filter = cli.log_level;
+    let filter_layer = EnvFilter::from_default_env()
+        .add_directive(level_filter.into())
+        .add_directive("rustls=off".parse().unwrap()) // this crate generates tracing events we don't care about
+        .add_directive("hyper=off".parse().unwrap()) // this crate generates tracing events we don't care about
+        .add_directive("tower=off".parse().unwrap()); // this crate generates tracing events we don't care about
+    tracing_subscriber::registry()
+        .with(filter_layer)
+        .with(fmt::layer().with_writer(std::io::stderr))
+        .init();
+
+    let client = Client::try_default().await?;
+
+    let policy_server_deployment_name =
+        format!("policy-server-{}", cli.policy_server_deployment_name);
+
+    let lease_manager = init_lease(
+        client.clone(),
+        &cli.namespace,
+        &policy_server_deployment_name,
+    )
+    .await?;
+
+    let params = kubert::lease::ClaimParams {
+        lease_duration: LEASE_DURATION,
+        renew_grace_period: RENEW_GRACE_PERIOD,
+    };
+
+    let (mut claims, _task) = lease_manager.spawn(&claimant, params).await?;
+
+    tracing::info!("waiting to be leader");
+    claims
+        .wait_for(|receiver| receiver.is_current_for(&claimant))
+        .await
+        .unwrap();
+
+    // TODO: this is some code faking the policy download & optimize process
+    // replace it with the actual code later on
+    let worker = tokio::spawn(async move {
+        tracing::info!("starting job");
+        for i in 0..10 {
+            tracing::info!("{i} awake");
+            tokio::time::sleep(Duration::from_secs(5)).await;
+        }
+    });
+
+    worker.await.unwrap();
+
+    Ok(())
+}
+
+async fn init_lease(client: Client, ns: &str, deployment_name: &str) -> Result<LeaseManager> {
+    // Fetch the policy-server deployment so that we can use it as an owner
+    // reference of the Lease.
+    let api = kube::Api::<Deployment>::namespaced(client.clone(), ns);
+    let deployment = api.get(deployment_name).await?;
+
+    let api = kube::Api::namespaced(client, ns);
+    let params = PatchParams {
+        field_manager: Some("policy-server".to_string()),
+        ..Default::default()
+    };
+
+    let lease_name = format!("{LEASE_NAME_PREFIX}-{deployment_name}");
+
+    match api
+        .patch(
+            &lease_name,
+            &params,
+            &kube::api::Patch::Apply(coordv1::Lease {
+                metadata: ObjectMeta {
+                    name: Some(lease_name.clone()),
+                    namespace: Some(ns.to_string()),
+                    // Specifying a resource version of "0" means that we will
+                    // only create the Lease if it does not already exist.
+                    resource_version: Some("0".to_string()),
+                    owner_references: Some(vec![deployment.controller_owner_ref(&()).unwrap()]),
+                    labels: Some(
+                        [(
+                            "kubewarden.io/policy-server".to_string(),
+                            deployment_name.to_string(),
+                        )]
+                        .into_iter()
+                        .collect(),
+                    ),
+                    ..Default::default()
+                },
+                spec: None,
+            }),
+        )
+        .await
+    {
+        Ok(lease) => tracing::info!(?lease, "created Lease resource"),
+        Err(kube::Error::Api(_)) => tracing::info!("Lease already exists, no need to create it"),
+        Err(error) => {
+            tracing::error!(%error, "error creating Lease resource");
+            return Err(error.into());
+        }
+    };
+    // Create the lease manager used for trying to claim the policy
+    // controller write lease.
+    // todo: Do we need to use LeaseManager::field_manager here?
+    kubert::lease::LeaseManager::init(api, lease_name)
+        .await
+        .map_err(Into::into)
+}

crates/policy-server/Cargo.toml

@@ -0,0 +1,48 @@
+[package]
+name = "policy-server"
+version = "1.7.0-rc1"
+authors = [
+  "Kubewarden Developers <[email protected]>",
+  "Flavio Castelli <[email protected]>",
+  "Rafael Fernández López <[email protected]>",
+  "Víctor Cuadrado Juan <[email protected]>",
+  "José Guilherme Vanz <[email protected]>",
+]
+edition = "2021"
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["cargo"] }
+daemonize = "0.5"
+humansize = "2.1"
+itertools = "0.11.0"
+k8s-openapi = { workspace = true }
+lazy_static = "1.4.0"
+num_cpus = "1.16.0"
+opentelemetry-otlp = { version = "0.10.0", features = ["metrics", "tonic"] }
+opentelemetry = { version = "0.17", default-features = false, features = [
+  "metrics",
+  "trace",
+  "rt-tokio",
+  "serialize",
+] }
+policy-evaluator = { git = "https://github.com/kubewarden/policy-evaluator", tag = "v0.11.3" }
+procfs = "0.15"
+rayon = "1.6"
+semver = { version = "1.0.18", features = ["serde"] }
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.9.25"
+tokio = { workspace = true }
+tracing-futures = { workspace = true }
+tracing-opentelemetry = "0.17.4"
+tracing-subscriber = { workspace = true }
+tracing = { workspace = true }
+warp = { version = "0.3.5", default_features = false, features = [
+  "multipart",
+  "tls",
+] }
+
+[dev-dependencies]
+rstest = "0.18"
+tempfile = "3.7.0"