Alloy v2

.gitignore

@@ -4,4 +4,5 @@
 *.bke
 *.tgz
 *.tar.gz
+values-nats.yaml
 scripts/serverservice/add-server

Makefile

@@ -1,18 +1,22 @@
 .DEFAULT_GOAL := help
 
 ## install helm chart for the sandbox env
-local-devel: kubectl-ctx-kind
-	helm install hollow-sandbox . -f values.yaml
+install: kubectl-ctx-kind
+	cp ./scripts/nats-bootstrap/values-nats.yaml.tmpl values-nats.yaml
+	helm install hollow-sandbox . -f values.yaml -f values-nats.yaml
 	kubectl get pod
 	./scripts/nats-bootstrap/boostrap.sh
 
 ## upgrade helm chart for the sandbox environment
-local-devel-upgrade: kubectl-ctx-kind
-	helm upgrade hollow-sandbox . -f values.yaml
+upgrade: kubectl-ctx-kind
+	helm upgrade hollow-sandbox . -f values.yaml -f values-nats.yaml
 
 ## uninstall helm chart
-uninstall-local-devel: kubectl-ctx-kind
+clean: kubectl-ctx-kind
 	helm uninstall hollow-sandbox
+	rm values-nats.yaml
+	# incase the crdb pvc is stuck in terminating
+	# kubectl patch pvc db -p '{"metadata":{"finalizers":null}}'
 
 ## port forward condition orchestrator API  (runs in foreground)
 port-forward-conditionorc-api: kubectl-ctx-kind

README.md

@@ -19,29 +19,26 @@ To understand more about the firmware install and how these services interact, c
 - Setup a local KIND cluster with a registry using the script here: https://kind.sigs.k8s.io/docs/user/local-registry/
 - Export `KUBECONFIG=~/.kube/config_kind`
 - Docker images for Serverservice, Conditionorc, Alloy
+- Install [mctl](https://github.com/metal-toolbox/mctl#getting-started) and use the configuration from [here](https://github.com/metal-toolbox/sandbox/tree/main/scripts/mctl)
 
 ### 1. Build docker images and push to local registry
 
-Clone the serverservice repository and build the docker image,
+Clone each of the repositories and run `make push-image-devel`
 
-```sh
-export GIT_TAG="localhost:5001/serverservice:latest" && \
-    GOOS=linux GOARCH=amd64 go build -o serverservice && \
-    docker build -t "${GIT_TAG}" -f Dockerfile . && \
-    docker push localhost:5001/serverservice:latest && kind load docker-image "${GIT_TAG}"
-```
+ - [Serverservice](https://github.com/metal-toolbox/hollow-serverservice)
+ - [Conditionorc](https://github.com/metal-toolbox/conditionorc)
+ - [Alloy](https://github.com/metal-toolbox/alloy)
+ - [Flasher](https://github.com/metal-toolbox/flasher/)
+
+This will build and push the container images to the local container registry.
 
-Clone the Alloy, Conditionorc, Flasher repositories and build each of the container images using,
-```sh
-make push-image-devel
-```
 
 ### 2. Deploy helm chart
 
 Deploys the helm chart and bootstrap the NATS Jetstream, K/V store.
 
 ```sh
-make local-devel
+make install
 ```
 
 ### 3. Import a server
@@ -63,7 +60,21 @@ go run scripts/serverservice/add-server.go \
         -bmc-pass foo
 ```
 
-### 4. Import firmware definitions (optional)
+### 4. Collect information on the server
+
+This collects the current firmware, health information and BIOS configuration data for a server.
+
+```sh
+mctl collect inventory --server edeff024-f62a-4288-8730-3fab8cceab78
+```
+
+Inventory collection status can be checked with,
+
+```sh
+mctl collect status --server edeff024-f62a-4288-8730-3fab8cceab78
+```
+
+### 5. Import firmware definitions (optional)
 
 Note: replace `ARTIFACTS_ENDPOINT` in [firmwares.json](./scripts/mctl/firmwares.json) with endpoint serving the firmware files.
 
@@ -73,7 +84,7 @@ Import firmware defs from sample file using `mctl`.
 mctl create  firmware --from-file ./scripts/mctl/firmwares.json
 ```
 
-### 5. Create a firmware set (optional)
+### 6. Create a firmware set (optional)
 
 List the firmware using `mctl list firmware` and create a set that can be applied to a server.
 
@@ -83,7 +94,7 @@ mctl create firmware-set --firmware-uuids 5e574c96-6ba4-4078-9650-c52a64cc8cba,a
                          --name r6515
 ```
 
-### 4. Set a `firmwareInstall` condition on a server (optional)
+### 7. Set a `firmwareInstall` condition on a server (optional)
 
 With the server added, you can now get flasher to set a `firmwareInstall` condition,
 
@@ -106,13 +117,13 @@ mctl install status --server edeff024-f62a-4288-8730-3fab8cceab78
 To upgrade the helm install after changes to the templates,
 
 ```
-make local-devel-upgrade
+make upgrade
 ```
 
 To uninstall the helm chart
 
 ```
-make uninstall-local-devel
+make clean
 ```
 
 ## NATs Jetstream setup
@@ -153,13 +164,15 @@ Usage:
   make <target>
 
 Targets:
-  local-devel          install helm chart for the sandbox env
-  local-devel-upgrade  upgrade helm chart for the sandbox environment
+  install              install helm chart for the sandbox env
+  upgrade              upgrade helm chart for the sandbox environment
+  clean                uninstall helm chart
   port-forward-conditionorc-api port forward condition orchestrator API  (runs in foreground)
   port-forward-alloy-pprof port forward condition Alloy pprof endpoint  (runs in foreground)
   port-forward-hss     port forward hollow server service port (runs in foreground)
   port-forward-crdb    port forward crdb service port (runs in foreground)
   port-forward-chaos-dash port forward chaos-mesh dashboard (runs in foreground)
+  port-forward-jaeger-dash port forward jaeger frontend
   psql-crdb            connect to crdb with psql (requires port-forward-crdb)
   clean-nats           purge nats app and storage pvcs
   kubectl-ctx-kind     set kube ctx to kind cluster

scripts/mctl/config.yml

@@ -0,0 +1,7 @@
+---
+serverservice_api:
+  endpoint: http://localhost:8000
+  disable: true
+conditions_api:
+  endpoint: http://localhost:9001
+  disable: true

scripts/nats-bootstrap/boostrap.sh

@@ -16,9 +16,10 @@ source scripts/nats-bootstrap/functions.sh
 clean_natsbox
 clean_natsserver
 init_natsaccounts
-update_valuesyaml
+update_values_nats_yaml
 init_natsserver
 push_natsaccounts
 push_controller_secrets
+reload_controller_deployments
 push_serverservice_secrets
 backup_accounts

scripts/nats-bootstrap/functions.sh

@@ -12,11 +12,15 @@ function clean_natsserver() {
 }
 
 function init_natsserver() {
-	make local-devel-upgrade
+	kubectl get pods
+	make upgrade
+	kubectl get pods
 }
 
 function clean_natsbox() {
+	set -e
 	kubectl exec -ti deployments/nats-box -- rm -rf /root/*.* /nsc
+	set +e
 }
 
 function init_natsaccounts() {
@@ -106,10 +110,12 @@ nsc edit signing-key -a controllers --sk ${SK_A} \
 	--allow-pubsub '$JS.API.CONSUMER.DELETE.controllers.>' \
 	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_active-controllers' \
 	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_firmwareInstall' \
-	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_active-controllers.>' \
 	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_firmwareInstall.>' \
-	--allow-pubsub '$JS.API.CONSUMER.DELETE.KV_active-controllers.>' \
 	--allow-pubsub '$JS.API.CONSUMER.DELETE.KV_firmwareInstall.>' \
+	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_inventory' \
+	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_inventory.>' \
+	--allow-pubsub '$JS.API.CONSUMER.CREATE.KV_active-controllers.>' \
+	--allow-pubsub '$JS.API.CONSUMER.DELETE.KV_active-controllers.>' \
     --allow-pubsub '$JS.API.STREAM.INFO.KV_active-controllers' \
     --allow-pubsub '$JS.API.STREAM.INFO.KV_active-controllers.>' \
     --allow-pubsub '$JS.API.STREAM.CREATE.KV_active-controllers' \
@@ -118,6 +124,10 @@ nsc edit signing-key -a controllers --sk ${SK_A} \
     --allow-pubsub '$JS.API.STREAM.INFO.KV_firmwareInstall.>' \
     --allow-pubsub '$JS.API.STREAM.CREATE.KV_firmwareInstall' \
     --allow-pubsub '$JS.API.STREAM.CREATE.KV_firmwareInstall.>' \
+    --allow-pubsub '$JS.API.STREAM.INFO.KV_inventory' \
+    --allow-pubsub '$JS.API.STREAM.INFO.KV_inventory.>' \
+    --allow-pubsub '$JS.API.STREAM.CREATE.KV_inventory' \
+    --allow-pubsub '$JS.API.STREAM.CREATE.KV_inventory.>' \
     --allow-pubsub '$JS.ACK.controllers.>' \
 	--allow-sub 'com.hollow.sh.serverservice.events.>' \
 	--allow-pubsub 'com.hollow.sh.controllers.>' \
@@ -134,7 +144,6 @@ nsc add user -a controllers --name alloy -K controllers
 # create flasher user, with the controllers role
 nsc add user -a controllers --name flasher -K controllers
 
-
 EOF
 }
 
@@ -149,14 +158,18 @@ function push_natsaccounts() {
 		sleep 5
 	done
 
-	sleep 20
+	while ! kubectl exec -ti deployments/nats-box -- ping nats -W 1 -c 1; do
+		echo "waiting for nats service to be accessible on k8s.."
+		sleep 5
+	done
+
 	kuexec "nsc push --system-account SYS -u nats://nats:4222 -A"
 }
 
-function update_valuesyaml() {
+function update_values_nats_yaml() {
 	set -x
-	f=values.yaml.bk
-	cp values.yaml $f
+	f=values-nats.yaml.bk
+	cp values-nats.yaml $f
 
 	CURRENT_OPKEY=$(awk '/operator: /{print $2}' $f)
 	CURRENT_SYSKEY=$(awk '/systemAccount: /{print $2}' $f)
@@ -174,7 +187,7 @@ function update_valuesyaml() {
 	sed -ie 's/systemAccount: '${CURRENT_SYSKEY}'/systemAccount: '${SYSKEY}'/' $f
 	sed -ie 's/'${CURRENT_SYSKEY}': '${CURRENT_SYSPRELOADKEY}'/'${SYSKEY}': '${SYSPRELOADKEY}'/' $f
 
-	mv $f values.yaml
+	mv $f values-nats.yaml
 }
 
 function push_controller_secrets() {
@@ -209,6 +222,12 @@ type: Opaque" >/tmp/kind_serverservice_secret.yaml
 	kubectl apply -f /tmp/kind_serverservice_secret.yaml
 }
 
+function reload_controller_deployments() {
+	echo "restarting controller deployments for NATSs changes to take effect..."
+	kubectl delete deployments.apps flasher alloy conditionorc
+	make upgrade
+}
+
 function backup_accounts() {
 	kuexec "cd / && tar -czf nats-accounts.tar.gz /root/nsc /nsc"
 	kubectl cp $(kubectl get pods | awk '/nats-box/{print $1}'):/nats-accounts.tar.gz ./scripts/nats-bootstrap/nats-accounts.tar.gz

scripts/nats-bootstrap/values-nats.yaml.tmpl

@@ -0,0 +1,42 @@
+#
+# https://github.com/nats-io/k8s/tree/main/helm/charts/nats#using-nats-chart-as-a-dependency
+nats:
+  fullnameOverride: nats
+  nats:
+    image:
+      repository: nats
+      tag: "2.9.16-alpine"
+      pullPolicy: IfNotPresent
+    logging:
+      debug: true
+      trace: true
+      logtime: true
+    jetstream:
+      enabled: true
+      fileStorage:
+        size: 1Gi
+  auth:
+    enabled: true
+    timeout: "5s"
+    resolver:
+      type: full
+      # Operator named KO
+      operator: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUWEFVWUFLV1E3WklGWVNWSkRPTTVaNDQ0NFdKQlpCMkxJNEYzTFFGSElPRlhPMkxXUFNBIiwiaWF0IjoxNjkzODAyODg1LCJpc3MiOiJPREJNRUpMREJWUTVMTUVGQ1QzWEVLTUVSUkdJVVlCWlo3VFA3N0lWSjNSMzMyTDc0SUVVR05YRyIsIm5hbWUiOiJLTyIsInN1YiI6Ik9EQk1FSkxEQlZRNUxNRUZDVDNYRUtNRVJSR0lVWUJaWjdUUDc3SVZKM1IzMzJMNzRJRVVHTlhHIiwibmF0cyI6eyJhY2NvdW50X3NlcnZlcl91cmwiOiJuYXRzOi8vbmF0czo0MjIyIiwib3BlcmF0b3Jfc2VydmljZV91cmxzIjpbIm5hdHM6Ly9uYXRzOjQyMjIiXSwidHlwZSI6Im9wZXJhdG9yIiwidmVyc2lvbiI6Mn19.Grx7XJ1viEPEKQuKj26osCE1fy5DbWFIG5g3VD7s55db-ZRQq1lb4xdMLwvbsnhtrI0D0o24WgVsgJ80DXRUAA
+      # System account named SYS
+      systemAccount: ABUHJSADT3SLYFXMTRQ5YVBZP2OTH5UF7LZ6MXWH4FTTNLN7ABKCLROW
+      store:
+        dir: "/accounts/jwt"
+        size: "50Mi"
+      resolverPreload:
+        ABUHJSADT3SLYFXMTRQ5YVBZP2OTH5UF7LZ6MXWH4FTTNLN7ABKCLROW: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI3TFRLMzU2VFhMQTc0UlE1NlhIN1lBSUI1SkZFWEJWNExOVk1YVkRGVElLNUxQQU43UUtRIiwiaWF0IjoxNjkzODAyODg1LCJpc3MiOiJPREJNRUpMREJWUTVMTUVGQ1QzWEVLTUVSUkdJVVlCWlo3VFA3N0lWSjNSMzMyTDc0SUVVR05YRyIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBQlVISlNBRFQzU0xZRlhNVFJRNVlWQlpQMk9USDVVRjdMWjZNWFdINEZUVE5MTjdBQktDTFJPVyIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7ImF1dGhfdXNlcnMiOm51bGx9LCJ0eXBlIjoiYWNjb3VudCIsInZlcnNpb24iOjJ9fQ.ih3oQvM0KNoA5lcKTCbDZj-sbjNoThv-nTRkL_t5JVot_BmvxK8Rfi6ppIJrA_8k6Oh1m_x799Sy5kFJnU16DA
+  cluster:
+    enabled: false
+    noAdvertise: true
+  natsbox:
+    enabled: true
+    #image: natsio/nats-box:0.13.4
+    image:
+      repository: natsio/nats-box
+      tag: 0.13.8
+      pullPolicy: IfNotPresent
+

templates/alloy-configmap.yaml

@@ -11,30 +11,16 @@ data:
     collector_outofband:
       concurrency: 5
     store_kind: serverservice
-    serverservice:
-      endpoint: http://serverservice:8000
-      disable_oauth: true
-      facility_code: dc13
     events_broker_kind: nats
     nats:
-      url: nats://nats:4222
       app_name: alloy
-      publisher_subject_prefix: com.hollow.sh.controllers.responses
+      # TODO: remove deprecated stream_urn_ns field
       stream_urn_ns: hollow-controllers
-      #subscribe_subjects:
-      #  - com.hollow.sh.controllers.commands.>
-      connect_timeout: 30s
-      #creds_file:
       consumer:
         pull: true
-        name: alloy
         ack_wait: 5m
         max_ack_pending: 10
         queue_group: alloy
-        subscribe_subjects:
-          - com.hollow.sh.controllers.commands.servers.inventory.>
-        # filter_subject required to create consumer, must be unique on stream.
-        filter_subject: com.hollow.sh.controllers.commands.servers.inventory.>
       stream:
         name: controllers
         subjects:
@@ -43,3 +29,5 @@ data:
         acknowledgements: true
         duplicate_window: 5m
         retention: workQueue
+      kv:
+        replica_count: 1