feat: Add ability to enable Longhorn V2 Data Engine (backport #55)

Dockerfile.dapper

@@ -20,7 +20,7 @@ RUN  go install k8s.io/code-generator/cmd/[email protected]
 
 ENV DAPPER_ENV REPO TAG
 ENV DAPPER_SOURCE /go/src/github.com/harvester/node-manager/
-ENV DAPPER_OUTPUT ./bin ./manifests
+ENV DAPPER_OUTPUT ./bin ./manifests ./pkg
 ENV DAPPER_DOCKER_SOCKET true
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}

go.mod

@@ -25,7 +25,7 @@ replace (
 require (
 	github.com/ehazlett/simplelog v0.0.0-20200226020431-d374894e92a4
 	github.com/godbus/dbus/v5 v5.1.0
-	github.com/harvester/go-common v0.0.0-20231214093547-3e3f7fdd879a
+	github.com/harvester/go-common v0.0.0-20240822134235-198df47889ca
 	github.com/mudler/yip v1.1.0
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.16.0
@@ -36,7 +36,6 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.9.0
 	github.com/twpayne/go-vfs v1.7.2
-	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 	k8s.io/api v0.30.3
 	k8s.io/apimachinery v0.30.3
@@ -48,7 +47,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/harvester/webhook v0.1.4
 	github.com/urfave/cli/v2 v2.3.0
-	golang.org/x/sys v0.19.0
+	golang.org/x/sys v0.20.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -94,7 +93,7 @@ require (
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rancher/dynamiclistener v0.3.5 // indirect
 	github.com/rancher/wrangler v1.1.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect

go.sum

@@ -177,8 +177,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/harvester/go-common v0.0.0-20231214093547-3e3f7fdd879a h1:4TQVNT8HzY+xh7hXlclfxt/kaBoz1nMvOmmXIX2JTmg=
-github.com/harvester/go-common v0.0.0-20231214093547-3e3f7fdd879a/go.mod h1:xCmR8h47sSSz+vqDjGkSh5kMr3WqXZKruZyir/i166o=
+github.com/harvester/go-common v0.0.0-20240822134235-198df47889ca h1:gUgeAQ9NSFEBqVkKJxTmbhbob/6CWu/iDlD4V40IpdU=
+github.com/harvester/go-common v0.0.0-20240822134235-198df47889ca/go.mod h1:UNh9xAFqna6F2Fk4m4TTNWV0xaORlFwH7B+DepHT/gI=
 github.com/harvester/webhook v0.1.4 h1:6g5MkYXlGm0wABQ/Dm8g5sM7WAtPq2aiWOH9wmf2vUQ=
 github.com/harvester/webhook v0.1.4/go.mod h1:vfRPB26WHSPxMF/ONpUVzaEaewTUxpP9qAqu1ZyonR0=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -257,8 +257,8 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rancher/dynamiclistener v0.3.5 h1:5TaIHvkDGmZKvc96Huur16zfTKOiLhDtK4S+WV0JA6A=
 github.com/rancher/dynamiclistener v0.3.5/go.mod h1:dW/YF6/m2+uEyJ5VtEcd9THxda599HP6N9dSXk81+k0=
 github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1 h1:vv1jDlYbd4KhGbPNxmjs8CYgEHUrQm2bMtmULfXJ6iw=
@@ -347,8 +347,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
-golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -477,8 +475,8 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=

manifests/crds/node.harvesterhci.io_nodeconfigs.yaml

@@ -39,6 +39,11 @@ spec:
             type: object
           spec:
             properties:
+              longhornConfig:
+                properties:
+                  enableV2DataEngine:
+                    type: boolean
+                type: object
               ntpConfigs:
                 properties:
                   ntpServers:

manifests/daemonset.yaml

@@ -50,9 +50,12 @@ spec:
               cpu: 10m
               memory: 64Mi
           volumeMounts:
-            - mountPath: /sys/kernel/mm/ksm
-              name: ksm
+            - mountPath: /sys/kernel/mm
+              name: mm
               readOnly: false
+            - mountPath: /lib/modules
+              name: modules
+              readOnly: true
             - mountPath: /host/proc
               name: proc
               readOnly: true
@@ -64,9 +67,12 @@ spec:
             - mountPath: /host/oem
               name: host-oem
       volumes:
-        - name: ksm
+        - name: mm
+          hostPath:
+            path: /sys/kernel/mm
+        - name: modules
           hostPath:
-            path: /sys/kernel/mm/ksm
+            path: /lib/modules
         - name: proc
           hostPath:
             path: /proc
@@ -81,4 +87,4 @@ spec:
         - name: host-oem
           hostPath:
             path: /oem
-            type: ""
+            type: ""

package/Dockerfile

@@ -2,7 +2,9 @@
 
 FROM registry.suse.com/bci/bci-base:15.5
 
+# kmod -> for `modprobe` command
 RUN zypper -n rm container-suseconnect && \
+    zypper -n install kmod && \
     zypper -n clean -a && rm -rf /tmp/* /var/tmp/* /usr/share/doc/packages/*
 
 ARG TARGETPLATFORM

pkg/apis/node.harvesterhci.io/v1beta1/nodeconfig.go

@@ -22,12 +22,18 @@ type NodeConfig struct {
 }
 
 type NodeConfigSpec struct {
-	NTPConfig *NTPConfig `json:"ntpConfigs,omitempty"`
+	NTPConfig      *NTPConfig      `json:"ntpConfigs,omitempty"`
+	LonghornConfig *LonghornConfig `json:"longhornConfig,omitempty"`
 }
+
 type NTPConfig struct {
 	NTPServers string `json:"ntpServers"`
 }
 
+type LonghornConfig struct {
+	EnableV2DataEngine bool `json:"enableV2DataEngine,omitempty"`
+}
+
 type NodeConfigStatus struct {
 	NTPConditions []ConfigStatus `json:"ntpConditions,omitempty"`
 }

pkg/controller/nodeconfig/config/common.go

@@ -1,11 +1,28 @@
 package config
 
+import (
+	"fmt"
+	"os"
+	"slices"
+
+	"github.com/harvester/go-common/files"
+	"github.com/harvester/node-manager/pkg/utils"
+	"github.com/mudler/yip/pkg/schema"
+	"github.com/sirupsen/logrus"
+)
+
 const (
 	// we use `99_settings.yaml` because it needs to be run after `90_custom.yaml`
 	// with elemental works, the later change would override the previous one
+	yipStageInitramfs = "initramfs"
+)
+
+// The following would ordinarily be const, but we need to override them in unit tests
+
+var (
+	oemPath                   = "/host/oem/"
 	settingsOEMPath           = "/host/oem/99_settings.yaml"
 	settingsOEMPathBackupPath = "/host/oem/99_settings.yaml.bak"
-	yipStageInitramfs         = "initramfs"
 )
 
 type NTPConfigTemplate struct {
@@ -20,3 +37,89 @@ func generateNTPConfigData() string {
 {{- end }}
 `
 }
+
+func UpdatePersistentOEMSettings(stage schema.Stage) error {
+	_, err := os.Stat(settingsOEMPath)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("stat %s failed: %v", settingsOEMPath, err)
+	}
+
+	settings := utils.GenerateOEMTemplate()
+	doBackup := true
+	if os.IsNotExist(err) {
+		// New file, we can just set the stages to whatever was passed in.
+		settings.Stages = make(map[string][]schema.Stage)
+		settings.Stages[yipStageInitramfs] = []schema.Stage{stage}
+		doBackup = false
+	} else {
+		// Existing file, we need to load it...
+		err = utils.LoadYipConfigToTarget(settingsOEMPath, settings)
+		if err != nil {
+			return fmt.Errorf("load %s to YIP format failed: %v", settingsOEMPath, err)
+		}
+		logrus.Debugf("Loaded settings from file %s, content: %+v", settingsOEMPath, settings)
+		// ...then merge the new stage into whatever stages are already present,
+		// either overwriting or appending as necessary.
+		existingStage := slices.IndexFunc(settings.Stages[yipStageInitramfs], func(s schema.Stage) bool {
+			return s.Name == stage.Name
+		})
+		if existingStage == -1 {
+			settings.Stages[yipStageInitramfs] = append(settings.Stages[yipStageInitramfs], stage)
+		} else {
+			settings.Stages[yipStageInitramfs][existingStage] = stage
+		}
+	}
+
+	return writePersistentOEMSettings(settings, doBackup)
+}
+
+func RemovePersistentOEMSettings(stageName string) error {
+	yipConfig, err := utils.LoadYipConfig(settingsOEMPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("load %s failed: %v", settingsOEMPath, err)
+	}
+	logrus.Debugf("Loaded yipConfig: %+v, %p", yipConfig, yipConfig)
+
+	if _, found := yipConfig.Stages[yipStageInitramfs]; !found {
+		// this moment, we only have `initramfs` stage, so we could remove all OEM settings files.
+		logrus.Infof("No `initramfs` stage found, remove all OEM settings files.")
+		return files.RemoveFiles(settingsOEMPath)
+	}
+
+	pos := slices.IndexFunc(yipConfig.Stages[yipStageInitramfs], func(s schema.Stage) bool {
+		return s.Name == stageName
+	})
+
+	if pos >= 0 {
+		stages := yipConfig.Stages[yipStageInitramfs]
+		stages = append(stages[:pos], stages[pos+1:]...)
+		if len(stages) == 0 {
+			logrus.Infof("No other stages found, remove all OEM settings files.")
+			return files.RemoveFiles(settingsOEMPath)
+		}
+		yipConfig.Stages[yipStageInitramfs] = stages
+	}
+
+	// we still have other stages, so we need to backup/update OEM settings files
+	return writePersistentOEMSettings(yipConfig, true)
+}
+
+func writePersistentOEMSettings(yipConfig *schema.YipConfig, doBackup bool) error {
+	if doBackup {
+		if _, err := files.BackupFile(settingsOEMPath); err != nil {
+			return fmt.Errorf("backup %s failed: %v", settingsOEMPath, err)
+		}
+	}
+	logrus.Infof("Prepare to update new settings to persistent files: %+v", yipConfig)
+	tmpFileName, err := files.GenerateYAMLTempFileWithDir(yipConfig, "settings", oemPath)
+	if err != nil {
+		return fmt.Errorf("generate temp YAML file failed: %v", err)
+	}
+	if err = os.Rename(tmpFileName, settingsOEMPath); err != nil {
+		return fmt.Errorf("rename temp file to %s failed: %v", settingsOEMPath, err)
+	}
+	return nil
+}