Add --health-log-destination flag

Signed-off-by: Jan Rodák <[email protected]>

cmd/podman/common/create.go

@@ -184,6 +184,14 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
 		)
 		_ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone)
 
+		healthLogDestinationFlagName := "health-log-destination"
+		createFlags.StringVar(
+			&cf.HealthLogDestination,
+			healthLogDestinationFlagName, "",
+			"set the destination of the HealthCheck log. Directory path or journal (none use container state file)",
+		)
+		_ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone)
+
 		healthMaxLogCountFlagName := "health-max-log-count"
 		createFlags.UintVar(
 			&cf.HealthMaxLogCount,

docs/source/markdown/options/health-log-destination.md

@@ -0,0 +1,11 @@
+####> This option file is used in:
+####>   podman create, run
+####> If file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--health-log-destination**=*directory_path*
+
+Set the destination of the HealthCheck log. Directory path or journal (none use container state file)
+
+* `none`: (default) HealthCheck logs are stored in overlay containers. (For example: `./run/containers/storage/overlay-containers/<container-ID>/healthcheck.log`)
+* `directory`: creates a log file named `<container-ID>-healthcheck.log` with HealthCheck logs in the specified directory.
+* `journal`: The log will be written to the journal. A log with HealthCheck logs will also be written to the default path.

docs/source/markdown/podman-create.1.md.in

@@ -169,6 +169,8 @@ See [**Environment**](#environment) note below for precedence and examples.
 
 @@option health-interval
 
+@@option health-log-destination
+
 @@option health-max-log-count
 
 @@option health-max-log-size

docs/source/markdown/podman-run.1.md.in

@@ -203,6 +203,8 @@ See [**Environment**](#environment) note below for precedence and examples.
 
 @@option health-interval
 
+@@option health-log-destination
+
 @@option health-max-log-count
 
 @@option health-max-log-size

docs/source/markdown/podman-systemd.unit.5.md

@@ -279,6 +279,7 @@ Valid options for `[Container]` are listed below:
 | GroupAdd=keep-groups                 | --group-add=keep-groups                              |
 | HealthCmd=/usr/bin/command           | --health-cmd=/usr/bin/command                        |
 | HealthInterval=2m                    | --health-interval=2m                                 |
+| HealthLogDestination=/foo/log        | --health-log-destination=/foo/log                    |
 | HealthMaxLogCount=5                  | --health-max-log-count=5                             |
 | HealthMaxLogSize=500                 | --health-max-log-size=500                            |
 | HealthOnFailure=kill                 | --health-on-failure=kill                             |
@@ -517,6 +518,15 @@ Equivalent to the Podman `--health-cmd` option.
 Set an interval for the healthchecks. An interval of disable results in no automatic timer setup.
 Equivalent to the Podman `--health-interval` option.
 
+### `HealthLogDestination=`
+
+Set the destination of the HealthCheck log. Directory path or journal (none use container state file)
+Equivalent to the Podman `--health-log-destination` option.
+
+* `none`: (default) HealthCheck logs are stored in overlay containers. (For example: `./run/containers/storage/overlay-containers/<container-ID>/healthcheck.log`)
+* `directory`: creates a log file named `<container-ID>-healthcheck.log` with HealthCheck logs in the specified directory.
+* `journal`: The log will be written to the journal. A log with HealthCheck logs will also be written to the default path.
+
 ### `HealthMaxLogCount=`
 
 Set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)

libpod/container_config.go

@@ -413,6 +413,8 @@ type ContainerMiscConfig struct {
 	HealthCheckConfig *manifest.Schema2HealthConfig `json:"healthcheck"`
 	// HealthCheckOnFailureAction defines an action to take once the container turns unhealthy.
 	HealthCheckOnFailureAction define.HealthCheckOnFailureAction `json:"healthcheck_on_failure_action"`
+	// HealthLogDestination defines the destination where the log is stored
+	HealthLogDestination string `json:"healthLogDestination"`
 	// HealthMaxLogCount is maximum number of attempts in the HealthCheck log file.
 	// ('0' value means an infinite number of attempts in the log file)
 	HealthMaxLogCount uint `json:"healthMaxLogCount"`

libpod/container_inspect.go

@@ -426,6 +426,8 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp
 
 	ctrConfig.HealthcheckOnFailureAction = c.config.HealthCheckOnFailureAction.String()
 
+	ctrConfig.HealthLogDestination = c.config.HealthLogDestination
+
 	ctrConfig.HealthMaxLogCount = c.config.HealthMaxLogCount
 
 	ctrConfig.HealthMaxLogSize = c.config.HealthMaxLogSize

libpod/container_internal.go

@@ -1123,7 +1123,12 @@ func (c *Container) init(ctx context.Context, retainRetries bool) error {
 	// bugzilla.redhat.com/show_bug.cgi?id=2144754:
 	// In case of a restart, make sure to remove the healthcheck log to
 	// have a clean state.
-	if path := c.healthCheckLogPath(); path != "" {
+	path, err := c.healthCheckLogPath()
+	if err != nil {
+		logrus.Error(err)
+	}
+
+	if path != "" {
 		if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
 			logrus.Error(err)
 		}

libpod/define/container_inspect.go

@@ -61,11 +61,13 @@ type InspectContainerConfig struct {
 	Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"`
 	// HealthcheckOnFailureAction defines an action to take once the container turns unhealthy.
 	HealthcheckOnFailureAction string `json:"HealthcheckOnFailureAction,omitempty"`
+	// HealthLogDestination defines the destination where the log is stored
+	HealthLogDestination string `json:"HealthLogDestination,omitempty"`
 	// HealthMaxLogCount is maximum number of attempts in the HealthCheck log file.
 	// ('0' value means an infinite number of attempts in the log file)
 	HealthMaxLogCount uint `json:"HealthcheckMaxLogCount,omitempty"`
-	// HealthMaxLogSize is the maximum length in characters of log
-	// in the healthcheck history ("0" value means an infinite log length)
+	// HealthMaxLogSize is the maximum length in characters of stored HealthCheck log
+	// ("0" value means an infinite log length)
 	HealthMaxLogSize uint `json:"HealthcheckMaxLogSize,omitempty"`
 	// CreateCommand is the full command plus arguments of the process the
 	// container has been created with.

libpod/events/config.go

@@ -41,6 +41,8 @@ type Event struct {
 	Type Type
 	// Health status of the current container
 	HealthStatus string `json:"health_status,omitempty"`
+	// Healthcheck log of the current container
+	HealthLog string `json:"health_log,omitempty"`
 	// Error code for certain events involving errors.
 	Error string `json:"error,omitempty"`
 
@@ -150,6 +152,8 @@ const (
 	Export Status = "export"
 	// HealthStatus ...
 	HealthStatus Status = "health_status"
+	// HealthLog ...
+	HealthLog Status = "health_log"
 	// History ...
 	History Status = "history"
 	// Import ...

libpod/events/journal_linux.go

@@ -67,6 +67,10 @@ func (e EventJournalD) Write(ee Event) error {
 		}
 		m["PODMAN_HEALTH_STATUS"] = ee.HealthStatus
 
+		if ee.Status == HealthLog {
+			m["PODMAN_HEALTH_LOG"] = ee.HealthLog
+		}
+
 		if len(ee.Details.ContainerInspectData) > 0 {
 			m["PODMAN_CONTAINER_INSPECT_DATA"] = ee.Details.ContainerInspectData
 		}

libpod/healthcheck.go

@@ -148,7 +148,18 @@ func (c *Container) runHealthCheck(ctx context.Context, isStartup bool) (define.
 	hcl := newHealthCheckLog(timeStart, timeEnd, returnCode, eventLog)
 	logStatus, err := c.updateHealthCheckLog(hcl, inStartPeriod, isStartup)
 	if err != nil {
-		return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", c.healthCheckLogPath(), c.ID(), err)
+		path, err := c.healthCheckLogPath()
+		if err != nil {
+			return hcResult, "", err
+		}
+		return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", path, c.ID(), err)
+	}
+
+	if c.config.HealthLogDestination == "journal" {
+		err := c.sendHealthCheckLogToJournal(logStatus, eventLog)
+		if err != nil {
+			logrus.Error(err)
+		}
 	}
 
 	// Write HC event with appropriate status as the last thing before we
@@ -337,11 +348,7 @@ func (c *Container) updateHealthStatus(status string) error {
 		return err
 	}
 	healthCheck.Status = status
-	newResults, err := json.Marshal(healthCheck)
-	if err != nil {
-		return fmt.Errorf("unable to marshall healthchecks for writing status: %w", err)
-	}
-	return os.WriteFile(c.healthCheckLogPath(), newResults, 0700)
+	return c.storeHealthCheckResults(healthCheck)
 }
 
 // isUnhealthy returns true if the current health check status is unhealthy.
@@ -393,16 +400,41 @@ func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPerio
 	if c.config.HealthMaxLogCount != 0 && len(healthCheck.Log) > int(c.config.HealthMaxLogCount) {
 		healthCheck.Log = healthCheck.Log[1:]
 	}
-	newResults, err := json.Marshal(healthCheck)
+	return healthCheck.Status, c.storeHealthCheckResults(healthCheck)
+}
+
+func (c *Container) storeHealthCheckResults(result define.HealthCheckResults) error {
+	newResults, err := json.Marshal(result)
+	if err != nil {
+		return fmt.Errorf("unable to marshall healthchecks for writing: %w", err)
+	}
+	path, err := c.healthCheckLogPath()
 	if err != nil {
-		return "", fmt.Errorf("unable to marshall healthchecks for writing: %w", err)
+		return err
 	}
-	return healthCheck.Status, os.WriteFile(c.healthCheckLogPath(), newResults, 0700)
+	return os.WriteFile(path, newResults, 0700)
 }
 
 // HealthCheckLogPath returns the path for where the health check log is
-func (c *Container) healthCheckLogPath() string {
-	return filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log")
+func (c *Container) healthCheckLogPath() (string, error) {
+	if c.config.HealthLogDestination != "" && c.config.HealthLogDestination != "journal" {
+		fileInfo, err := os.Stat(c.config.HealthLogDestination)
+		if err != nil {
+			return "", err
+		}
+		mode := fileInfo.Mode()
+		if !mode.IsDir() {
+			return "", fmt.Errorf("Log '%s' destination must be directory", c.config.HealthLogDestination)
+		}
+
+		logFileName := c.ID() + "-healthcheck.log"
+		absPath, err := filepath.Abs(c.config.HealthLogDestination)
+		if err != nil {
+			return "", err
+		}
+		return filepath.Join(absPath, logFileName), nil
+	}
+	return filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log"), nil
 }
 
 // getHealthCheckLog returns HealthCheck results by reading the container's
@@ -411,7 +443,11 @@ func (c *Container) healthCheckLogPath() string {
 // The caller should lock the container before this function is called.
 func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) {
 	var healthCheck define.HealthCheckResults
-	b, err := os.ReadFile(c.healthCheckLogPath())
+	path, err := c.healthCheckLogPath()
+	if err != nil {
+		return healthCheck, err
+	}
+	b, err := os.ReadFile(path)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			// If the file does not exists just return empty healthcheck and no error.
@@ -420,7 +456,7 @@ func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) {
 		return healthCheck, fmt.Errorf("failed to read health check log file: %w", err)
 	}
 	if err := json.Unmarshal(b, &healthCheck); err != nil {
-		return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", c.healthCheckLogPath(), err)
+		return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", path, err)
 	}
 	return healthCheck, nil
 }

libpod/healthcheck_linux.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	systemdCommon "github.com/containers/common/pkg/systemd"
+	"github.com/containers/podman/v5/libpod/events"
 	"github.com/containers/podman/v5/pkg/errorhandling"
 	"github.com/containers/podman/v5/pkg/rootless"
 	"github.com/containers/podman/v5/pkg/systemd"
@@ -186,3 +187,21 @@ func (c *Container) hcUnitName(isStartup, bare bool) string {
 	}
 	return unitName
 }
+
+func (c *Container) sendHealthCheckLogToJournal(status string, log string) error {
+	e := events.NewEvent(events.HealthLog)
+	e.ID = c.ID()
+	e.Name = c.Name()
+	e.Image = c.config.RootfsImageName
+	e.Type = events.Container
+	e.HealthStatus = status
+	e.HealthLog = log
+
+	e.Details = events.Details{
+		PodID:      c.PodID(),
+		Attributes: c.Labels(),
+	}
+
+	journal := events.EventJournalD{}
+	return journal.Write(e)
+}

libpod/healthcheck_nosystemd_linux.go

@@ -21,3 +21,7 @@ func (c *Container) startTimer(isStartup bool) error {
 func (c *Container) removeTransientFiles(ctx context.Context, isStartup bool, unitName string) error {
 	return nil
 }
+
+func (c *Container) sendHealthCheckLogToJournal(status string, log string) error {
+	return nil
+}

libpod/healthcheck_unsupported.go

@@ -21,3 +21,7 @@ func (c *Container) startTimer(isStartup bool) error {
 func (c *Container) removeTransientFiles(ctx context.Context, isStartup bool, unitName string) error {
 	return nil
 }
+
+func (c *Container) sendHealthCheckLogToJournal(status string, log string) error {
+	return nil
+}

libpod/options.go

@@ -1500,6 +1500,17 @@ func WithHealthCheck(healthCheck *manifest.Schema2HealthConfig) CtrCreateOption
 	}
 }
 
+// WithHealthCheckLogDestination adds the healthLogDestination to the container config
+func WithHealthCheckLogDestination(destination string) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.HealthLogDestination = destination
+		return nil
+	}
+}
+
 // WithHealthCheckMaxLogCount adds the healthMaxLogCount to the container config
 func WithHealthCheckMaxLogCount(maxLogCount uint) CtrCreateOption {
 	return func(ctr *Container) error {