From 91b3e96672f6f02567e43c392ffe425c88c6b06c Mon Sep 17 00:00:00 2001 From: satotake Date: Sun, 20 Jun 2021 09:03:46 +0000 Subject: [PATCH] Add --dry-run option to pull cmd Signed-off-by: GitHub --- cmd/compose/pull.go | 6 + local/e2e/compose/compose_dry_run_test.go | 51 ++++++ .../fixtures/dry-run-test/pull/compose.yaml | 14 ++ pkg/api/api.go | 3 + pkg/compose/build.go | 77 ++++++++- pkg/compose/images.go | 1 + pkg/compose/pull.go | 148 +++++++++++++++--- 7 files changed, 272 insertions(+), 28 deletions(-) create mode 100644 local/e2e/compose/compose_dry_run_test.go create mode 100644 local/e2e/compose/fixtures/dry-run-test/pull/compose.yaml diff --git a/cmd/compose/pull.go b/cmd/compose/pull.go index 6d3417ca2..6af92262c 100644 --- a/cmd/compose/pull.go +++ b/cmd/compose/pull.go @@ -36,6 +36,8 @@ type pullOptions struct { noParallel bool includeDeps bool ignorePullFailures bool + dryRun bool + format string } func pullCommand(p *projectOptions, backend api.Service) *cobra.Command { @@ -63,6 +65,8 @@ func pullCommand(p *projectOptions, backend api.Service) *cobra.Command { cmd.Flags().BoolVar(&opts.parallel, "no-parallel", true, "DEPRECATED disable parallel pulling.") flags.MarkHidden("no-parallel") //nolint:errcheck cmd.Flags().BoolVar(&opts.ignorePullFailures, "ignore-pull-failures", false, "Pull what it can and ignores images with pull failures") + cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Calc and print effects of execution of pull command without any actual effects") + cmd.Flags().StringVar(&opts.format, "format", "", `Format the output. Only compatible with "--dry-run". Values: [pretty | json]. (Default: pretty)`) return cmd } @@ -88,5 +92,7 @@ func runPull(ctx context.Context, backend api.Service, opts pullOptions, service return backend.Pull(ctx, project, api.PullOptions{ Quiet: opts.quiet, IgnoreFailures: opts.ignorePullFailures, + DryRun: opts.dryRun, + Format: opts.format, }) } diff --git a/local/e2e/compose/compose_dry_run_test.go b/local/e2e/compose/compose_dry_run_test.go new file mode 100644 index 000000000..223008690 --- /dev/null +++ b/local/e2e/compose/compose_dry_run_test.go @@ -0,0 +1,51 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package e2e + +import ( + "strings" + "testing" + + "gotest.tools/v3/assert" + + . "github.com/docker/compose-cli/utils/e2e" +) + +func TestComposePullDryRun(t *testing.T) { + c := NewParallelE2eCLI(t, binDir) + + t.Run("compose pull dry run", func(t *testing.T) { + // ensure storing alpine image and deleting hello-world + c.RunDockerCmd("pull", "alpine") + c.RunDockerCmd("rmi", "hello-world", "-f") + + res := c.RunDockerCmd("compose", "-f", "./fixtures/dry-run-test/pull/compose.yaml", "pull", "--dry-run") + lines := Lines(res.Stdout()) + for _, line := range lines { + if strings.Contains(line, "expected-skip") { + assert.Assert(t, strings.Contains(line, " skip ")) + } + if strings.Contains(line, "expected-fail") { + assert.Assert(t, strings.Contains(line, " fail ")) + } + if strings.Contains(line, "expected-fetch") { + assert.Assert(t, strings.Contains(line, " fetch ")) + } + } + }) + +} diff --git a/local/e2e/compose/fixtures/dry-run-test/pull/compose.yaml b/local/e2e/compose/fixtures/dry-run-test/pull/compose.yaml new file mode 100644 index 000000000..9f27fab9e --- /dev/null +++ b/local/e2e/compose/fixtures/dry-run-test/pull/compose.yaml @@ -0,0 +1,14 @@ +services: + expected-fetch: + image: hello-world + command: top + expected-skip-already-exist: + image: alpine + command: top + expected-fail: + # this image is not expected to be registered on any registries + image: expected-not-to-be-registered + command: top + expected-skip-build: + build: . + command: top \ No newline at end of file diff --git a/pkg/api/api.go b/pkg/api/api.go index 8fc5619de..877c85dec 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -175,6 +175,8 @@ type PushOptions struct { type PullOptions struct { Quiet bool IgnoreFailures bool + DryRun bool + Format string } // ImagesOptions group options of the Images API @@ -308,6 +310,7 @@ type ImageSummary struct { Repository string Tag string Size int64 + Digests []string } // ServiceStatus hold status about a service diff --git a/pkg/compose/build.go b/pkg/compose/build.go index 7d978ffde..7124ebedc 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "os" + "strings" + "sync" "github.com/compose-spec/compose-go/types" "github.com/containerd/containerd/platforms" @@ -29,10 +31,12 @@ import ( "github.com/docker/buildx/util/buildflags" xprogress "github.com/docker/buildx/util/progress" moby "github.com/docker/docker/api/types" + "github.com/docker/docker/registry" bclient "github.com/moby/buildkit/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" specs "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" "github.com/docker/compose-cli/pkg/api" "github.com/docker/compose-cli/pkg/progress" @@ -97,7 +101,7 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. } } - images, err := s.getLocalImagesDigests(ctx, project) + images, err := s.getLocalImagesIDs(ctx, project) if err != nil { return err } @@ -163,7 +167,19 @@ func (s *composeService) getBuildOptions(project *types.Project, images map[stri } -func (s *composeService) getLocalImagesDigests(ctx context.Context, project *types.Project) (map[string]string, error) { +func (s *composeService) getLocalImagesIDs(ctx context.Context, project *types.Project) (map[string]string, error) { + imgs, err := s.getLocalImageSummaries(ctx, project) + if err != nil { + return nil, err + } + images := map[string]string{} + for name, info := range imgs { + images[name] = info.ID + } + return images, nil +} + +func (s *composeService) getLocalImageSummaries(ctx context.Context, project *types.Project) (map[string]api.ImageSummary, error) { imageNames := []string{} for _, s := range project.Services { imgName := getImageName(s, project.Name) @@ -175,13 +191,64 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ if err != nil { return nil, err } - images := map[string]string{} - for name, info := range imgs { - images[name] = info.ID + return imgs, nil +} + +func (s *composeService) getLocalImagesDigests(ctx context.Context, project *types.Project) (map[string][]string, error) { + imgs, err := s.getLocalImageSummaries(ctx, project) + if err != nil { + return nil, err + } + images := map[string][]string{} + for name, summary := range imgs { + for _, d := range summary.Digests { + s := strings.Split(d, "@") + images[name] = append(images[name], s[len(s)-1]) + } } return images, nil } +func (s *composeService) getDistributionImagesDigests(ctx context.Context, project *types.Project) (map[string]string, error) { + images := map[string]string{} + + for _, service := range project.Services { + if service.Image == "" { + continue + } + images[service.Image] = "" + } + + info, err := s.apiClient.Info(ctx) + if err != nil { + return nil, err + } + + if info.IndexServerAddress == "" { + info.IndexServerAddress = registry.IndexServer + } + + l := sync.Mutex{} + eg, ctx := errgroup.WithContext(ctx) + for img := range images { + img := img + eg.Go(func() error { + registryAuth, err := getEncodedRegistryAuth(img, info, s.configFile) + if err != nil { + return err + } + inspect, _ := s.apiClient.DistributionInspect(ctx, img, registryAuth) + // Ignore error here. + // If you catch error here, all inspect requests will fail. + l.Lock() + images[img] = inspect.Descriptor.Digest.String() + l.Unlock() + return nil + }) + } + return images, eg.Wait() +} + func (s *composeService) doBuild(ctx context.Context, project *types.Project, opts map[string]build.Options, observedState Containers, mode string) (map[string]string, error) { info, err := s.apiClient.Info(ctx) if err != nil { diff --git a/pkg/compose/images.go b/pkg/compose/images.go index 92351570c..08b778ba3 100644 --- a/pkg/compose/images.go +++ b/pkg/compose/images.go @@ -105,6 +105,7 @@ func (s *composeService) getImages(ctx context.Context, images []string) (map[st Repository: repository, Tag: tag, Size: inspect.Size, + Digests: inspect.RepoDigests, } l.Unlock() return nil diff --git a/pkg/compose/pull.go b/pkg/compose/pull.go index 50327010b..4b82c34df 100644 --- a/pkg/compose/pull.go +++ b/pkg/compose/pull.go @@ -21,7 +21,9 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "io" + "os" "strings" "github.com/compose-spec/compose-go/types" @@ -32,11 +34,33 @@ import ( "github.com/docker/docker/registry" "golang.org/x/sync/errgroup" + "github.com/docker/compose-cli/cli/formatter" "github.com/docker/compose-cli/pkg/api" "github.com/docker/compose-cli/pkg/progress" ) +const ( + pullPlanFetch = "fetch" + pullPlanFail = "fail" + pullPlanSkip = "skip" +) + +type pullDryRunServiceResult struct { + Name string + Image string + LocalDigests []string + DistributionDigest string + Plan string +} + +type pullDryRunResults struct { + Services []pullDryRunServiceResult +} + func (s *composeService) Pull(ctx context.Context, project *types.Project, opts api.PullOptions) error { + if opts.DryRun { + return s.pullDryRun(ctx, project, opts) + } if opts.Quiet { return s.pull(ctx, project, opts) } @@ -45,6 +69,51 @@ func (s *composeService) Pull(ctx context.Context, project *types.Project, opts }) } +func (s *composeService) pullDryRun(ctx context.Context, project *types.Project, opts api.PullOptions) error { + results, err := s.pullDryRunSimulate(ctx, project, opts) + if err != nil { + return err + } + return formatter.Print(results, opts.Format, os.Stdout, func(w io.Writer) { + for _, service := range results.Services { + d := service.DistributionDigest + if d == "" { + // follow `docker images --digests` format + d = "" + } + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", service.Name, service.Image, service.Plan, d) + } + }, "SERVICE", "IMAGE", "PLAN", "REMOTE DIGEST") +} + +func getEncodedRegistryAuth(image string, info moby.Info, configFile driver.Auth) (string, error) { + ref, err := reference.ParseNormalizedNamed(image) + if err != nil { + return "", err + } + + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return "", err + } + + key := repoInfo.Index.Name + if repoInfo.Index.Official { + key = info.IndexServerAddress + } + + authConfig, err := configFile.GetAuthConfig(key) + if err != nil { + return "", err + } + + buf, err := json.Marshal(authConfig) + if err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(buf), nil +} + func (s *composeService) pull(ctx context.Context, project *types.Project, opts api.PullOptions) error { info, err := s.apiClient.Info(ctx) if err != nil { @@ -89,33 +158,12 @@ func (s *composeService) pullServiceImage(ctx context.Context, service types.Ser Status: progress.Working, Text: "Pulling", }) - ref, err := reference.ParseNormalizedNamed(service.Image) + registryAuth, err := getEncodedRegistryAuth(service.Image, info, configFile) if err != nil { return err } - - repoInfo, err := registry.ParseRepositoryInfo(ref) - if err != nil { - return err - } - - key := repoInfo.Index.Name - if repoInfo.Index.Official { - key = info.IndexServerAddress - } - - authConfig, err := configFile.GetAuthConfig(key) - if err != nil { - return err - } - - buf, err := json.Marshal(authConfig) - if err != nil { - return err - } - stream, err := s.apiClient.ImagePull(ctx, service.Image, moby.ImagePullOptions{ - RegistryAuth: base64.URLEncoding.EncodeToString(buf), + RegistryAuth: registryAuth, Platform: service.Platform, }) if err != nil { @@ -151,6 +199,60 @@ func (s *composeService) pullServiceImage(ctx context.Context, service types.Ser return nil } +func getPullPlan(service types.ServiceConfig, localDigests []string, dstrDigest string) (plan string) { + canSkip := false + for _, l := range localDigests { + if dstrDigest == l { + canSkip = true + break + } + } + + if service.Image == "" { + // build only service + plan = pullPlanSkip + } else if dstrDigest == "" { + plan = pullPlanFail + } else if canSkip { + plan = pullPlanSkip + } else { + plan = pullPlanFetch + } + return +} + +func (s *composeService) pullDryRunSimulate(ctx context.Context, project *types.Project, opts api.PullOptions) (*pullDryRunResults, error) { + // ignore errors + dstrDigests, _ := s.getDistributionImagesDigests(ctx, project) + + localDigests, err := s.getLocalImagesDigests(ctx, project) + if err != nil { + return nil, err + } + + var results []pullDryRunServiceResult + + for _, service := range project.Services { + l, ok := localDigests[service.Image] + if !ok { + l = []string{} + } + d := dstrDigests[service.Image] + plan := getPullPlan(service, l, d) + result := &pullDryRunServiceResult{ + Name: service.Name, + Image: service.Image, + LocalDigests: l, + DistributionDigest: d, + Plan: plan, + } + results = append(results, *result) + } + + return &pullDryRunResults{Services: results}, nil + +} + func (s *composeService) pullRequiredImages(ctx context.Context, project *types.Project, images map[string]string, quietPull bool) error { info, err := s.apiClient.Info(ctx) if err != nil {