diff --git a/tests/workers/docker-container.go b/tests/workers/docker-container.go
index b3880a79e8f..51f253ead2d 100644
--- a/tests/workers/docker-container.go
+++ b/tests/workers/docker-container.go
@@ -73,6 +73,11 @@ func (w *containerWorker) New(ctx context.Context, cfg *integration.BackendConfi
 
 	cl := func() error {
 		cmd := exec.Command("buildx", "rm", "-f", name)
+		cmd.Env = append(
+			os.Environ(),
+			"BUILDX_CONFIG=/tmp/buildx-"+name,
+			"DOCKER_CONTEXT="+w.docker.DockerAddress(),
+		)
 		return cmd.Run()
 	}
 
diff --git a/tests/workers/remote.go b/tests/workers/remote.go
index 1e2038b8301..14c09cacc47 100644
--- a/tests/workers/remote.go
+++ b/tests/workers/remote.go
@@ -56,6 +56,7 @@ func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 	cl = func() error {
 		err := bkclose()
 		cmd := exec.Command("buildx", "rm", "-f", name)
+		cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
 		if err1 := cmd.Run(); err == nil {
 			err = err1
 		}