diff --git a/.gitignore b/.gitignore index cb05d4f4216..9bd117b8363 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ __pycache__ /tools/appsre-ansible/inventory /docs/osbuild-composer.7 +go.local.mod +go.local.sum +container_worker_built.info +container_composer_built.info diff --git a/Makefile b/Makefile index feb82ab86f0..40e67bac96e 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,10 @@ install: build clean: rm -rf $(BUILDDIR)/bin/ rm -rf $(CURDIR)/rpmbuild + rm -rf $(BUILDDIR)/build/ + rm -f $(BUILDDIR)/go.local.* + rm -f $(BUILDDIR)/container_worker_built.info + rm -f $(BUILDDIR)/container_composer_built.info .PHONY: push-check push-check: build unit-tests srpm man @@ -275,3 +279,71 @@ scratch: $(RPM_SPECFILE) $(RPM_TARBALL) --nocheck \ $(RPM_SPECFILE) + +# either "docker" or "sudo podman" +# podman needs to build as root as it also needs to run as root afterwards +CONTAINER_EXECUTABLE ?= sudo podman + +DOCKER_IMAGE_WORKER := osbuild-worker_devel +DOCKERFILE_WORKER := distribution/Dockerfile-worker_srcinstall + +DOCKER_IMAGE_COMPOSER := osbuild-composer_devel +DOCKERFILE_COMPOSER := distribution/Dockerfile-composer + +GOPROXY ?= https://proxy.golang.org,direct + +# source where the other repos are locally +# has to end with a trailing slash +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../ + +# names of folder that have to be git-cloned additionally to be able +# to build all code +SRC_DEPS_EXTERNAL_NAMES := images pulp-client +SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SRC_DEPS_EXTERNAL_NAMES)) + +$(SRC_DEPS_EXTERNAL_DIRS): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + + +SRC_DEPS_DIRS := internal cmd pkg repositories + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(SRC_DEPS_DIRS) -name *.go -or -name *.json) +SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go) + +# dependencies to rebuild worker +WORKER_SRC_DEPS := $(SRC_DEPS) +# dependencies to rebuild composer +COMPOSER_SRC_DEPS := $(SRC_DEPS) + +GOMODARGS ?= -modfile=go.local.mod + +USE_BTRFS ?= yes + +go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(WORKER_SRC_DEPS) $(COMPOSER_SRC_DEPS) Makefile + cp go.mod go.local.mod + cp go.sum go.local.sum + + go mod edit $(GOMODARGS) -replace github.com/osbuild/images=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)images + go mod edit $(GOMODARGS) -replace github.com/osbuild/pulp-client=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)pulp-client + go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=./pkg/splunk_logger + env GOPROXY=$(GOPROXY) go mod tidy $(GOMODARGS) + env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS) + +container_worker_built.info: go.local.mod $(WORKER_SRC_DEPS) $(DOCKERFILE_WORKER) + $(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_WORKER) -f $(DOCKERFILE_WORKER) --build-arg GOMODARGS=$(GOMODARGS) --build-arg USE_BTRFS=$(USE_BTRFS) . + echo "Worker last built on" > $@ + date >> $@ + +container_composer_built.info: go.local.mod $(COMPOSER_SRC_DEPS) $(DOCKERFILE_COMPOSER) + $(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_COMPOSER) -f $(DOCKERFILE_COMPOSER) --build-arg GOMODARGS=$(GOMODARGS) . + echo "Composer last built on" > $@ + date >> $@ + +# build a container with a worker from full source +.PHONY: container_worker +container_worker: container_worker_built.info + +# build a container with the composer from full source +.PHONY: container_composer +container_composer: container_composer_built.info diff --git a/distribution/Dockerfile-composer b/distribution/Dockerfile-composer new file mode 100644 index 00000000000..7975a5327a0 --- /dev/null +++ b/distribution/Dockerfile-composer @@ -0,0 +1,37 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-composer/ + +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +FROM fedora:39 + +RUN dnf install -y python3 python3-dnf gpgme libassuan device-mapper-libs +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/usr/share/osbuild-composer/" +RUN mkdir -p "/opt/migrate/" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-composer /usr/libexec/osbuild-composer/ +COPY ./containers/osbuild-composer/entrypoint.py /opt/entrypoint.py + +COPY ./pkg/jobqueue/dbjobqueue/schemas /opt/migrate/schemas +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ + +EXPOSE 8008 8080 8700 +ENTRYPOINT ["python3", "/opt/entrypoint.py", "--remote-worker-api", "--composer-api", "--prometheus", "--shutdown-wait-period", "15"] diff --git a/distribution/Dockerfile-fauxauth b/distribution/Dockerfile-fauxauth index 5bb23e47735..52c423f6273 100644 --- a/distribution/Dockerfile-fauxauth +++ b/distribution/Dockerfile-fauxauth @@ -4,6 +4,10 @@ FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder # a repository owned by a different user. COPY --chown=1001 . . ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + RUN go install ./cmd/osbuild-mock-openid-provider/ FROM registry.access.redhat.com/ubi9/ubi-minimal:latest diff --git a/distribution/Dockerfile-fauxauth_srcinstall b/distribution/Dockerfile-fauxauth_srcinstall new file mode 100644 index 00000000000..ef6ed4adff4 --- /dev/null +++ b/distribution/Dockerfile-fauxauth_srcinstall @@ -0,0 +1,25 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +COPY . . +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-mock-openid-provider/ + +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y python3 +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" + +COPY --from=builder /opt/app-root/src/go/bin/osbuild-mock-openid-provider /usr/libexec/osbuild-composer/ +COPY ./containers/fauxauth/fauxauth.py /opt/fauxauth.py + +EXPOSE 8080 8080 +ENTRYPOINT "/opt/fauxauth.py" diff --git a/distribution/Dockerfile-ubi b/distribution/Dockerfile-ubi index d17c2ee562e..2fc20dc4935 100644 --- a/distribution/Dockerfile-ubi +++ b/distribution/Dockerfile-ubi @@ -13,11 +13,18 @@ ARG COMMIT ENV LDFLAGS="${COMMIT:+-X \'github.com/osbuild/osbuild-composer/internal/common.GitRev=${COMMIT}\'}" ENV LDFLAGS="${LDFLAGS:+-ldflags=\"${LDFLAGS}\"}" ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ENV GOMODARGS=$GOMODARGS + # if run without "sh -c", podman for some reason executes the command in a way, # which results in the following error: # [1/3] STEP 12/12: RUN go install ${LDFLAGS} ./cmd/osbuild-composer/ # invalid value "\"-X" for flag -ldflags: missing = in = -RUN /usr/bin/sh -c "go install ${LDFLAGS} ./cmd/osbuild-composer/" +RUN /usr/bin/sh -c "go install ${LDFLAGS} ${GOMODARGS} ./cmd/osbuild-composer/" FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 RUN go install github.com/jackc/tern@latest diff --git a/distribution/Dockerfile-worker b/distribution/Dockerfile-worker index a35e36e1c59..87901bbf9bd 100644 --- a/distribution/Dockerfile-worker +++ b/distribution/Dockerfile-worker @@ -1,4 +1,4 @@ -FROM fedora:37 AS builder +FROM fedora:39 AS builder ENV GOBIN=/opt/app-root/src/go/bin # extra packages are needed # to compile osbuild @@ -6,14 +6,22 @@ RUN dnf install -y golang \ krb5-devel \ gpgme-devel \ libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + WORKDIR /osbuild-composer COPY . /osbuild-composer ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + RUN go install ./cmd/osbuild-worker FROM fedora RUN dnf update -y && dnf upgrade -y -RUN dnf install -y libxcrypt-compat qemu-img osbuild osbuild-ostree +RUN dnf install -y libxcrypt-compat qemu-img osbuild osbuild-ostree osbuild-depsolve-dnf RUN mkdir -p "/usr/libexec/osbuild-composer" RUN mkdir -p "/etc/osbuild-composer/" RUN mkdir -p "/run/osbuild-composer/" diff --git a/distribution/Dockerfile-worker_srcinstall b/distribution/Dockerfile-worker_srcinstall new file mode 100644 index 00000000000..6eb78d4ffe2 --- /dev/null +++ b/distribution/Dockerfile-worker_srcinstall @@ -0,0 +1,34 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin +# extra packages are needed +# to compile osbuild +RUN dnf install -y golang \ + krb5-devel \ + gpgme-devel \ + libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-worker + +FROM osbuild_devel + +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-worker/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/output" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-worker /usr/libexec/osbuild-composer/ + +ENTRYPOINT ["/usr/libexec/osbuild-composer/osbuild-worker"]