Changes to allow staging/testing a container image:

New $PRERELEASE variable pulls pkgs from prerelease dir on download.ceph.com
(requires PRERELEASE_USERNAME/PRERELEASE_PASSWORD in env)
Existing $FORCE_BUILD ignores images on quay.io and builds unconditionally
Add ability to pass in desired container flavors
Hack ceph-release after installation, because ceph-release is built
  during ceph build, and doesn't know what we need
Use tempfile to pass tagname back to parent
New $TEST_BUILD_ONLY avoids attempting to push image to quay.io
Avoid logging into quay.io if PRERELEASE

Note: only x86_64 supported

Signed-off-by: Dan Mick <[email protected]>

Makefile

@@ -43,6 +43,11 @@ BASEOS_TAG ?= ""
 CEPH_DEVEL ?= false
 OSD_FLAVOR ?= "default"
 
+# Default to not doing prerelease (env var must have a value)
+PRERELEASE ?= false
+PRERELEASE_USERNAME ?= ""
+PRERELEASE_PASSWORD ?= ""
+
 
 # ==============================================================================
 # Internal definitions

ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__

@@ -93,9 +93,21 @@ bash -c ' \
     fi ;\
   else \
     RELEASE_VER=1 ;\
-    REPO_URL="http://download.ceph.com/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \
+    if [[ __ENV_[PRERELEASE]__ = true ]] ; then \
+      REPO_URL="https://__ENV_[PRERELEASE_USERNAME]__:__ENV_[PRERELEASE_PASSWORD][email protected]/prerelease/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \
+    else \
+      REPO_URL="http://download.ceph.com/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \
+    fi \
   fi && \
   rpm -Uvh "$REPO_URL/noarch/ceph-release-1-${RELEASE_VER}.el__ENV_[DISTRO_VERSION]__.noarch.rpm" && \
+  #
+  # We have to change this after installing because ceph-release is built long
+  # ago in the ceph build
+  #
+  if [[ __ENV_[PRERELEASE]__ = true ]] ; then \
+    sed -i "s;http://download.ceph.com/;https://__ENV_[PRERELEASE_USERNAME]__:__ENV_[PRERELEASE_PASSWORD][email protected]/prerelease/;" /etc/yum.repos.d/ceph.repo ; \
+    dnf clean expire-cache ; \
+  fi && \
   if [[ __ENV_[DISTRO_VERSION]__ -eq 8 ]]; then \
     yum install -y dnf-plugins-core ; \
     yum copr enable -y tchaikov/python-scikit-learn ; \

contrib/build-ceph-base.sh

@@ -17,7 +17,9 @@ source "${SCRIPT_DIR}/ceph-build-config.sh"
 flavors_to_build="$(get_flavors_to_build "${ARCH}")"
 
 install_docker
-do_login
+if [[ "${PRERELEASE}" != true ]] ; then
+  do_login
+fi
 do_clean
 
 echo ''
@@ -73,18 +75,30 @@ for flavor in $flavors_to_build; do
         FLAVORS="${ceph_codename}-${version_without_build},${ceph_container_distro_dir},${distro_release}" \
         IMAGES_TO_BUILD=daemon-base \
         TAG_REGISTRY="" \
-        DAEMON_BASE_TAG="${full_build_tag}" \
-        BASEOS_REGISTRY="${baseos_registry_setting}" \
-        BASEOS_REPO="${baseos_repo_setting}" \
-        BASEOS_TAG="${baseos_tag_setting}" \
+        DAEMON_BASE_TAG=${full_build_tag} \
+        BASEOS_REGISTRY=${baseos_registry_setting} \
+        BASEOS_REPO=${baseos_repo_setting} \
+        BASEOS_TAG=${baseos_tag_setting} \
       build"
     if [ -z "${DRY_RUN:-}" ]; then
       ${make_cmd}
     else
       # Just echo the make command we would've executed if this is a dry run
       dry_run_info "${make_cmd}"
     fi
-    do_push "${full_build_tag}"
+    if [ "$PRERELEASE" = true ] ; then
+      if [ -z "$FULL_BUILD_TAG_TMPFILE" ] ; then
+        echo "FULL_BUILD_TAG_TMPFILE not passed, can't pass back container tag name"
+      else
+        echo "${full_build_tag}" > "$FULL_BUILD_TAG_TMPFILE"
+      fi
+      continue
+    fi
+    if [ "$TEST_BUILD_ONLY" = true ] ; then
+      echo "would push ${full_build_tag}"
+    else
+      do_push "${full_build_tag}"
+    fi
   done  # for version in ${ceph_version_list}
 
 done  # for flavor in $flavors_to_build

contrib/ceph-build-config.sh

@@ -13,8 +13,8 @@ trap 'exit $?' ERR
 # These build scripts don't need to have the aarch64 part of the distro specified
 # I.e., specifying 'luminous,centos-arm64,7' is not necessary for aarch64 builds; these scripts
 #       will do the right build. See configurable CENTOS_AARCH64_FLAVOR_DISTRO below
-X86_64_FLAVORS_TO_BUILD="pacific,centos,8 quincy,centos,8 reef,centos,8"
-AARCH64_FLAVORS_TO_BUILD="pacific,centos,8 quincy,centos,8 reef,centos,8"
+X86_64_FLAVORS_TO_BUILD="${X86_64_FLAVORS_TO_BUILD:-pacific,centos,8 quincy,centos,8 reef,centos,8}"
+AARCH64_FLAVORS_TO_BUILD="${AARCH64_FLAVORS_TO_BUILD:-pacific,centos,8 quincy,centos,8 reef,centos,8}"
 
 # Allow running this script with the env var ARCH='aarch64' to build arm images
 # ARCH='x86_64'
@@ -226,7 +226,11 @@ function get_ceph_download_url () {
     *)
       error "get_ceph_download_url - unknown distro '${distro}''"
   esac
-  echo "https://download.ceph.com/${flavor_path}/${arch}/"
+  if [ "$PRERELEASE" = true ] ; then
+    echo "https://$PRERELEASE_USERNAME:$PRERELEASE_PASSWORD@download.ceph.com/prerelease/${flavor_path}/${arch}/"
+  else
+    echo "https://download.ceph.com/${flavor_path}/${arch}/"
+  fi
 }
 
 # Return a list of the ceph version strings available on the server as:

maint-lib/makelib.mk

@@ -37,6 +37,9 @@ $(shell bash -c 'set -eu ; \
 	set_var IMAGES_TO_BUILD    "$(IMAGES_TO_BUILD)" ; \
 	set_var STAGING_DIR        "staging/$$CEPH_VERSION$$CEPH_POINT_RELEASE-$$DISTRO-$$DISTRO_VERSION-$$HOST_ARCH" ; \
 	set_var RELEASE            "$(RELEASE)" ; \
+	set_var PRERELEASE         "$(PRERELEASE)" ; \
+	set_var PRERELEASE_USERNAME "$(PRERELEASE_USERNAME)" ; \
+	set_var PRERELEASE_PASSWORD "$(PRERELEASE_PASSWORD)" ; \
 	\
 	daemon_base_img="$$(val_or_default "$(DAEMON_BASE_TAG)" \
 		"daemon-base:$(RELEASE)-$$CEPH_VERSION-$$DISTRO-$$BASEOS_TAG-$$HOST_ARCH")" ; \