From 32e5e98490abe5a4a04b3640aacadf8a6b577388 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 00:52:08 +0200 Subject: [PATCH 01/17] removed obsolete v3.x workflow files --- .github/workflows/pythonpackage_future3.yml | 58 --------------------- 1 file changed, 58 deletions(-) delete mode 100644 .github/workflows/pythonpackage_future3.yml diff --git a/.github/workflows/pythonpackage_future3.yml b/.github/workflows/pythonpackage_future3.yml deleted file mode 100644 index 9844039b8..000000000 --- a/.github/workflows/pythonpackage_future3.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Python + Docs Checks and Tests - -on: - push: - branches: - - 'future3/**' - paths: - - '**.py' - - '**.py.*' - - 'docs/sphinx/**' - pull_request: - branches: - - 'future3/**' - paths: - - '**.py' - - '**.py.*' - - 'docs/sphinx/**' - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libasound2-dev - python -m pip install --upgrade pip - pip install wheel - pip install spidev - pip install -r requirements.txt - # For operation of the Jukebox, ZMQ must be compiled from sources due to Websocket support - # When just building the docs, the regular ZMQ package is sufficient - pip install -r docs/sphinx/requirements_pyzmq.txt - pip install -r docs/sphinx/requirements.txt - # Also install all optional dependencies - pip install -r src/jukebox/components/rfid/fake_reader_gui/requirements.txt - - name: Lint with flake8 - run: | - pip install flake8 - # Stop the build if linting fails - ./run_flake8.sh - - name: Build the docs - working-directory: ./docs/sphinx - run: | - # Stop the build if documentation cannot be built - # Treat all warnings as errors - sphinx-build -W --keep-going -T -a -E -b html . _build From e6f6ef7dc00cd5ec734d188c56a63c3a1a390937 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 01:17:52 +0200 Subject: [PATCH 02/17] set user and group as build args to test altuser removed now obsolete "*_altuser" Dockerfile --- .github/workflows/docker_bullseye_altuser.yml | 5 ++- ci/Dockerfile.bullseye.test_install.amd64 | 7 +++- ci/Dockerfile.bullseye.test_install.armv7 | 7 +++- ...erfile.bullseye.test_install_altuser.armv7 | 37 ------------------- ci/Dockerfile.buster.test_install.armv7 | 7 +++- 5 files changed, 19 insertions(+), 44 deletions(-) delete mode 100644 ci/Dockerfile.bullseye.test_install_altuser.armv7 diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index 59e6d91d6..c60c25605 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -32,11 +32,14 @@ jobs: context: . load: true push: false - file: ./ci/Dockerfile.bullseye.test_install_altuser.armv7 + file: ./ci/Dockerfile.bullseye.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-bullseye-altuser:latest cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye-altuser cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye-altuser + build-args: | + USER_NAME=hans + USER_GROUP=wurst - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 diff --git a/ci/Dockerfile.bullseye.test_install.amd64 b/ci/Dockerfile.bullseye.test_install.amd64 index 971c03537..4e9d15118 100644 --- a/ci/Dockerfile.bullseye.test_install.amd64 +++ b/ci/Dockerfile.bullseye.test_install.amd64 @@ -1,11 +1,14 @@ FROM debian:bullseye +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME + ENV DOCKER_RUNNING=true -ENV USER=pi +ENV USER=$USER_NAME COPY . /code WORKDIR /code -RUN groupadd --gid 1000 $USER ;\ +RUN groupadd --gid 1000 $USER_GROUP ;\ useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ chown -R 1000:1000 /code /home/$USER ;\ chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 index d7fbf9b97..e6ed6d6e0 100644 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ b/ci/Dockerfile.bullseye.test_install.armv7 @@ -1,11 +1,14 @@ FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME + ENV DOCKER_RUNNING=true -ENV USER=pi +ENV USER=$USER_NAME COPY . /code WORKDIR /code -RUN groupadd --gid 1000 $USER ;\ +RUN groupadd --gid 1000 $USER_GROUP ;\ useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ chown -R 1000:1000 /code /home/$USER ;\ chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ diff --git a/ci/Dockerfile.bullseye.test_install_altuser.armv7 b/ci/Dockerfile.bullseye.test_install_altuser.armv7 deleted file mode 100644 index 9043d7308..000000000 --- a/ci/Dockerfile.bullseye.test_install_altuser.armv7 +++ /dev/null @@ -1,37 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim -ENV DOCKER_RUNNING=true -ENV USER=hans - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 wurst ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 index f3cd3f3e6..20878df23 100644 --- a/ci/Dockerfile.buster.test_install.armv7 +++ b/ci/Dockerfile.buster.test_install.armv7 @@ -1,11 +1,14 @@ FROM --platform=linux/arm/v7 arm32v7/debian:buster-slim +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME + ENV DOCKER_RUNNING=true -ENV USER=pi +ENV USER=$USER_NAME COPY . /code WORKDIR /code -RUN groupadd --gid 1000 pi ;\ +RUN groupadd --gid 1000 $USER_GROUP ;\ useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ chown -R 1000:1000 /code /home/$USER ;\ chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ From 87faa7e11bd4f9d16e86993275630869bb1097d1 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 01:43:34 +0200 Subject: [PATCH 03/17] set GIT_BRANCH and GIT_URL as build args makes sure that the installation script uses the current repo/branch --- .github/workflows/docker_bullseye.yml | 3 +++ .github/workflows/docker_bullseye_altuser.yml | 2 ++ .github/workflows/docker_buster.yml | 3 +++ ci/Dockerfile.bullseye.test_install.amd64 | 4 ++++ ci/Dockerfile.bullseye.test_install.armv7 | 4 ++++ ci/Dockerfile.buster.test_install.armv7 | 4 ++++ scripts/installscripts/tests/run_installation_tests.sh | 3 +-- scripts/installscripts/tests/run_installation_tests2.sh | 3 +-- scripts/installscripts/tests/run_installation_tests3.sh | 3 +-- 9 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml index e15ae2173..17244b69e 100644 --- a/.github/workflows/docker_bullseye.yml +++ b/.github/workflows/docker_bullseye.yml @@ -37,6 +37,9 @@ jobs: tags: rpi-jukebox-rfid-bullseye:latest cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye + build-args: | + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index c60c25605..66cb6e3da 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -40,6 +40,8 @@ jobs: build-args: | USER_NAME=hans USER_GROUP=wurst + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml index 69690ed4f..9f812b943 100644 --- a/.github/workflows/docker_buster.yml +++ b/.github/workflows/docker_buster.yml @@ -37,6 +37,9 @@ jobs: tags: rpi-jukebox-rfid-buster:latest cache-from: type=gha,scope=$GITHUB_REF_NAME-buster cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-buster + build-args: | + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} - name: Run run_installation_tests.sh Buster ARMv7 uses: tj-actions/docker-run@v2 diff --git a/ci/Dockerfile.bullseye.test_install.amd64 b/ci/Dockerfile.bullseye.test_install.amd64 index 4e9d15118..c43cb58f0 100644 --- a/ci/Dockerfile.bullseye.test_install.amd64 +++ b/ci/Dockerfile.bullseye.test_install.amd64 @@ -1,9 +1,13 @@ FROM debian:bullseye ARG USER_NAME=pi ARG USER_GROUP=$USER_NAME +ARG GIT_BRANCH +ARG GIT_URL ENV DOCKER_RUNNING=true ENV USER=$USER_NAME +ENV GIT_BRANCH=$GIT_BRANCH +ENV GIT_URL=$GIT_URL COPY . /code WORKDIR /code diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 index e6ed6d6e0..accc91d85 100644 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ b/ci/Dockerfile.bullseye.test_install.armv7 @@ -1,9 +1,13 @@ FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim ARG USER_NAME=pi ARG USER_GROUP=$USER_NAME +ARG GIT_BRANCH +ARG GIT_URL ENV DOCKER_RUNNING=true ENV USER=$USER_NAME +ENV GIT_BRANCH=$GIT_BRANCH +ENV GIT_URL=$GIT_URL COPY . /code WORKDIR /code diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 index 20878df23..5d58e5c9d 100644 --- a/ci/Dockerfile.buster.test_install.armv7 +++ b/ci/Dockerfile.buster.test_install.armv7 @@ -1,9 +1,13 @@ FROM --platform=linux/arm/v7 arm32v7/debian:buster-slim ARG USER_NAME=pi ARG USER_GROUP=$USER_NAME +ARG GIT_BRANCH +ARG GIT_URL ENV DOCKER_RUNNING=true ENV USER=$USER_NAME +ENV GIT_BRANCH=$GIT_BRANCH +ENV GIT_URL=$GIT_URL COPY . /code WORKDIR /code diff --git a/scripts/installscripts/tests/run_installation_tests.sh b/scripts/installscripts/tests/run_installation_tests.sh index a22e23d99..a19261cc8 100644 --- a/scripts/installscripts/tests/run_installation_tests.sh +++ b/scripts/installscripts/tests/run_installation_tests.sh @@ -25,8 +25,7 @@ export DEBIAN_FRONTEND=noninteractive # n no RFID registration # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation diff --git a/scripts/installscripts/tests/run_installation_tests2.sh b/scripts/installscripts/tests/run_installation_tests2.sh index f907bf29d..1beb95a22 100644 --- a/scripts/installscripts/tests/run_installation_tests2.sh +++ b/scripts/installscripts/tests/run_installation_tests2.sh @@ -27,8 +27,7 @@ export DEBIAN_FRONTEND=noninteractive # yes, reader is connected # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' +bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' INSTALLATION_EXITCODE=$? # Test installation diff --git a/scripts/installscripts/tests/run_installation_tests3.sh b/scripts/installscripts/tests/run_installation_tests3.sh index aa75c92b7..2e0290b27 100644 --- a/scripts/installscripts/tests/run_installation_tests3.sh +++ b/scripts/installscripts/tests/run_installation_tests3.sh @@ -25,8 +25,7 @@ export DEBIAN_FRONTEND=noninteractive # n no RFID registration # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation From 2bc3109eec5ae8d974aa5bd486cd3e32cfaa1672 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 01:44:41 +0200 Subject: [PATCH 04/17] use correct variables for refname resolving --- .github/workflows/docker_bullseye.yml | 4 ++-- .github/workflows/docker_bullseye_altuser.yml | 4 ++-- .github/workflows/docker_buster.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml index 17244b69e..5717e400a 100644 --- a/.github/workflows/docker_bullseye.yml +++ b/.github/workflows/docker_bullseye.yml @@ -35,8 +35,8 @@ jobs: file: ./ci/Dockerfile.bullseye.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-bullseye:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye + cache-from: type=gha,scope=${{ github.ref_name }}-bullseye + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-bullseye build-args: | GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index 66cb6e3da..378f5988d 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -35,8 +35,8 @@ jobs: file: ./ci/Dockerfile.bullseye.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-bullseye-altuser:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye-altuser - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye-altuser + cache-from: type=gha,scope=${{ github.ref_name }}-bullseye-altuser + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-bullseye-altuser build-args: | USER_NAME=hans USER_GROUP=wurst diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml index 9f812b943..272108c0c 100644 --- a/.github/workflows/docker_buster.yml +++ b/.github/workflows/docker_buster.yml @@ -35,8 +35,8 @@ jobs: file: ./ci/Dockerfile.buster.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-buster:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-buster - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-buster + cache-from: type=gha,scope=${{ github.ref_name }}-buster + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-buster build-args: | GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} From 6f13b756f0358d8fc57eb7319632e3ee80f2ecf6 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 01:45:52 +0200 Subject: [PATCH 05/17] harmonized workflow and Dockerfiles --- .github/workflows/docker_bullseye.yml | 16 +-- .github/workflows/docker_bullseye_altuser.yml | 14 +-- .github/workflows/docker_buster.yml | 100 +++++++++--------- ci/Dockerfile.bullseye.test_install.amd64 | 6 +- ci/Dockerfile.bullseye.test_install.armv7 | 6 +- ci/Dockerfile.buster.test_install.armv7 | 8 +- 6 files changed, 76 insertions(+), 74 deletions(-) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml index 5717e400a..bbd72200d 100644 --- a/.github/workflows/docker_bullseye.yml +++ b/.github/workflows/docker_bullseye.yml @@ -14,15 +14,15 @@ on: jobs: build: - + runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3.0.0 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 @@ -40,22 +40,22 @@ jobs: build-args: | GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} - + - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests.sh options: --platform linux/arm/v7 - args: | + name: run_installation_tests.sh + args: | /code/scripts/installscripts/tests/run_installation_tests.sh - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests2.sh options: --platform linux/arm/v7 + name: run_installation_tests2.sh args: | /code/scripts/installscripts/tests/run_installation_tests2.sh @@ -63,7 +63,7 @@ jobs: uses: tj-actions/docker-run@v2 with: image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests3.sh options: --platform linux/arm/v7 + name: run_installation_tests3.sh args: | /code/scripts/installscripts/tests/run_installation_tests3.sh diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index 378f5988d..0024ac8df 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -14,19 +14,19 @@ on: jobs: build: - + runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3.0.0 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 - - name: Build altuser Bullseye ARMv7 + - name: Build Bullseye ARMv7 altuser uses: docker/build-push-action@v5 with: context: . @@ -42,7 +42,7 @@ jobs: USER_GROUP=wurst GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} - + - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -51,7 +51,7 @@ jobs: name: run_installation_tests.sh args: | /code/scripts/installscripts/tests/run_installation_tests.sh - + - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -60,7 +60,7 @@ jobs: name: run_installation_tests2.sh args: | /code/scripts/installscripts/tests/run_installation_tests2.sh - + - name: Run run_installation_tests3.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml index 272108c0c..8d2260605 100644 --- a/.github/workflows/docker_buster.yml +++ b/.github/workflows/docker_buster.yml @@ -13,57 +13,57 @@ on: jobs: - build: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 - runs-on: ubuntu-latest + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 - steps: - - uses: actions/checkout@v4 + - name: Build Buster ARMv7 + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + file: ./ci/Dockerfile.buster.test_install.armv7 + platforms: linux/arm/v7 + tags: rpi-jukebox-rfid-buster:latest + cache-from: type=gha,scope=${{ github.ref_name }}-buster + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-buster + build-args: | + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + - name: Run run_installation_tests.sh Buster ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: rpi-jukebox-rfid-buster:latest + options: --platform linux/arm/v7 + name: run_installation_tests.sh + args: | + /code/scripts/installscripts/tests/run_installation_tests.sh - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Buster ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.buster.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-buster:latest - cache-from: type=gha,scope=${{ github.ref_name }}-buster - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-buster - build-args: | - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Run run_installation_tests.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests.sh - - - name: Run run_installation_tests2.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh + - name: Run run_installation_tests2.sh Buster ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: rpi-jukebox-rfid-buster:latest + options: --platform linux/arm/v7 + name: run_installation_tests2.sh + args: | + /code/scripts/installscripts/tests/run_installation_tests2.sh + + - name: Run run_installation_tests3.sh Buster ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: rpi-jukebox-rfid-buster:latest + options: --platform linux/arm/v7 + name: run_installation_tests3.sh + args: | + /code/scripts/installscripts/tests/run_installation_tests3.sh diff --git a/ci/Dockerfile.bullseye.test_install.amd64 b/ci/Dockerfile.bullseye.test_install.amd64 index c43cb58f0..39e15907c 100644 --- a/ci/Dockerfile.bullseye.test_install.amd64 +++ b/ci/Dockerfile.bullseye.test_install.amd64 @@ -22,7 +22,7 @@ RUN groupadd --gid 1000 $USER_GROUP ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ + apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ @@ -33,8 +33,10 @@ RUN export DEBIAN_FRONTEND=noninteractive ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ + # install here to speed up GitHub Action + apt-get -y install raspberrypi-kernel-headers ;\ apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ + apt-get -y install wget build-essential git iw locales wpasupplicant ;\ apt-get clean ;\ touch /boot/cmdlinetxt ;\ rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 index accc91d85..467dca71b 100644 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ b/ci/Dockerfile.bullseye.test_install.armv7 @@ -22,7 +22,7 @@ RUN groupadd --gid 1000 $USER_GROUP ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ + apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ @@ -34,9 +34,9 @@ RUN export DEBIAN_FRONTEND=noninteractive ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel raspberrypi-kernel-headers;\ + apt-get -y install raspberrypi-kernel-headers ;\ apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ + apt-get -y install wget build-essential git iw locales wpasupplicant ;\ apt-get clean ;\ touch /boot/cmdlinetxt ;\ rm -rf /var/cache/apt/* /var/lib/apt/lists/* diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 index 5d58e5c9d..3a8a3c793 100644 --- a/ci/Dockerfile.buster.test_install.armv7 +++ b/ci/Dockerfile.buster.test_install.armv7 @@ -22,9 +22,7 @@ RUN groupadd --gid 1000 $USER_GROUP ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers;\ + apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ @@ -35,8 +33,10 @@ RUN export DEBIAN_FRONTEND=noninteractive ;\ RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ + # install here to speed up GitHub Action + apt-get -y install raspberrypi-kernel-headers ;\ apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ + apt-get -y install wget build-essential git iw locales wpasupplicant ;\ apt-get clean ;\ touch /boot/cmdlinetxt ;\ rm -rf /var/cache/apt/* /var/lib/apt/lists/* From 187ff6069280bdd26b6957dfa6a75e72bb4de9d4 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:28:06 +0200 Subject: [PATCH 06/17] optimize test docker performance. copy changed only include the scripts that are needed. Otherwise every code change will invalidate the container cache. changed internal path of container and test scripts --- .github/workflows/docker_bullseye.yml | 6 +++--- .github/workflows/docker_bullseye_altuser.yml | 6 +++--- .github/workflows/docker_buster.yml | 6 +++--- ci/Dockerfile.bullseye.test_install.armv7 | 16 +++++++++------- ci/Dockerfile.buster.test_install.armv7 | 16 +++++++++------- .../tests/run_installation_tests.sh | 4 ++-- .../tests/run_installation_tests2.sh | 4 ++-- .../tests/run_installation_tests3.sh | 4 ++-- 8 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml index bbd72200d..da52fc442 100644 --- a/.github/workflows/docker_bullseye.yml +++ b/.github/workflows/docker_bullseye.yml @@ -48,7 +48,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests.sh args: | - /code/scripts/installscripts/tests/run_installation_tests.sh + ./run_installation_tests.sh - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 @@ -57,7 +57,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests2.sh args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh + ./run_installation_tests2.sh - name: Run run_installation_tests3.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 @@ -66,4 +66,4 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests3.sh args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh + ./run_installation_tests3.sh diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index 0024ac8df..725bfae33 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -50,7 +50,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests.sh args: | - /code/scripts/installscripts/tests/run_installation_tests.sh + ./run_installation_tests.sh - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 @@ -59,7 +59,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests2.sh args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh + ./run_installation_tests2.sh - name: Run run_installation_tests3.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 @@ -68,4 +68,4 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests3.sh args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh + ./run_installation_tests3.sh diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml index 8d2260605..2aabb36e7 100644 --- a/.github/workflows/docker_buster.yml +++ b/.github/workflows/docker_buster.yml @@ -48,7 +48,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests.sh args: | - /code/scripts/installscripts/tests/run_installation_tests.sh + ./run_installation_tests.sh - name: Run run_installation_tests2.sh Buster ARMv7 uses: tj-actions/docker-run@v2 @@ -57,7 +57,7 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests2.sh args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh + ./run_installation_tests2.sh - name: Run run_installation_tests3.sh Buster ARMv7 uses: tj-actions/docker-run@v2 @@ -66,4 +66,4 @@ jobs: options: --platform linux/arm/v7 name: run_installation_tests3.sh args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh + ./run_installation_tests3.sh diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 index 467dca71b..d55b1373a 100644 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ b/ci/Dockerfile.bullseye.test_install.armv7 @@ -9,16 +9,18 @@ ENV USER=$USER_NAME ENV GIT_BRANCH=$GIT_BRANCH ENV GIT_URL=$GIT_URL -COPY . /code -WORKDIR /code +COPY scripts/installscripts/tests/*.sh /tests/ +COPY scripts/installscripts/buster-install-default.sh / RUN groupadd --gid 1000 $USER_GROUP ;\ useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh + chown -R 1000:1000 /tests /home/$USER ;\ + chmod +x /buster-install-default.sh ;\ + chmod +x /tests/run_installation_tests.sh ;\ + chmod +x /tests/run_installation_tests2.sh ;\ + chmod +x /tests/run_installation_tests3.sh + +WORKDIR /tests RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 index 3a8a3c793..87dc7afbe 100644 --- a/ci/Dockerfile.buster.test_install.armv7 +++ b/ci/Dockerfile.buster.test_install.armv7 @@ -9,16 +9,18 @@ ENV USER=$USER_NAME ENV GIT_BRANCH=$GIT_BRANCH ENV GIT_URL=$GIT_URL -COPY . /code -WORKDIR /code +COPY scripts/installscripts/tests/*.sh /tests/ +COPY scripts/installscripts/buster-install-default.sh / RUN groupadd --gid 1000 $USER_GROUP ;\ useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh + chown -R 1000:1000 /tests /home/$USER ;\ + chmod +x /buster-install-default.sh ;\ + chmod +x /tests/run_installation_tests.sh ;\ + chmod +x /tests/run_installation_tests2.sh ;\ + chmod +x /tests/run_installation_tests3.sh + +WORKDIR /tests RUN export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ diff --git a/scripts/installscripts/tests/run_installation_tests.sh b/scripts/installscripts/tests/run_installation_tests.sh index a19261cc8..41fa3da3c 100644 --- a/scripts/installscripts/tests/run_installation_tests.sh +++ b/scripts/installscripts/tests/run_installation_tests.sh @@ -25,8 +25,8 @@ export DEBIAN_FRONTEND=noninteractive # n no RFID registration # n No reboot -bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE diff --git a/scripts/installscripts/tests/run_installation_tests2.sh b/scripts/installscripts/tests/run_installation_tests2.sh index 1beb95a22..6162df3b0 100644 --- a/scripts/installscripts/tests/run_installation_tests2.sh +++ b/scripts/installscripts/tests/run_installation_tests2.sh @@ -27,8 +27,8 @@ export DEBIAN_FRONTEND=noninteractive # yes, reader is connected # n No reboot -bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE diff --git a/scripts/installscripts/tests/run_installation_tests3.sh b/scripts/installscripts/tests/run_installation_tests3.sh index 2e0290b27..d983a2db1 100644 --- a/scripts/installscripts/tests/run_installation_tests3.sh +++ b/scripts/installscripts/tests/run_installation_tests3.sh @@ -25,8 +25,8 @@ export DEBIAN_FRONTEND=noninteractive # n no RFID registration # n No reboot -bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE From 79f6c4426490b1c95f38318b8c28b21dabb6766d Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Fri, 13 Oct 2023 22:11:25 +0200 Subject: [PATCH 07/17] optimize test docker performance. refactored changed thestructure to use the cache as much as possible. repo url -> packages -> user -> scripts --- ci/Dockerfile.bullseye.test_install.armv7 | 77 ++++++++++++----------- ci/Dockerfile.buster.test_install.armv7 | 77 ++++++++++++----------- 2 files changed, 84 insertions(+), 70 deletions(-) diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 index d55b1373a..07de9782e 100644 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ b/ci/Dockerfile.bullseye.test_install.armv7 @@ -1,46 +1,53 @@ FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME + +# Define constants +ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive +RUN touch /boot/cmdlinetxt + +# Define Git Repo variables. If this changes all should be build again ARG GIT_BRANCH ARG GIT_URL -ENV DOCKER_RUNNING=true -ENV USER=$USER_NAME -ENV GIT_BRANCH=$GIT_BRANCH -ENV GIT_URL=$GIT_URL - -COPY scripts/installscripts/tests/*.sh /tests/ -COPY scripts/installscripts/buster-install-default.sh / - -RUN groupadd --gid 1000 $USER_GROUP ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /tests /home/$USER ;\ - chmod +x /buster-install-default.sh ;\ - chmod +x /tests/run_installation_tests.sh ;\ - chmod +x /tests/run_installation_tests2.sh ;\ - chmod +x /tests/run_installation_tests3.sh +ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL -WORKDIR /tests +# Install packages +RUN apt-get update && \ + apt-get -y install \ + apt-utils \ + curl \ + gnupg -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ +RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant ;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* + apt-get update && \ + apt-get -y install \ + build-essential \ + git \ + iw \ + locales \ + # install here to speed up GitHub Action + raspberrypi-kernel-headers \ + sudo \ + systemd \ + wget \ + wpasupplicant ;\ + rm -rf /var/lib/apt/lists/* + +# Define user info that my change, but are irrelevant for package installation +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME +ENV USER=$USER_NAME + +RUN groupadd --gid 1000 $USER_GROUP && \ + useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ + echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER USER $USER + +# Define needed files for installation test +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / + +WORKDIR /tests diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 index 87dc7afbe..d436d5b24 100644 --- a/ci/Dockerfile.buster.test_install.armv7 +++ b/ci/Dockerfile.buster.test_install.armv7 @@ -1,46 +1,53 @@ FROM --platform=linux/arm/v7 arm32v7/debian:buster-slim -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME -ARG GIT_BRANCH -ARG GIT_URL -ENV DOCKER_RUNNING=true -ENV USER=$USER_NAME -ENV GIT_BRANCH=$GIT_BRANCH -ENV GIT_URL=$GIT_URL +# Define constants +ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive +RUN touch /boot/cmdlinetxt -COPY scripts/installscripts/tests/*.sh /tests/ -COPY scripts/installscripts/buster-install-default.sh / +# Define Git Repo variables. If this changes all should be build again +ARG GIT_BRANCH +ARG GIT_URL -RUN groupadd --gid 1000 $USER_GROUP ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /tests /home/$USER ;\ - chmod +x /buster-install-default.sh ;\ - chmod +x /tests/run_installation_tests.sh ;\ - chmod +x /tests/run_installation_tests2.sh ;\ - chmod +x /tests/run_installation_tests3.sh +ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL -WORKDIR /tests +# Install packages +RUN apt-get update && \ + apt-get -y install \ + apt-utils \ + curl \ + gnupg -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ +RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant ;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* + apt-get update && \ + apt-get -y install \ + build-essential \ + git \ + iw \ + locales \ + # install here to speed up GitHub Action + raspberrypi-kernel-headers \ + sudo \ + systemd \ + wget \ + wpasupplicant ;\ + rm -rf /var/lib/apt/lists/* + +# Define user info that my change, but are irrelevant for package installation +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME +ENV USER=$USER_NAME + +RUN groupadd --gid 1000 $USER_GROUP && \ + useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ + echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER USER $USER + +# Define needed files for installation test +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / + +WORKDIR /tests From 77d8b47fb9a413c3d77e33f3082d6ec578a43dde Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Mon, 16 Oct 2023 01:38:37 +0200 Subject: [PATCH 08/17] changed cache key with ref instead of ref_name pull request will get a different cache --- .github/workflows/docker_bullseye.yml | 20 +++++++++---------- .github/workflows/docker_bullseye_altuser.yml | 20 +++++++++---------- .github/workflows/docker_buster.yml | 20 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml index da52fc442..039c8f5ef 100644 --- a/.github/workflows/docker_bullseye.yml +++ b/.github/workflows/docker_bullseye.yml @@ -14,18 +14,18 @@ on: jobs: build: - + runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3.0.0 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 - + - name: Build Bullseye ARMv7 uses: docker/build-push-action@v5 with: @@ -35,12 +35,12 @@ jobs: file: ./ci/Dockerfile.bullseye.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-bullseye:latest - cache-from: type=gha,scope=${{ github.ref_name }}-bullseye - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-bullseye + cache-from: type=gha,scope=${{ github.ref }}-bullseye + cache-to: type=gha,mode=max,scope=${{ github.ref }}-bullseye build-args: | GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} - + - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -49,7 +49,7 @@ jobs: name: run_installation_tests.sh args: | ./run_installation_tests.sh - + - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -58,7 +58,7 @@ jobs: name: run_installation_tests2.sh args: | ./run_installation_tests2.sh - + - name: Run run_installation_tests3.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml index 725bfae33..60ab0fde2 100644 --- a/.github/workflows/docker_bullseye_altuser.yml +++ b/.github/workflows/docker_bullseye_altuser.yml @@ -14,18 +14,18 @@ on: jobs: build: - + runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3.0.0 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 - + - name: Build Bullseye ARMv7 altuser uses: docker/build-push-action@v5 with: @@ -35,14 +35,14 @@ jobs: file: ./ci/Dockerfile.bullseye.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-bullseye-altuser:latest - cache-from: type=gha,scope=${{ github.ref_name }}-bullseye-altuser - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-bullseye-altuser + cache-from: type=gha,scope=${{ github.ref }}-bullseye-altuser + cache-to: type=gha,mode=max,scope=${{ github.ref }}-bullseye-altuser build-args: | USER_NAME=hans USER_GROUP=wurst GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} - + - name: Run run_installation_tests.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -51,7 +51,7 @@ jobs: name: run_installation_tests.sh args: | ./run_installation_tests.sh - + - name: Run run_installation_tests2.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: @@ -60,7 +60,7 @@ jobs: name: run_installation_tests2.sh args: | ./run_installation_tests2.sh - + - name: Run run_installation_tests3.sh Bullseye ARMv7 uses: tj-actions/docker-run@v2 with: diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml index 2aabb36e7..cec1f2509 100644 --- a/.github/workflows/docker_buster.yml +++ b/.github/workflows/docker_buster.yml @@ -14,18 +14,18 @@ on: jobs: build: - + runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3.0.0 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 - + - name: Build Buster ARMv7 uses: docker/build-push-action@v5 with: @@ -35,12 +35,12 @@ jobs: file: ./ci/Dockerfile.buster.test_install.armv7 platforms: linux/arm/v7 tags: rpi-jukebox-rfid-buster:latest - cache-from: type=gha,scope=${{ github.ref_name }}-buster - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-buster + cache-from: type=gha,scope=${{ github.ref }}-buster + cache-to: type=gha,mode=max,scope=${{ github.ref }}-buster build-args: | GIT_BRANCH=${{ github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} - + - name: Run run_installation_tests.sh Buster ARMv7 uses: tj-actions/docker-run@v2 with: @@ -49,7 +49,7 @@ jobs: name: run_installation_tests.sh args: | ./run_installation_tests.sh - + - name: Run run_installation_tests2.sh Buster ARMv7 uses: tj-actions/docker-run@v2 with: @@ -58,7 +58,7 @@ jobs: name: run_installation_tests2.sh args: | ./run_installation_tests2.sh - + - name: Run run_installation_tests3.sh Buster ARMv7 uses: tj-actions/docker-run@v2 with: From 6f950478a0ce2f82c7b68debf4f60d806e6c66cb Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Mon, 16 Oct 2023 01:44:40 +0200 Subject: [PATCH 09/17] get username with whoami to be compatible with containers --- .../buster-install-default-with-autohotspot.sh | 6 +++--- scripts/installscripts/buster-install-default.sh | 8 ++++---- scripts/installscripts/tests/test_installation.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/installscripts/buster-install-default-with-autohotspot.sh b/scripts/installscripts/buster-install-default-with-autohotspot.sh index 917030b35..948b3ebbe 100644 --- a/scripts/installscripts/buster-install-default-with-autohotspot.sh +++ b/scripts/installscripts/buster-install-default-with-autohotspot.sh @@ -28,7 +28,7 @@ DATETIME=$(date +"%Y%m%d_%H%M%S") SCRIPTNAME="$(basename $0)" JOB="${SCRIPTNAME}" -CURRENT_USER="${SUDO_USER:-$USER}" +CURRENT_USER="${SUDO_USER:-$(whoami)}" HOME_DIR=$(getent passwd "$CURRENT_USER" | cut -d: -f6) echo "Current User: $CURRENT_USER" echo "User home dir: $HOME_DIR" @@ -97,7 +97,7 @@ checkPrerequisite() { #currently the user 'pi' is mandatory #https://github.com/MiczFlor/RPi-Jukebox-RFID/issues/1785 if [ "${CURRENT_USER}" != "pi" ]; then - echo + echo echo "ERROR: User must be 'pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -105,7 +105,7 @@ checkPrerequisite() { fi if [ "${HOME_DIR}" != "/home/pi" ]; then - echo + echo echo "ERROR: HomeDir must be '/home/pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" diff --git a/scripts/installscripts/buster-install-default.sh b/scripts/installscripts/buster-install-default.sh index d46e61e7d..b5c9108bc 100644 --- a/scripts/installscripts/buster-install-default.sh +++ b/scripts/installscripts/buster-install-default.sh @@ -28,7 +28,7 @@ DATETIME=$(date +"%Y%m%d_%H%M%S") SCRIPTNAME="$(basename $0)" JOB="${SCRIPTNAME}" -CURRENT_USER="${SUDO_USER:-$USER}" +CURRENT_USER="${SUDO_USER:-$(whoami)}" HOME_DIR=$(getent passwd "$CURRENT_USER" | cut -d: -f6) echo "Current User: $CURRENT_USER" echo "User home dir: $HOME_DIR" @@ -97,7 +97,7 @@ checkPrerequisite() { #currently the user 'pi' is mandatory #https://github.com/MiczFlor/RPi-Jukebox-RFID/issues/1785 if [ "${CURRENT_USER}" != "pi" ]; then - echo + echo echo "ERROR: User must be 'pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -105,7 +105,7 @@ checkPrerequisite() { fi if [ "${HOME_DIR}" != "/home/pi" ]; then - echo + echo echo "ERROR: HomeDir must be '/home/pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -864,7 +864,7 @@ install_main() { sudo wget -q -O /etc/apt/keyrings/mopidy-archive-keyring.gpg https://apt.mopidy.com/mopidy.gpg sudo wget -q -O /etc/apt/sources.list.d/mopidy.list https://apt.mopidy.com/${OS_CODENAME}.list - + ${apt_get} update ${apt_get} upgrade ${apt_get} install libspotify-dev diff --git a/scripts/installscripts/tests/test_installation.sh b/scripts/installscripts/tests/test_installation.sh index fddeafa2e..332cfb8cb 100755 --- a/scripts/installscripts/tests/test_installation.sh +++ b/scripts/installscripts/tests/test_installation.sh @@ -7,8 +7,8 @@ INSTALLATION_EXITCODE="${1:-0}" PATHDATA="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -USER_NAME="$USER" -HOME_DIR="$HOME" +USER_NAME="$(whoami)" +HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6) tests=0 failed_tests=0 From 6160b75aff67fc6e144f10c00b0b6c6b1965a05c Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Tue, 17 Oct 2023 20:10:37 +0200 Subject: [PATCH 10/17] implemented different strategies for uniformity --- .github/workflows/docker_bullseye_cache.yml | 95 +++++++++++++ .github/workflows/docker_bullseye_target.yml | 97 +++++++++++++ .github/workflows/docker_bullseye_unified.yml | 129 ++++++++++++++++++ .github/workflows/docker_bullseye_upload.yml | 95 +++++++++++++ ...erfile.bullseye.test_install_targets.armv7 | 55 ++++++++ ...erfile.bullseye.test_install_unified.armv7 | 73 ++++++++++ 6 files changed, 544 insertions(+) create mode 100644 .github/workflows/docker_bullseye_cache.yml create mode 100644 .github/workflows/docker_bullseye_target.yml create mode 100644 .github/workflows/docker_bullseye_unified.yml create mode 100644 .github/workflows/docker_bullseye_upload.yml create mode 100644 ci/Dockerfile.bullseye.test_install_targets.armv7 create mode 100644 ci/Dockerfile.bullseye.test_install_unified.armv7 diff --git a/.github/workflows/docker_bullseye_cache.yml b/.github/workflows/docker_bullseye_cache.yml new file mode 100644 index 000000000..db8600a69 --- /dev/null +++ b/.github/workflows/docker_bullseye_cache.yml @@ -0,0 +1,95 @@ +name: Test Install Scripts Cache + +on: + schedule: + # run at 5 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +env: + DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye + TEMP_DOCKER_PATH: /tmp/ + CACHE_KEY_SUFFIX: -${{ github.sha }}-${{ github.run_attempt }} + + +jobs: + + prepare: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Build Bullseye ARMv7 + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + file: ./ci/Dockerfile.bullseye.test_install.armv7 + platforms: linux/arm/v7 + tags: local/${{ env.DOCKER_IMAGE_NAME }}:cache + cache-from: type=gha,scope=${{ github.ref }}-cache + cache-to: type=gha,mode=max,scope=${{ github.ref }}-cache + outputs: type=docker,dest=${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + build-args: | + USER_NAME=hans + USER_GROUP=wurst + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + - name: Cache Save Docker Image + uses: actions/cache/save@v3 + with: + path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + key: ${{ env.DOCKER_IMAGE_NAME }}${{ env.CACHE_KEY_SUFFIX }} + + + test: + + needs: prepare + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Cache Restore Docker Image + uses: actions/cache/restore@v3 + with: + path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + key: ${{ env.DOCKER_IMAGE_NAME }}${{ env.CACHE_KEY_SUFFIX }} + fail-on-cache-miss: true + + - name: Load Docker Image + run: | + docker load --input ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + + - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: local/${{ env.DOCKER_IMAGE_NAME }}:cache + options: --platform linux/arm/v7 + name: ${{ matrix.test_Script }} + args: | + ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_target.yml b/.github/workflows/docker_bullseye_target.yml new file mode 100644 index 000000000..e12c309a2 --- /dev/null +++ b/.github/workflows/docker_bullseye_target.yml @@ -0,0 +1,97 @@ +name: Test Install Scripts Target + +on: + schedule: + # run at 5 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +env: + DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye + + +jobs: + + prepare: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Build Bullseye ARMv7 + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + file: ./ci/Dockerfile.bullseye.test_install_targets.armv7 + target: base + platforms: linux/arm/v7 + tags: local/${{ env.DOCKER_IMAGE_NAME }}_base:cache + cache-from: type=gha,scope=${{ github.ref }}-target + cache-to: type=gha,mode=max,scope=${{ github.ref }}-target + + + test: + + needs: prepare + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] + username: [hans] #[pi, hans] + include: + # - username: pi + # usergroup: pi + - username: hans + usergroup: wurst + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Build Bullseye ARMv7 + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + file: ./ci/Dockerfile.bullseye.test_install_targets.armv7 + target: user + platforms: linux/arm/v7 + tags: local/${{ env.DOCKER_IMAGE_NAME }}_user:cache + cache-from: type=gha,scope=${{ github.ref }}-target + build-args: | + USER_NAME=${{ matrix.username }} + USER_GROUP=${{ matrix.usergroup }} + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: local/${{ env.DOCKER_IMAGE_NAME }}_user:cache + options: --platform linux/arm/v7 + name: ${{ matrix.test_Script }} + args: | + ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_unified.yml b/.github/workflows/docker_bullseye_unified.yml new file mode 100644 index 000000000..1bb7ddc75 --- /dev/null +++ b/.github/workflows/docker_bullseye_unified.yml @@ -0,0 +1,129 @@ +name: Test Install Scripts Target Unified + +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ github.ref }}-unified + cancel-in-progress: true + +on: + schedule: + # run at 5 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +env: + DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye + _PATH_DOCKER_IMAGE_TAR: ./rpi-jukebox-rfid-bullseye.tar + CACHE_SCOPE: ${{ github.ref }}-unified + +jobs: + prepare: + runs-on: ubuntu-latest + + outputs: + image_name: ${{ steps.get-image-name.outputs.image_name }} + cache_key: ${{ steps.cache-restore.outputs.cache-primary-key }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Build Bullseye ARMv7 - Base + id: docker-build-base + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: false + file: ./ci/Dockerfile.bullseye.test_install_unified.armv7 + target: user + platforms: linux/arm/v7 + tags: ${{ env.DOCKER_IMAGE_NAME }}:test-base + cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} + cache-to: type=gha,mode=max,scope=${{ env.CACHE_SCOPE }} + + - name: Set output image_name + id: get-image-name + run: echo "image_name=${{ env.DOCKER_IMAGE_NAME }}:test-update" >> $GITHUB_OUTPUT + + - name: Build Bullseye ARMv7 - Update + id: docker-build-update + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: false + file: ./ci/Dockerfile.bullseye.test_install_unified.armv7 + target: test-update + platforms: linux/arm/v7 + tags: ${{ steps.get-image-name.outputs.image_name }} + cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} + # DON'T use 'cache-to' here as then the layer is cached and this build would be useless + outputs: type=docker,dest=${{ env._PATH_DOCKER_IMAGE_TAR }} + build-args: | + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + - name: Cache Check Docker Image + uses: actions/cache/restore@v3 + id: cache-restore + with: + lookup-only: true + key: ${{ env.DOCKER_IMAGE_NAME }}-${{ steps.docker-build-update.outputs.imageid }} + path: ${{ env._PATH_DOCKER_IMAGE_TAR }} + + - name: Cache Save Docker Image + uses: actions/cache/save@v3 + if: ${{ ! steps.cache-restore.outputs.cache-hit }} + with: + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + path: ${{ env._PATH_DOCKER_IMAGE_TAR }} + + + test: + needs: prepare + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] + username: [pi, hans] + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Cache Restore Docker Image + uses: actions/cache/restore@v3 + with: + key: ${{ needs.prepare.outputs.cache_key }} + path: ${{ env._PATH_DOCKER_IMAGE_TAR }} + fail-on-cache-miss: true + + - name: Load Docker Image + run: | + docker load --input ${{ env._PATH_DOCKER_IMAGE_TAR }} + + - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: ${{ needs.prepare.outputs.image_name }} + options: --platform linux/arm/v7 --user ${{ matrix.username }} + name: ${{ matrix.test_Script }} + args: | + ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_upload.yml b/.github/workflows/docker_bullseye_upload.yml new file mode 100644 index 000000000..f8857c721 --- /dev/null +++ b/.github/workflows/docker_bullseye_upload.yml @@ -0,0 +1,95 @@ +name: Test Install Scripts Upload + +on: + schedule: + # run at 5 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +env: + DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye + TEMP_DOCKER_PATH: /tmp/ + + +jobs: + + prepare: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Build Bullseye ARMv7 + uses: docker/build-push-action@v5 + with: + context: . + load: true + push: false + file: ./ci/Dockerfile.bullseye.test_install.armv7 + platforms: linux/arm/v7 + tags: local/${{ env.DOCKER_IMAGE_NAME }}:cache + cache-from: type=gha,scope=${{ github.ref }}-upload + cache-to: type=gha,mode=max,scope=${{ github.ref }}-upload + outputs: type=docker,dest=${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + build-args: | + USER_NAME=hans + USER_GROUP=wurst + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + - name: Upload Docker Build Image + uses: actions/upload-artifact@v3 + with: + name: ${{ env.DOCKER_IMAGE_NAME }}.tar + path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + + + test: + + needs: prepare + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] + + steps: + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Download Docker Build Image + uses: actions/download-artifact@v3 + with: + name: ${{ env.DOCKER_IMAGE_NAME }}.tar + path: ${{ env.TEMP_DOCKER_PATH }} + + - name: Load Docker Build Image + run: | + docker load --input ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar + + - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 + uses: tj-actions/docker-run@v2 + with: + image: local/${{ env.DOCKER_IMAGE_NAME }}:cache + options: --platform linux/arm/v7 + name: ${{ matrix.test_Script }} + args: | + ./${{ matrix.test_Script }} diff --git a/ci/Dockerfile.bullseye.test_install_targets.armv7 b/ci/Dockerfile.bullseye.test_install_targets.armv7 new file mode 100644 index 000000000..180f26228 --- /dev/null +++ b/ci/Dockerfile.bullseye.test_install_targets.armv7 @@ -0,0 +1,55 @@ +FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim as base + +# Define constants +ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive +RUN touch /boot/cmdlinetxt + +# Install packages +RUN apt-get update && \ + apt-get -y install \ + apt-utils \ + curl \ + gnupg + +RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ + echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ + curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ + curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ + apt-get update && \ + apt-get -y install \ + build-essential \ + git \ + iw \ + locales \ + # install here to speed up GitHub Action + raspberrypi-kernel-headers \ + sudo \ + systemd \ + wget \ + wpasupplicant ;\ + rm -rf /var/lib/apt/lists/* + + +FROM base as user +# Define user info +ARG USER_NAME=pi +ARG USER_GROUP=$USER_NAME +# Define Git Repo variables for installationscript. +ARG GIT_BRANCH +ARG GIT_URL + +ENV USER=$USER_NAME + +RUN groupadd --gid 1000 $USER_GROUP && \ + useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ + echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER + +USER $USER + +# Define needed files for installation test +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ +COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / + +ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL + +WORKDIR /tests diff --git a/ci/Dockerfile.bullseye.test_install_unified.armv7 b/ci/Dockerfile.bullseye.test_install_unified.armv7 new file mode 100644 index 000000000..d89342913 --- /dev/null +++ b/ci/Dockerfile.bullseye.test_install_unified.armv7 @@ -0,0 +1,73 @@ +# Target to build and install all needed base configuration and packages +FROM debian:bullseye-slim as base + +ENV DOCKER_RUNNING=true +ENV DEBIAN_FRONTEND=noninteractive +RUN touch /boot/cmdlinetxt + +RUN apt-get update \ + && apt-get -y install \ + apt-utils \ + curl \ + gnupg \ + && echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list \ + && echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list \ + && curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - \ + && curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \ + && apt-get update \ + && apt-get -y upgrade \ + && apt-get -y install \ + build-essential \ + git \ + iw \ + locales \ + sudo \ + systemd \ + wget \ + wpasupplicant \ + # install internally used packages here to speed up GitHub Action + raspberrypi-kernel-headers \ + && rm -rf /var/lib/apt/lists/* +# ------ + + +# Target for setting up user for test. user can be selected with the docker '--user $USERNAME' option +FROM base as user +ENV TEST_USER_GROUP=test + + # Group used for assigning file permissions +RUN groupadd --gid 1002 $TEST_USER_GROUP + +RUN export USER_PI=pi \ + && export USER_PI_GROUP=$USER_PI \ + && groupadd --gid 1000 $USER_PI_GROUP \ + && useradd -u 1000 -g $USER_PI_GROUP -G sudo,$TEST_USER_GROUP -d /home/$USER_PI -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER_PI \ + && echo "$USER_PI ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER_PI + +RUN export USER_ALT=hans \ + && export USER_ALT_GROUP=wurst \ + && groupadd --gid 1001 $USER_ALT_GROUP \ + && useradd -u 1001 -g $USER_ALT_GROUP -G sudo,$TEST_USER_GROUP -d /home/$USER_ALT -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER_ALT \ + && echo "$USER_ALT ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER_ALT +# ------ + + +# Target for adding envs and scripts from the repo to test installation +FROM user as test +ARG GIT_BRANCH +ARG GIT_URL + +ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL + +COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/buster-install-default.sh . +WORKDIR /tests +COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh . +# ------ + + +# Target for applying latest updates (should not be cached!) +FROM test as test-update +RUN apt-get update \ + && apt-get -y upgrade \ + && rm -rf /var/lib/apt/lists/* +# ------ From c05da89a4299ef7c3c4e30dd36152d88330c4624 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Tue, 17 Oct 2023 20:12:51 +0200 Subject: [PATCH 11/17] set noninteractive on debconf get rid of warning "debconf: unable to initialize frontend: Dialog" export DEBIAN_FRONTEND=noninteractive does not work with sudo --- scripts/installscripts/tests/run_installation_tests.sh | 3 ++- scripts/installscripts/tests/run_installation_tests2.sh | 1 + scripts/installscripts/tests/run_installation_tests3.sh | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/installscripts/tests/run_installation_tests.sh b/scripts/installscripts/tests/run_installation_tests.sh index 41fa3da3c..fb711048b 100644 --- a/scripts/installscripts/tests/run_installation_tests.sh +++ b/scripts/installscripts/tests/run_installation_tests.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive @@ -21,7 +22,7 @@ export DEBIAN_FRONTEND=noninteractive # n no spotify # y configure mpd # y audio default location -# y config gpio +# y config gpio # n no RFID registration # n No reboot diff --git a/scripts/installscripts/tests/run_installation_tests2.sh b/scripts/installscripts/tests/run_installation_tests2.sh index 6162df3b0..58c2a378d 100644 --- a/scripts/installscripts/tests/run_installation_tests2.sh +++ b/scripts/installscripts/tests/run_installation_tests2.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive diff --git a/scripts/installscripts/tests/run_installation_tests3.sh b/scripts/installscripts/tests/run_installation_tests3.sh index d983a2db1..09340da8e 100644 --- a/scripts/installscripts/tests/run_installation_tests3.sh +++ b/scripts/installscripts/tests/run_installation_tests3.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive From bc7e9c212bd20931772b26c87e0cd417200b5a41 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:19:09 +0200 Subject: [PATCH 12/17] unified workflow for debian. delete obsolete files --- .github/workflows/docker_bullseye.yml | 69 ------- .github/workflows/docker_bullseye_altuser.yml | 71 ------- .github/workflows/docker_bullseye_cache.yml | 95 --------- .github/workflows/docker_bullseye_target.yml | 97 --------- .github/workflows/docker_bullseye_unified.yml | 129 ------------ .github/workflows/docker_bullseye_upload.yml | 95 --------- .github/workflows/docker_buster.yml | 69 ------- .github/workflows/docker_debian.yml | 185 ++++++++++++++++++ ci/Dockerfile.bullseye.test_install.amd64 | 44 ----- ci/Dockerfile.bullseye.test_install.armv7 | 53 ----- ...erfile.bullseye.test_install_targets.armv7 | 55 ------ ci/Dockerfile.buster.test_install.armv7 | 53 ----- ...7 => Dockerfile.debian.test_install.armv7} | 16 +- 13 files changed, 195 insertions(+), 836 deletions(-) delete mode 100644 .github/workflows/docker_bullseye.yml delete mode 100644 .github/workflows/docker_bullseye_altuser.yml delete mode 100644 .github/workflows/docker_bullseye_cache.yml delete mode 100644 .github/workflows/docker_bullseye_target.yml delete mode 100644 .github/workflows/docker_bullseye_unified.yml delete mode 100644 .github/workflows/docker_bullseye_upload.yml delete mode 100644 .github/workflows/docker_buster.yml create mode 100644 .github/workflows/docker_debian.yml delete mode 100644 ci/Dockerfile.bullseye.test_install.amd64 delete mode 100644 ci/Dockerfile.bullseye.test_install.armv7 delete mode 100644 ci/Dockerfile.bullseye.test_install_targets.armv7 delete mode 100644 ci/Dockerfile.buster.test_install.armv7 rename ci/{Dockerfile.bullseye.test_install_unified.armv7 => Dockerfile.debian.test_install.armv7} (82%) diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml deleted file mode 100644 index 039c8f5ef..000000000 --- a/.github/workflows/docker_bullseye.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Test Install Scripts for Bullseye on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-bullseye:latest - cache-from: type=gha,scope=${{ github.ref }}-bullseye - cache-to: type=gha,mode=max,scope=${{ github.ref }}-bullseye - build-args: | - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Run run_installation_tests.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - ./run_installation_tests.sh - - - name: Run run_installation_tests2.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - ./run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - ./run_installation_tests3.sh diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml deleted file mode 100644 index 60ab0fde2..000000000 --- a/.github/workflows/docker_bullseye_altuser.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Test Install Scripts for Bullseye (alternative user) on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 altuser - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-bullseye-altuser:latest - cache-from: type=gha,scope=${{ github.ref }}-bullseye-altuser - cache-to: type=gha,mode=max,scope=${{ github.ref }}-bullseye-altuser - build-args: | - USER_NAME=hans - USER_GROUP=wurst - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Run run_installation_tests.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - ./run_installation_tests.sh - - - name: Run run_installation_tests2.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - ./run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - ./run_installation_tests3.sh diff --git a/.github/workflows/docker_bullseye_cache.yml b/.github/workflows/docker_bullseye_cache.yml deleted file mode 100644 index db8600a69..000000000 --- a/.github/workflows/docker_bullseye_cache.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Test Install Scripts Cache - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -env: - DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye - TEMP_DOCKER_PATH: /tmp/ - CACHE_KEY_SUFFIX: -${{ github.sha }}-${{ github.run_attempt }} - - -jobs: - - prepare: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install.armv7 - platforms: linux/arm/v7 - tags: local/${{ env.DOCKER_IMAGE_NAME }}:cache - cache-from: type=gha,scope=${{ github.ref }}-cache - cache-to: type=gha,mode=max,scope=${{ github.ref }}-cache - outputs: type=docker,dest=${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - build-args: | - USER_NAME=hans - USER_GROUP=wurst - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Cache Save Docker Image - uses: actions/cache/save@v3 - with: - path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - key: ${{ env.DOCKER_IMAGE_NAME }}${{ env.CACHE_KEY_SUFFIX }} - - - test: - - needs: prepare - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] - - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Cache Restore Docker Image - uses: actions/cache/restore@v3 - with: - path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - key: ${{ env.DOCKER_IMAGE_NAME }}${{ env.CACHE_KEY_SUFFIX }} - fail-on-cache-miss: true - - - name: Load Docker Image - run: | - docker load --input ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - - - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: local/${{ env.DOCKER_IMAGE_NAME }}:cache - options: --platform linux/arm/v7 - name: ${{ matrix.test_Script }} - args: | - ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_target.yml b/.github/workflows/docker_bullseye_target.yml deleted file mode 100644 index e12c309a2..000000000 --- a/.github/workflows/docker_bullseye_target.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Test Install Scripts Target - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -env: - DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye - - -jobs: - - prepare: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install_targets.armv7 - target: base - platforms: linux/arm/v7 - tags: local/${{ env.DOCKER_IMAGE_NAME }}_base:cache - cache-from: type=gha,scope=${{ github.ref }}-target - cache-to: type=gha,mode=max,scope=${{ github.ref }}-target - - - test: - - needs: prepare - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] - username: [hans] #[pi, hans] - include: - # - username: pi - # usergroup: pi - - username: hans - usergroup: wurst - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install_targets.armv7 - target: user - platforms: linux/arm/v7 - tags: local/${{ env.DOCKER_IMAGE_NAME }}_user:cache - cache-from: type=gha,scope=${{ github.ref }}-target - build-args: | - USER_NAME=${{ matrix.username }} - USER_GROUP=${{ matrix.usergroup }} - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: local/${{ env.DOCKER_IMAGE_NAME }}_user:cache - options: --platform linux/arm/v7 - name: ${{ matrix.test_Script }} - args: | - ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_unified.yml b/.github/workflows/docker_bullseye_unified.yml deleted file mode 100644 index 1bb7ddc75..000000000 --- a/.github/workflows/docker_bullseye_unified.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: Test Install Scripts Target Unified - -# let only one instance run the test so cache is not corrupted. -# cancel already running instances as only the last run will be relevant -concurrency: - group: ${{ github.ref }}-unified - cancel-in-progress: true - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -env: - DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye - _PATH_DOCKER_IMAGE_TAR: ./rpi-jukebox-rfid-bullseye.tar - CACHE_SCOPE: ${{ github.ref }}-unified - -jobs: - prepare: - runs-on: ubuntu-latest - - outputs: - image_name: ${{ steps.get-image-name.outputs.image_name }} - cache_key: ${{ steps.cache-restore.outputs.cache-primary-key }} - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - Base - id: docker-build-base - uses: docker/build-push-action@v5 - with: - context: . - load: false - push: false - file: ./ci/Dockerfile.bullseye.test_install_unified.armv7 - target: user - platforms: linux/arm/v7 - tags: ${{ env.DOCKER_IMAGE_NAME }}:test-base - cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} - cache-to: type=gha,mode=max,scope=${{ env.CACHE_SCOPE }} - - - name: Set output image_name - id: get-image-name - run: echo "image_name=${{ env.DOCKER_IMAGE_NAME }}:test-update" >> $GITHUB_OUTPUT - - - name: Build Bullseye ARMv7 - Update - id: docker-build-update - uses: docker/build-push-action@v5 - with: - context: . - load: false - push: false - file: ./ci/Dockerfile.bullseye.test_install_unified.armv7 - target: test-update - platforms: linux/arm/v7 - tags: ${{ steps.get-image-name.outputs.image_name }} - cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} - # DON'T use 'cache-to' here as then the layer is cached and this build would be useless - outputs: type=docker,dest=${{ env._PATH_DOCKER_IMAGE_TAR }} - build-args: | - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Cache Check Docker Image - uses: actions/cache/restore@v3 - id: cache-restore - with: - lookup-only: true - key: ${{ env.DOCKER_IMAGE_NAME }}-${{ steps.docker-build-update.outputs.imageid }} - path: ${{ env._PATH_DOCKER_IMAGE_TAR }} - - - name: Cache Save Docker Image - uses: actions/cache/save@v3 - if: ${{ ! steps.cache-restore.outputs.cache-hit }} - with: - key: ${{ steps.cache-restore.outputs.cache-primary-key }} - path: ${{ env._PATH_DOCKER_IMAGE_TAR }} - - - test: - needs: prepare - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] - username: [pi, hans] - - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Cache Restore Docker Image - uses: actions/cache/restore@v3 - with: - key: ${{ needs.prepare.outputs.cache_key }} - path: ${{ env._PATH_DOCKER_IMAGE_TAR }} - fail-on-cache-miss: true - - - name: Load Docker Image - run: | - docker load --input ${{ env._PATH_DOCKER_IMAGE_TAR }} - - - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: ${{ needs.prepare.outputs.image_name }} - options: --platform linux/arm/v7 --user ${{ matrix.username }} - name: ${{ matrix.test_Script }} - args: | - ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_bullseye_upload.yml b/.github/workflows/docker_bullseye_upload.yml deleted file mode 100644 index f8857c721..000000000 --- a/.github/workflows/docker_bullseye_upload.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Test Install Scripts Upload - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -env: - DOCKER_IMAGE_NAME: rpi-jukebox-rfid-bullseye - TEMP_DOCKER_PATH: /tmp/ - - -jobs: - - prepare: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install.armv7 - platforms: linux/arm/v7 - tags: local/${{ env.DOCKER_IMAGE_NAME }}:cache - cache-from: type=gha,scope=${{ github.ref }}-upload - cache-to: type=gha,mode=max,scope=${{ github.ref }}-upload - outputs: type=docker,dest=${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - build-args: | - USER_NAME=hans - USER_GROUP=wurst - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Upload Docker Build Image - uses: actions/upload-artifact@v3 - with: - name: ${{ env.DOCKER_IMAGE_NAME }}.tar - path: ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - - - test: - - needs: prepare - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - test_Script: [run_installation_tests.sh, run_installation_tests2.sh, run_installation_tests3.sh] - - steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Download Docker Build Image - uses: actions/download-artifact@v3 - with: - name: ${{ env.DOCKER_IMAGE_NAME }}.tar - path: ${{ env.TEMP_DOCKER_PATH }} - - - name: Load Docker Build Image - run: | - docker load --input ${{ env.TEMP_DOCKER_PATH }}${{ env.DOCKER_IMAGE_NAME }}.tar - - - name: Run ${{ matrix.test_Script }} Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: local/${{ env.DOCKER_IMAGE_NAME }}:cache - options: --platform linux/arm/v7 - name: ${{ matrix.test_Script }} - args: | - ./${{ matrix.test_Script }} diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml deleted file mode 100644 index cec1f2509..000000000 --- a/.github/workflows/docker_buster.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Test Install Scripts for Buster on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Buster ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.buster.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-buster:latest - cache-from: type=gha,scope=${{ github.ref }}-buster - cache-to: type=gha,mode=max,scope=${{ github.ref }}-buster - build-args: | - GIT_BRANCH=${{ github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - - name: Run run_installation_tests.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - ./run_installation_tests.sh - - - name: Run run_installation_tests2.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - ./run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - ./run_installation_tests3.sh diff --git a/.github/workflows/docker_debian.yml b/.github/workflows/docker_debian.yml new file mode 100644 index 000000000..5a2d6cea5 --- /dev/null +++ b/.github/workflows/docker_debian.yml @@ -0,0 +1,185 @@ +name: Test Install Scripts Debian + +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ github.ref }}-debian + cancel-in-progress: true + +on: + schedule: + # run at 5:00 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +env: + DOCKER_IMAGE_NAME: rpi-jukebox-rfid + CACHE_SCOPE: ${{ github.ref }}-debian + MATRIX_DEBIAN_VERSION_NAME: "['bullseye', 'buster']" + MATRIX_USERNAME: "['pi', 'hans']" + MATRIX_TEST_SCRIPT: "['run_installation_tests.sh', 'run_installation_tests2.sh', 'run_installation_tests3.sh']" + +jobs: + + # Prepare variables for the next jobs + prepare: + runs-on: ubuntu-latest + + outputs: + runs_on: ubuntu-latest + platform: linux/arm/v7 + + matrix_debian_version_name: ${{ env.MATRIX_DEBIAN_VERSION_NAME }} + matrix_username: ${{ env.MATRIX_USERNAME }} + matrix_test_script: ${{ env.MATRIX_TEST_SCRIPT }} + + image_tag_name: ${{ steps.set-image_tag_name.outputs.image_tag_name }} + image_tag_name_local_base: ${{ steps.set-image_tag_name_local_base.outputs.image_tag_name_local_base }} + image_file_path: ${{ steps.set-image_file_path.outputs.image_file_path }} + cache_scope: ${{ steps.set-cache_scope.outputs.cache_scope }} + cache_key: ${{ steps.set-cache_key.outputs.cache_key }} + local_registry_port: ${{ steps.set-local_registry_port.outputs.local_registry_port }} + + steps: + - name: Set Output cache_scope + id: set-cache_scope + run: echo "cache_scope=${{ env.CACHE_SCOPE }}-{0}" >> $GITHUB_OUTPUT + + - name: Set Output image_tag_name + id: set-image_tag_name + run: echo "image_tag_name=${{ env.DOCKER_IMAGE_NAME }}:{0}-test" >> $GITHUB_OUTPUT + + - name: Set Output image_file_path + id: set-image_file_path + run: echo "image_file_path=./${{ env.DOCKER_IMAGE_NAME }}-{0}.tar" >> $GITHUB_OUTPUT + + - name: Set Output cache_key + id: set-cache_key + run: echo "cache_key=${{ steps.set-cache_scope.outputs.cache_scope }}-${{ github.sha }}#${{ github.run_attempt }}" >> $GITHUB_OUTPUT + + - name: Set Output local_registry_port + id: set-local_registry_port + run: echo "local_registry_port=5000" >> $GITHUB_OUTPUT + + - name: Set Output image_tag_name_local_base + id: set-image_tag_name_local_base + run: echo "image_tag_name_local_base=localhost:${{ steps.set-local_registry_port.outputs.local_registry_port }}/${{ steps.set-image_tag_name.outputs.image_tag_name }}-base" >> $GITHUB_OUTPUT + + + # Build container for test execution + build: + needs: [prepare] + runs-on: ${{ needs.prepare.outputs.runs_on }} + + # create local docker registry to use locally build images + services: + registry: + image: registry:2 + ports: + - ${{ needs.prepare.outputs.local_registry_port }}:5000 + + strategy: + fail-fast: false + matrix: + debian_version_name: ${{ fromJSON(needs.prepare.outputs.matrix_debian_version_name) }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + with: + # network=host driver-opt needed to push to local registry + driver-opts: network=host + + # Build base image for debian version name. Layers will be cached and image pushes to local registry + - name: Build Image - Base + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: true + file: ./ci/Dockerfile.debian.test_install.armv7 + target: test + platforms: ${{ needs.prepare.outputs.platform }} + tags: ${{ format( needs.prepare.outputs.image_tag_name_local_base, matrix.debian_version_name) }} + cache-from: type=gha,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} + cache-to: type=gha,mode=max,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} + build-args: | + DEBIAN_VERSION_NAME=${{ matrix.debian_version_name }} + GIT_BRANCH=${{ github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. + - name: Build Image - Update + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: false + file: ./ci/Dockerfile.debian.test_install.armv7 + target: test-update + platforms: ${{ needs.prepare.outputs.platform }} + tags: ${{ format( needs.prepare.outputs.image_tag_name, matrix.debian_version_name) }} + cache-from: type=gha,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} + # DON'T use 'cache-to' here as the layer is then cached and this build would be useless + outputs: type=docker,dest=${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} + build-args: | + BASE_TEST_IMAGE=${{ format( needs.prepare.outputs.image_tag_name_local_base, matrix.debian_version_name) }} + + # Cache image file for next jobs + - name: Cache Save Docker Image + uses: actions/cache/save@v3 + with: + key: ${{ format( needs.prepare.outputs.cache_key, matrix.debian_version_name) }} + path: ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} + + + # Run tests with build image + test: + needs: [prepare, build] + runs-on: ${{ needs.prepare.outputs.runs_on }} + + strategy: + fail-fast: false + matrix: + debian_version_name: ${{ fromJSON(needs.prepare.outputs.matrix_debian_version_name) }} + username: ${{ fromJSON(needs.prepare.outputs.matrix_username) }} + test_script: ${{ fromJSON(needs.prepare.outputs.matrix_test_script) }} + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + # Load cached image file + - name: Cache Restore Docker Image + uses: actions/cache/restore@v3 + with: + key: ${{ format( needs.prepare.outputs.cache_key, matrix.debian_version_name) }} + path: ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} + fail-on-cache-miss: true + + - name: Load Docker Image + run: | + docker load --input ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} + + # Run test + - name: Run Test ${{ matrix.debian_version_name }}-${{ matrix.username }}-${{ matrix.test_script }} + uses: tj-actions/docker-run@v2 + with: + image: ${{ format( needs.prepare.outputs.image_tag_name, matrix.debian_version_name) }} + options: --platform ${{ needs.prepare.outputs.platform }} --user ${{ matrix.username }} + name: ${{ matrix.test_script }} + args: | + ./${{ matrix.test_script }} diff --git a/ci/Dockerfile.bullseye.test_install.amd64 b/ci/Dockerfile.bullseye.test_install.amd64 deleted file mode 100644 index 39e15907c..000000000 --- a/ci/Dockerfile.bullseye.test_install.amd64 +++ /dev/null @@ -1,44 +0,0 @@ -FROM debian:bullseye -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME -ARG GIT_BRANCH -ARG GIT_URL - -ENV DOCKER_RUNNING=true -ENV USER=$USER_NAME -ENV GIT_BRANCH=$GIT_BRANCH -ENV GIT_URL=$GIT_URL - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER_GROUP ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils ;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant ;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 deleted file mode 100644 index 07de9782e..000000000 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ /dev/null @@ -1,53 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim - -# Define constants -ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive -RUN touch /boot/cmdlinetxt - -# Define Git Repo variables. If this changes all should be build again -ARG GIT_BRANCH -ARG GIT_URL - -ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL - -# Install packages -RUN apt-get update && \ - apt-get -y install \ - apt-utils \ - curl \ - gnupg - -RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - apt-get update && \ - apt-get -y install \ - build-essential \ - git \ - iw \ - locales \ - # install here to speed up GitHub Action - raspberrypi-kernel-headers \ - sudo \ - systemd \ - wget \ - wpasupplicant ;\ - rm -rf /var/lib/apt/lists/* - -# Define user info that my change, but are irrelevant for package installation -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME -ENV USER=$USER_NAME - -RUN groupadd --gid 1000 $USER_GROUP && \ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER - -USER $USER - -# Define needed files for installation test -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / - -WORKDIR /tests diff --git a/ci/Dockerfile.bullseye.test_install_targets.armv7 b/ci/Dockerfile.bullseye.test_install_targets.armv7 deleted file mode 100644 index 180f26228..000000000 --- a/ci/Dockerfile.bullseye.test_install_targets.armv7 +++ /dev/null @@ -1,55 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim as base - -# Define constants -ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive -RUN touch /boot/cmdlinetxt - -# Install packages -RUN apt-get update && \ - apt-get -y install \ - apt-utils \ - curl \ - gnupg - -RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - apt-get update && \ - apt-get -y install \ - build-essential \ - git \ - iw \ - locales \ - # install here to speed up GitHub Action - raspberrypi-kernel-headers \ - sudo \ - systemd \ - wget \ - wpasupplicant ;\ - rm -rf /var/lib/apt/lists/* - - -FROM base as user -# Define user info -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME -# Define Git Repo variables for installationscript. -ARG GIT_BRANCH -ARG GIT_URL - -ENV USER=$USER_NAME - -RUN groupadd --gid 1000 $USER_GROUP && \ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER - -USER $USER - -# Define needed files for installation test -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / - -ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL - -WORKDIR /tests diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 deleted file mode 100644 index d436d5b24..000000000 --- a/ci/Dockerfile.buster.test_install.armv7 +++ /dev/null @@ -1,53 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:buster-slim - -# Define constants -ENV DOCKER_RUNNING=true DEBIAN_FRONTEND=noninteractive -RUN touch /boot/cmdlinetxt - -# Define Git Repo variables. If this changes all should be build again -ARG GIT_BRANCH -ARG GIT_URL - -ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL - -# Install packages -RUN apt-get update && \ - apt-get -y install \ - apt-utils \ - curl \ - gnupg - -RUN echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - apt-get update && \ - apt-get -y install \ - build-essential \ - git \ - iw \ - locales \ - # install here to speed up GitHub Action - raspberrypi-kernel-headers \ - sudo \ - systemd \ - wget \ - wpasupplicant ;\ - rm -rf /var/lib/apt/lists/* - -# Define user info that my change, but are irrelevant for package installation -ARG USER_NAME=pi -ARG USER_GROUP=$USER_NAME -ENV USER=$USER_NAME - -RUN groupadd --gid 1000 $USER_GROUP && \ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER && \ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER - -USER $USER - -# Define needed files for installation test -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/tests/*.sh /tests/ -COPY --chown=$USER:$USER_GROUP --chmod=750 scripts/installscripts/buster-install-default.sh / - -WORKDIR /tests diff --git a/ci/Dockerfile.bullseye.test_install_unified.armv7 b/ci/Dockerfile.debian.test_install.armv7 similarity index 82% rename from ci/Dockerfile.bullseye.test_install_unified.armv7 rename to ci/Dockerfile.debian.test_install.armv7 index d89342913..d434f9c5e 100644 --- a/ci/Dockerfile.bullseye.test_install_unified.armv7 +++ b/ci/Dockerfile.debian.test_install.armv7 @@ -1,17 +1,21 @@ # Target to build and install all needed base configuration and packages -FROM debian:bullseye-slim as base +ARG DEBIAN_VERSION_NAME=bullseye +ARG BASE_TEST_IMAGE=test +FROM debian:${DEBIAN_VERSION_NAME}-slim as base +ARG DEBIAN_VERSION_NAME ENV DOCKER_RUNNING=true -ENV DEBIAN_FRONTEND=noninteractive RUN touch /boot/cmdlinetxt +ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update \ && apt-get -y install \ apt-utils \ curl \ gnupg \ - && echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list \ - && echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list \ + && echo "deb http://raspbian.raspberrypi.org/raspbian/ ${DEBIAN_VERSION_NAME} main contrib non-free rpi" > /etc/apt/sources.list.d/raspi.list \ + && echo "deb http://archive.raspberrypi.org/debian/ ${DEBIAN_VERSION_NAME} main" >> /etc/apt/sources.list.d/raspi.list \ && curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - \ && curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \ && apt-get update \ @@ -26,6 +30,7 @@ RUN apt-get update \ wget \ wpasupplicant \ # install internally used packages here to speed up GitHub Action + raspberrypi-kernel \ raspberrypi-kernel-headers \ && rm -rf /var/lib/apt/lists/* # ------ @@ -35,7 +40,6 @@ RUN apt-get update \ FROM base as user ENV TEST_USER_GROUP=test - # Group used for assigning file permissions RUN groupadd --gid 1002 $TEST_USER_GROUP RUN export USER_PI=pi \ @@ -66,7 +70,7 @@ COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh # Target for applying latest updates (should not be cached!) -FROM test as test-update +FROM $BASE_TEST_IMAGE as test-update RUN apt-get update \ && apt-get -y upgrade \ && rm -rf /var/lib/apt/lists/* From d00551485c4471443ac16ba5b000846358899948 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 19 Oct 2023 01:41:47 +0200 Subject: [PATCH 13/17] fix branch name on pr --- .github/workflows/docker_debian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_debian.yml b/.github/workflows/docker_debian.yml index 5a2d6cea5..925076b1c 100644 --- a/.github/workflows/docker_debian.yml +++ b/.github/workflows/docker_debian.yml @@ -115,7 +115,7 @@ jobs: cache-to: type=gha,mode=max,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} build-args: | DEBIAN_VERSION_NAME=${{ matrix.debian_version_name }} - GIT_BRANCH=${{ github.ref_name }} + GIT_BRANCH=${{ github.head_ref || github.ref_name }} GIT_URL=${{ github.server_url }}/${{ github.repository }} # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. From 3dd5b1c90348a226ac3b9b932f74e172d0c13014 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 19 Oct 2023 06:52:33 +0200 Subject: [PATCH 14/17] refactored into subworkflow --- .github/workflows/docker_debian.yml | 184 ++---------------- .../docker_debian_versionname_sub.yml | 172 ++++++++++++++++ 2 files changed, 190 insertions(+), 166 deletions(-) create mode 100644 .github/workflows/docker_debian_versionname_sub.yml diff --git a/.github/workflows/docker_debian.yml b/.github/workflows/docker_debian.yml index 925076b1c..45cfd1a4e 100644 --- a/.github/workflows/docker_debian.yml +++ b/.github/workflows/docker_debian.yml @@ -1,11 +1,5 @@ name: Test Install Scripts Debian -# let only one instance run the test so cache is not corrupted. -# cancel already running instances as only the last run will be relevant -concurrency: - group: ${{ github.ref }}-debian - cancel-in-progress: true - on: schedule: # run at 5:00 every sunday @@ -17,169 +11,27 @@ on: # The branches below must be a subset of the branches above branches: [ develop ] -env: - DOCKER_IMAGE_NAME: rpi-jukebox-rfid - CACHE_SCOPE: ${{ github.ref }}-debian - MATRIX_DEBIAN_VERSION_NAME: "['bullseye', 'buster']" - MATRIX_USERNAME: "['pi', 'hans']" - MATRIX_TEST_SCRIPT: "['run_installation_tests.sh', 'run_installation_tests2.sh', 'run_installation_tests3.sh']" +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ github.ref }}-debian + cancel-in-progress: true jobs: - # Prepare variables for the next jobs - prepare: - runs-on: ubuntu-latest - - outputs: - runs_on: ubuntu-latest - platform: linux/arm/v7 - - matrix_debian_version_name: ${{ env.MATRIX_DEBIAN_VERSION_NAME }} - matrix_username: ${{ env.MATRIX_USERNAME }} - matrix_test_script: ${{ env.MATRIX_TEST_SCRIPT }} - - image_tag_name: ${{ steps.set-image_tag_name.outputs.image_tag_name }} - image_tag_name_local_base: ${{ steps.set-image_tag_name_local_base.outputs.image_tag_name_local_base }} - image_file_path: ${{ steps.set-image_file_path.outputs.image_file_path }} - cache_scope: ${{ steps.set-cache_scope.outputs.cache_scope }} - cache_key: ${{ steps.set-cache_key.outputs.cache_key }} - local_registry_port: ${{ steps.set-local_registry_port.outputs.local_registry_port }} - - steps: - - name: Set Output cache_scope - id: set-cache_scope - run: echo "cache_scope=${{ env.CACHE_SCOPE }}-{0}" >> $GITHUB_OUTPUT - - - name: Set Output image_tag_name - id: set-image_tag_name - run: echo "image_tag_name=${{ env.DOCKER_IMAGE_NAME }}:{0}-test" >> $GITHUB_OUTPUT - - - name: Set Output image_file_path - id: set-image_file_path - run: echo "image_file_path=./${{ env.DOCKER_IMAGE_NAME }}-{0}.tar" >> $GITHUB_OUTPUT - - - name: Set Output cache_key - id: set-cache_key - run: echo "cache_key=${{ steps.set-cache_scope.outputs.cache_scope }}-${{ github.sha }}#${{ github.run_attempt }}" >> $GITHUB_OUTPUT - - - name: Set Output local_registry_port - id: set-local_registry_port - run: echo "local_registry_port=5000" >> $GITHUB_OUTPUT - - - name: Set Output image_tag_name_local_base - id: set-image_tag_name_local_base - run: echo "image_tag_name_local_base=localhost:${{ steps.set-local_registry_port.outputs.local_registry_port }}/${{ steps.set-image_tag_name.outputs.image_tag_name }}-base" >> $GITHUB_OUTPUT - - - # Build container for test execution - build: - needs: [prepare] - runs-on: ${{ needs.prepare.outputs.runs_on }} - - # create local docker registry to use locally build images - services: - registry: - image: registry:2 - ports: - - ${{ needs.prepare.outputs.local_registry_port }}:5000 - - strategy: - fail-fast: false - matrix: - debian_version_name: ${{ fromJSON(needs.prepare.outputs.matrix_debian_version_name) }} - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - with: - # network=host driver-opt needed to push to local registry - driver-opts: network=host - - # Build base image for debian version name. Layers will be cached and image pushes to local registry - - name: Build Image - Base - uses: docker/build-push-action@v5 - with: - context: . - load: false - push: true - file: ./ci/Dockerfile.debian.test_install.armv7 - target: test - platforms: ${{ needs.prepare.outputs.platform }} - tags: ${{ format( needs.prepare.outputs.image_tag_name_local_base, matrix.debian_version_name) }} - cache-from: type=gha,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} - cache-to: type=gha,mode=max,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} - build-args: | - DEBIAN_VERSION_NAME=${{ matrix.debian_version_name }} - GIT_BRANCH=${{ github.head_ref || github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} - - # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. - - name: Build Image - Update - uses: docker/build-push-action@v5 - with: - context: . - load: false - push: false - file: ./ci/Dockerfile.debian.test_install.armv7 - target: test-update - platforms: ${{ needs.prepare.outputs.platform }} - tags: ${{ format( needs.prepare.outputs.image_tag_name, matrix.debian_version_name) }} - cache-from: type=gha,scope=${{ format( needs.prepare.outputs.cache_scope, matrix.debian_version_name) }} - # DON'T use 'cache-to' here as the layer is then cached and this build would be useless - outputs: type=docker,dest=${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} - build-args: | - BASE_TEST_IMAGE=${{ format( needs.prepare.outputs.image_tag_name_local_base, matrix.debian_version_name) }} - - # Cache image file for next jobs - - name: Cache Save Docker Image - uses: actions/cache/save@v3 - with: - key: ${{ format( needs.prepare.outputs.cache_key, matrix.debian_version_name) }} - path: ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} - - - # Run tests with build image - test: - needs: [prepare, build] - runs-on: ${{ needs.prepare.outputs.runs_on }} - + # Build container and run tests + run: + name: ${{ matrix.debian_version_name }} strategy: fail-fast: false matrix: - debian_version_name: ${{ fromJSON(needs.prepare.outputs.matrix_debian_version_name) }} - username: ${{ fromJSON(needs.prepare.outputs.matrix_username) }} - test_script: ${{ fromJSON(needs.prepare.outputs.matrix_test_script) }} - - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - # Load cached image file - - name: Cache Restore Docker Image - uses: actions/cache/restore@v3 - with: - key: ${{ format( needs.prepare.outputs.cache_key, matrix.debian_version_name) }} - path: ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} - fail-on-cache-miss: true - - - name: Load Docker Image - run: | - docker load --input ${{ format( needs.prepare.outputs.image_file_path, matrix.debian_version_name) }} - - # Run test - - name: Run Test ${{ matrix.debian_version_name }}-${{ matrix.username }}-${{ matrix.test_script }} - uses: tj-actions/docker-run@v2 - with: - image: ${{ format( needs.prepare.outputs.image_tag_name, matrix.debian_version_name) }} - options: --platform ${{ needs.prepare.outputs.platform }} --user ${{ matrix.username }} - name: ${{ matrix.test_script }} - args: | - ./${{ matrix.test_script }} + debian_version_name: ['bullseye', 'buster'] + uses: ./.github/workflows/docker_debian_versionname_sub.yml + with: + runs_on: ubuntu-latest + platform: linux/arm/v7 + docker_image_name: rpi-jukebox-rfid + cache_scope: ${{ github.ref }}-debian + matrix_usernames: "['pi', 'hans']" + matrix_test_scripts: "['run_installation_tests.sh', 'run_installation_tests2.sh', 'run_installation_tests3.sh']" + debian_version_name: ${{ matrix.debian_version_name }} diff --git a/.github/workflows/docker_debian_versionname_sub.yml b/.github/workflows/docker_debian_versionname_sub.yml new file mode 100644 index 000000000..98b4e20e7 --- /dev/null +++ b/.github/workflows/docker_debian_versionname_sub.yml @@ -0,0 +1,172 @@ +name: Subworkflow Test Install Scripts Debian + +on: + workflow_call: + inputs: + runs_on: + required: true + type: string + platform: + required: true + type: string + debian_version_name: + required: true + type: string + cache_scope: + required: true + type: string + docker_image_name: + required: true + type: string + matrix_usernames: + required: true + type: string + matrix_test_scripts: + required: true + type: string + local_registry_port: + required: false + type: number + default: 5000 + +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ inputs.cache_scope }}-${{ inputs.debian_version_name }} + cancel-in-progress: true + +jobs: + + # Build container for test execution + build: + runs-on: ${{ inputs.runs_on }} + + outputs: + cache_key: ${{ steps.vars.outputs.cache_key }} + image_file_path: ${{ steps.vars.outputs.image_file_path }} + image_tag_name: ${{ steps.vars.outputs.image_tag_name }} + + # create local docker registry to use locally build images + services: + registry: + image: registry:2 + ports: + - ${{ inputs.local_registry_port }}:5000 + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + with: + # network=host driver-opt needed to push to local registry + driver-opts: network=host + + - name: Set Output pre-vars + id: pre-vars + env: + DEBIAN_VERSION_NAME: ${{ inputs.debian_version_name }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} + CACHE_SCOPE: ${{ inputs.cache_scope }} + run: | + echo "image_tag_name=${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEBIAN_VERSION_NAME }}-test" >> $GITHUB_OUTPUT + echo "cache_scope=${{ env.CACHE_SCOPE }}-${{ env.DEBIAN_VERSION_NAME }}" >> $GITHUB_OUTPUT + + - name: Set Output vars + id: vars + env: + DEBIAN_VERSION_NAME: ${{ inputs.debian_version_name }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} + LOCAL_REGISTRY_PORT: ${{ inputs.local_registry_port }} + run: | + echo "image_tag_name=${{ steps.pre-vars.outputs.image_tag_name }}" >> $GITHUB_OUTPUT + echo "image_tag_name_local_base=localhost:${{ env.LOCAL_REGISTRY_PORT }}/${{ steps.pre-vars.outputs.image_tag_name }}-base" >> $GITHUB_OUTPUT + echo "image_file_path=./${{ env.DOCKER_IMAGE_NAME }}-${{ env.DEBIAN_VERSION_NAME }}.tar" >> $GITHUB_OUTPUT + echo "cache_scope=${{ steps.pre-vars.outputs.cache_scope }}" >> $GITHUB_OUTPUT + echo "cache_key=${{ steps.pre-vars.outputs.cache_scope }}-${{ github.sha }}#${{ github.run_attempt }}" >> $GITHUB_OUTPUT + + # Build base image for debian version name. Layers will be cached and image pushes to local registry + - name: Build Image - Base + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: true + file: ./ci/Dockerfile.debian.test_install.armv7 + target: test + platforms: ${{ inputs.platform }} + tags: ${{ steps.vars.outputs.image_tag_name_local_base }} + cache-from: type=gha,scope=${{ steps.vars.outputs.cache_scope }} + cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.cache_scope }} + build-args: | + DEBIAN_VERSION_NAME=${{ inputs.debian_version_name }} + GIT_BRANCH=${{ github.head_ref || github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.repository }} + + # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. + - name: Build Image - Update + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: false + file: ./ci/Dockerfile.debian.test_install.armv7 + target: test-update + platforms: ${{ inputs.platform }} + tags: ${{ steps.vars.outputs.image_tag_name }} + cache-from: type=gha,scope=${{ steps.vars.outputs.cache_scope }} + # DON'T use 'cache-to' here as the layer is then cached and this build would be useless + outputs: type=docker,dest=${{ steps.vars.outputs.image_file_path }} + build-args: | + BASE_TEST_IMAGE=${{ steps.vars.outputs.image_tag_name_local_base }} + + # Cache image file for next jobs + - name: Cache Save Docker Image + uses: actions/cache/save@v3 + with: + key: ${{ steps.vars.outputs.cache_key }} + path: ${{ steps.vars.outputs.image_file_path }} + + + # Run tests with build image + test: + needs: [build] + runs-on: ${{ inputs.runs_on }} + + strategy: + fail-fast: false + matrix: + username: ${{ fromJSON(inputs.matrix_usernames) }} + test_script: ${{ fromJSON(inputs.matrix_test_scripts) }} + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + # Load cached image file + - name: Cache Restore Docker Image + uses: actions/cache/restore@v3 + with: + key: ${{ needs.build.outputs.cache_key }} + path: ${{ needs.build.outputs.image_file_path }} + fail-on-cache-miss: true + + - name: Load Docker Image + run: | + docker load --input ${{ needs.build.outputs.image_file_path }} + + # Run test + - name: Run Test ${{ inputs.debian_version_name }}-${{ matrix.username }}-${{ matrix.test_script }} + uses: tj-actions/docker-run@v2 + with: + image: ${{ needs.build.outputs.image_tag_name }} + options: --platform ${{inputs.platform }} --user ${{ matrix.username }} + name: ${{ matrix.test_script }} + args: | + ./${{ matrix.test_script }} From ace029b342962447590501df49442b00227122b6 Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 19 Oct 2023 06:52:42 +0200 Subject: [PATCH 15/17] fixed apt-key deprecation --- ci/Dockerfile.debian.test_install.armv7 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/Dockerfile.debian.test_install.armv7 b/ci/Dockerfile.debian.test_install.armv7 index d434f9c5e..6d861b822 100644 --- a/ci/Dockerfile.debian.test_install.armv7 +++ b/ci/Dockerfile.debian.test_install.armv7 @@ -14,10 +14,10 @@ RUN apt-get update \ apt-utils \ curl \ gnupg \ - && echo "deb http://raspbian.raspberrypi.org/raspbian/ ${DEBIAN_VERSION_NAME} main contrib non-free rpi" > /etc/apt/sources.list.d/raspi.list \ - && echo "deb http://archive.raspberrypi.org/debian/ ${DEBIAN_VERSION_NAME} main" >> /etc/apt/sources.list.d/raspi.list \ - && curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - \ - && curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \ + && curl -fsSL http://raspbian.raspberrypi.org/raspbian.public.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-raspbian-keyring.gpg \ + && curl -fsSL http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-raspbian-keyring.gpg] http://raspbian.raspberrypi.org/raspbian/ ${DEBIAN_VERSION_NAME} main contrib non-free rpi" > /etc/apt/sources.list.d/raspi.list \ + && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg] http://archive.raspberrypi.org/debian/ ${DEBIAN_VERSION_NAME} main" >> /etc/apt/sources.list.d/raspi.list \ && apt-get update \ && apt-get -y upgrade \ && apt-get -y install \ From 99393b6914f7cf8128b679f70acaea64b46240db Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Thu, 19 Oct 2023 23:54:49 +0200 Subject: [PATCH 16/17] unified Dockerfile. renamed workflow files --- ...cker_debian.yml => test_docker_debian.yml} | 6 +- ...=> test_docker_debian_versionname_sub.yml} | 6 +- ci/Dockerfile.buster.amd64 | 32 ----------- ci/Dockerfile.buster.armv7 | 32 ----------- ...n.test_install.armv7 => Dockerfile.debian} | 56 +++++++++++++------ ci/README.md | 45 +++++++-------- 6 files changed, 65 insertions(+), 112 deletions(-) rename .github/workflows/{docker_debian.yml => test_docker_debian.yml} (85%) rename .github/workflows/{docker_debian_versionname_sub.yml => test_docker_debian_versionname_sub.yml} (97%) delete mode 100644 ci/Dockerfile.buster.amd64 delete mode 100644 ci/Dockerfile.buster.armv7 rename ci/{Dockerfile.debian.test_install.armv7 => Dockerfile.debian} (64%) diff --git a/.github/workflows/docker_debian.yml b/.github/workflows/test_docker_debian.yml similarity index 85% rename from .github/workflows/docker_debian.yml rename to .github/workflows/test_docker_debian.yml index 45cfd1a4e..ab0b9ec8a 100644 --- a/.github/workflows/docker_debian.yml +++ b/.github/workflows/test_docker_debian.yml @@ -14,7 +14,7 @@ on: # let only one instance run the test so cache is not corrupted. # cancel already running instances as only the last run will be relevant concurrency: - group: ${{ github.ref }}-debian + group: ${{ github.ref }}-test-debian cancel-in-progress: true jobs: @@ -26,12 +26,12 @@ jobs: fail-fast: false matrix: debian_version_name: ['bullseye', 'buster'] - uses: ./.github/workflows/docker_debian_versionname_sub.yml + uses: ./.github/workflows/test_docker_debian_versionname_sub.yml with: runs_on: ubuntu-latest platform: linux/arm/v7 docker_image_name: rpi-jukebox-rfid - cache_scope: ${{ github.ref }}-debian + cache_scope: ${{ github.ref }}-test-debian matrix_usernames: "['pi', 'hans']" matrix_test_scripts: "['run_installation_tests.sh', 'run_installation_tests2.sh', 'run_installation_tests3.sh']" debian_version_name: ${{ matrix.debian_version_name }} diff --git a/.github/workflows/docker_debian_versionname_sub.yml b/.github/workflows/test_docker_debian_versionname_sub.yml similarity index 97% rename from .github/workflows/docker_debian_versionname_sub.yml rename to .github/workflows/test_docker_debian_versionname_sub.yml index 98b4e20e7..f605aaa0a 100644 --- a/.github/workflows/docker_debian_versionname_sub.yml +++ b/.github/workflows/test_docker_debian_versionname_sub.yml @@ -95,8 +95,8 @@ jobs: context: . load: false push: true - file: ./ci/Dockerfile.debian.test_install.armv7 - target: test + file: ./ci/Dockerfile.debian + target: test-code platforms: ${{ inputs.platform }} tags: ${{ steps.vars.outputs.image_tag_name_local_base }} cache-from: type=gha,scope=${{ steps.vars.outputs.cache_scope }} @@ -113,7 +113,7 @@ jobs: context: . load: false push: false - file: ./ci/Dockerfile.debian.test_install.armv7 + file: ./ci/Dockerfile.debian target: test-update platforms: ${{ inputs.platform }} tags: ${{ steps.vars.outputs.image_tag_name }} diff --git a/ci/Dockerfile.buster.amd64 b/ci/Dockerfile.buster.amd64 deleted file mode 100644 index 9c1ff6e09..000000000 --- a/ci/Dockerfile.buster.amd64 +++ /dev/null @@ -1,32 +0,0 @@ -FROM debian:buster -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdline.txt /etc/sysctl.conf ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.buster.armv7 b/ci/Dockerfile.buster.armv7 deleted file mode 100644 index 3c6f2841a..000000000 --- a/ci/Dockerfile.buster.armv7 +++ /dev/null @@ -1,32 +0,0 @@ -FROM arm32v7/debian:buster-slim -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' >> /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' > /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant ;\ - apt-get clean ;\ - touch /boot/cmdline.txt /etc/sysctl.conf ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.debian.test_install.armv7 b/ci/Dockerfile.debian similarity index 64% rename from ci/Dockerfile.debian.test_install.armv7 rename to ci/Dockerfile.debian index 6d861b822..ecfca4460 100644 --- a/ci/Dockerfile.debian.test_install.armv7 +++ b/ci/Dockerfile.debian @@ -1,15 +1,14 @@ -# Target to build and install all needed base configuration and packages +# Base Target to build and install all needed base configuration and packages. Specifie the needed platform with the docker '--platform XXX' option ARG DEBIAN_VERSION_NAME=bullseye -ARG BASE_TEST_IMAGE=test +ARG BASE_TEST_IMAGE=test-code FROM debian:${DEBIAN_VERSION_NAME}-slim as base ARG DEBIAN_VERSION_NAME ENV DOCKER_RUNNING=true RUN touch /boot/cmdlinetxt -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update \ +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ && apt-get -y install \ apt-utils \ curl \ @@ -35,18 +34,42 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # ------ - -# Target for setting up user for test. user can be selected with the docker '--user $USERNAME' option +# Base Target for setting up the default user. user can be selected with the docker '--user YYY' option FROM base as user -ENV TEST_USER_GROUP=test +ARG USER=pi +ARG USER_GROUP=$USER + +ENV USER=$USER USER_GROUP=$USER_GROUP + +RUN groupadd --gid 1000 $USER_GROUP \ + && useradd -u 1000 -g $USER_GROUP -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER \ + && echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER +# ------ + + + +####### Code Targets ####### -RUN groupadd --gid 1002 $TEST_USER_GROUP +# Target for adding code from the repo. Set Default User +FROM user as code -RUN export USER_PI=pi \ - && export USER_PI_GROUP=$USER_PI \ - && groupadd --gid 1000 $USER_PI_GROUP \ - && useradd -u 1000 -g $USER_PI_GROUP -G sudo,$TEST_USER_GROUP -d /home/$USER_PI -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER_PI \ - && echo "$USER_PI ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER_PI +COPY --chown=$USER:$USER_GROUP --chmod=770 . /code +WORKDIR /code + +USER $USER +# ------ + + + +####### Test Targets ####### + +# Target for setting up an alternativ user 'hans:wurst'. user can be selected with the docker '--user YYY' option +FROM user as test-user + +ENV TEST_USER_GROUP=test + +RUN groupadd --gid 1002 $TEST_USER_GROUP \ + && usermod -a -G $TEST_USER_GROUP $USER RUN export USER_ALT=hans \ && export USER_ALT_GROUP=wurst \ @@ -57,7 +80,7 @@ RUN export USER_ALT=hans \ # Target for adding envs and scripts from the repo to test installation -FROM user as test +FROM test-user as test-code ARG GIT_BRANCH ARG GIT_URL @@ -71,7 +94,8 @@ COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh # Target for applying latest updates (should not be cached!) FROM $BASE_TEST_IMAGE as test-update -RUN apt-get update \ +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ && apt-get -y upgrade \ && rm -rf /var/lib/apt/lists/* # ------ diff --git a/ci/README.md b/ci/README.md index 867fe1607..4ed2910e7 100644 --- a/ci/README.md +++ b/ci/README.md @@ -19,28 +19,33 @@ This is a work in progress so expect things to fail or being flaky. sudo reboot ``` * login to your RPi -* clone this repo and cd into its local clone: +* clone the repo and cd into its local clone: ```bash + cd /home/pi/ + # optional: change to your fork appropriately git clone https://github.com/MiczFlor/RPi-Jukebox-RFID.git - cd /home/pi/RPi-Jukebox-RFID/ + cd RPi-Jukebox-RFID/ + # optional: switch to another branch + git checkout ``` * build the docker image: - * **on normal PCs:** - ```bash - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile.buster.amd64 . - ``` + ```bash + docker build -t rpi-jukebox-rfid:buster-latest -f ci/Dockerfile.debian --platform=linux/arm/v7 --target=code --build-arg="DEBIAN_VERSION_NAME=buster" . + ``` + * additional arguments + * for builds + on normal PCs use `--platform=linux/amd64` + on a raspberry pi use `--platform=linux/arm/v7` + * to use a different debian version change `--build-arg="DEBIAN_VERSION_NAME=buster"` appropriately. - * **on a raspberry pi:** - ```bash - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile.buster.armv7 . - ``` * get something to drink or eat * run the freshly built docker image and start testing. For example: ```bash - docker run --rm -ti rpi-jukebox-rfid-buster:latest /bin/bash + docker run --rm -ti rpi-jukebox-rfid:buster-latest /bin/bash cd /home/pi/ cp /code/scripts/installscripts/buster-install-default.sh /home/pi/ - bash buster-install-default.sh + # set GIT_URL and GIT_BRANCH appropriately to your checkout + bash GIT_URL=https://github.com/MiczFlor/RPi-Jukebox-RFID.git GIT_BRANCH=main buster-install-default.sh ``` NOTE: Get familiar with docker and its flags - `--rm` for example will remove the @@ -48,19 +53,7 @@ This is a work in progress so expect things to fail or being flaky. ### mount hosts code as volume -The created image now contains all the code in the directory `/code` - if you do not want to -rebuild the image after each code-change you can 'mount' the RPi's code version into the -container: - -```bash - git clone https://github.com/MiczFlor/RPi-Jukebox-RFID.git - cd /home/pi/RPi-Jukebox-RFID/ - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile . - docker run --rm -ti -w /code -v $PWD:/code rpi-jukebox-rfid-buster:latest /bin/bash - - cd /home/pi/ - cp /code/scripts/installscripts/buster-install-default.sh /home/pi/ - bash buster-install-default.sh -``` +The created image now contains all the code in the directory `/code` - if you do not want to rebuild the image after each code-change you can 'mount' the RPi's code version into the container. +Add `-w /code -v $PWD:/code` to the `docker run` parameter. In that way every change to the code in the container will be available on the RPi as well as vice versa. From 6bd86c2837297a2f2f19c03a8b8615221c34759c Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Fri, 20 Oct 2023 08:45:47 +0200 Subject: [PATCH 17/17] fix repo name on pullrequest --- .github/workflows/test_docker_debian_versionname_sub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_docker_debian_versionname_sub.yml b/.github/workflows/test_docker_debian_versionname_sub.yml index f605aaa0a..24be6ef74 100644 --- a/.github/workflows/test_docker_debian_versionname_sub.yml +++ b/.github/workflows/test_docker_debian_versionname_sub.yml @@ -104,7 +104,7 @@ jobs: build-args: | DEBIAN_VERSION_NAME=${{ inputs.debian_version_name }} GIT_BRANCH=${{ github.head_ref || github.ref_name }} - GIT_URL=${{ github.server_url }}/${{ github.repository }} + GIT_URL=${{ github.server_url }}/${{ github.event.pull_request.head.repo.full_name || github.repository }} # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. - name: Build Image - Update