Merge remote-tracking branch 'origin/main' into 172--global-binaries

.github/workflows/TestTugboat.yml

@@ -0,0 +1,75 @@
+name: "Test Tugboat"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  Test-Tugboat:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create a Drupal project
+        run: composer create-project drupal/recommended-project . --ignore-platform-req=ext-gd
+
+      - uses: actions/checkout@v3
+        with:
+          path: drainpipe
+
+      - uses: ./drainpipe/scaffold/github/actions/common/set-env
+
+      - name: Install DDEV
+        uses: ./drainpipe/scaffold/github/actions/common/ddev
+        with:
+          git-name: Drainpipe Bot
+          git-email: [email protected]
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Setup Project
+        run: |
+          ddev config --auto
+          ddev config --php-version "8.1"
+          ddev config --nodejs-version "18"
+          ddev config --database=mariadb:10.4
+          ddev start
+          ddev composer config extra.drupal-scaffold.gitignore true
+          ddev composer config --json extra.drupal-scaffold.allowed-packages \[\"lullabot/drainpipe\"]
+          ddev composer config --no-plugins allow-plugins.composer/installers true
+          ddev composer config --no-plugins allow-plugins.drupal/core-composer-scaffold true
+          ddev composer config --no-plugins allow-plugins.lullabot/drainpipe true
+          ddev composer config repositories.drainpipe --json '{"type": "path", "url": "drainpipe", "options": {"symlink": false}}'
+          ddev composer config minimum-stability dev
+          ddev composer config extra.drainpipe --json '{"tugboat": {}}'
+          ddev composer require lullabot/drainpipe --with-all-dependencies
+
+      # Compare the generated files to the ones used to build this repository
+      # preview - they should be the same.
+      - name: Test Generated Files
+        run: |
+          cmp -b drainpipe/.tugboat/config.yml .tugboat/config.yml
+          sed -i '/#drainpipe-start/,/#drainpipe-end/d' drainpipe/.tugboat/steps/1-init.sh
+          sed -i '/#drainpipe-start/,/#drainpipe-end/d' drainpipe/.tugboat/steps/2-update.sh
+          sed -i '/#drainpipe-start/,/#drainpipe-end/d' drainpipe/.tugboat/steps/3-build.sh
+          cmp -b drainpipe/.tugboat/steps/1-init.sh .tugboat/steps/1-init.sh
+          cmp -b drainpipe/.tugboat/steps/2-update.sh .tugboat/steps/2-update.sh
+          cmp -b drainpipe/.tugboat/steps/3-build.sh .tugboat/steps/3-build.sh
+
+      - name: Add a build:tugboat step
+        run: |
+          echo "  build:tugboat:" >> Taskfile.yml
+          echo "    cmds:" >> Taskfile.yml
+          echo "      - echo \"Tugboat build\"" >> Taskfile.yml
+          ddev composer install
+          if ! grep -q './vendor/bin/task build:tugboat' .tugboat/steps/3-build.sh; then
+            exit 1
+          fi
+
+      - name: Upload test artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: .tugboat
+          path: .tugboat

.tugboat/config.yml

@@ -0,0 +1,16 @@
+services:
+  php:
+    http: false
+    image: tugboatqa/php-nginx:8.1-fpm
+    default: true
+
+    depends:
+      - mariadb
+
+    commands:
+      init: ./.tugboat/steps/1-init.sh
+      update: ./.tugboat/steps/2-update.sh
+      build: ./.tugboat/steps/3-build.sh
+
+  mariadb:
+    image: tugboatqa/mariadb:10.4

.tugboat/steps/1-init.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -eux
+echo "Initializing..."
+
+# Install task
+sh -c "$(curl --location https://raw.githubusercontent.com/go-task/task/v3.24.0/install-task.sh)" -- -d -b /usr/local/bin
+
+# Install mysql or mariadb client.
+apt-get update
+apt-get install -y mariadb-client
+
+# Link the document root to the expected path. Tugboat uses /docroot
+# by default. So, if Drupal is located at any other path in your git
+# repository, change that here. This example links /web to the docroot
+ln -snf "${TUGBOAT_ROOT}/web" "${DOCROOT}"
+
+# Create the Drupal private and public files directories if they aren't
+# already present.
+mkdir -p "${TUGBOAT_ROOT}/web/sites/default/files"
+chmod 777 "${TUGBOAT_ROOT}/web/sites/default/files"
+
+# Install the PHP opcache as it's not included by default and needed for
+# decent performance.
+docker-php-ext-install opcache
+
+# GD dependencies.
+apt-get install -y libpng-dev libjpeg-dev libfreetype6-dev
+
+# WebP dependencies.
+apt-get install -y libwebp-dev libwebp7 webp libmagickwand-dev
+
+# Build and install gd.
+docker-php-ext-configure gd --with-freetype --with-jpeg --with-webp
+docker-php-ext-install gd
+
+# Install ImageMagick. This is recommended by both Acquia and Pantheon instead
+# of GD. Lullabot will likely be publishing an ADR recommending it too.
+apt-get install -y imagemagick
+
+# Install node
+curl -fsSL https://deb.nodesource.com/setup_current.x | bash -
+apt-get install -y nodejs
+npm install -g n
+n 18
+# This only works for node > 16, but that version is unsupported now anyway.
+corepack enable
+#drainpipe-start
+# This is necessary for testing as this repository doesn't hold a Drupal site.
+shopt -s dotglob
+mkdir ../drainpipe-tmp
+mv * ../drainpipe-tmp/
+composer create-project drupal/recommended-project .
+mv ../drainpipe-tmp drainpipe
+composer config extra.drupal-scaffold.gitignore true
+composer config --json extra.drupal-scaffold.allowed-packages \[\"lullabot/drainpipe\"]
+composer config --no-plugins allow-plugins.composer/installers true
+composer config --no-plugins allow-plugins.drupal/core-composer-scaffold true
+composer config --no-plugins allow-plugins.lullabot/drainpipe true
+composer config repositories.drainpipe --json '{"type": "path", "url": "drainpipe", "options": {"symlink": true}}'
+composer config extra.drainpipe --json '{"tugboat": {}}'
+composer config minimum-stability dev
+composer require lullabot/drainpipe --with-all-dependencies
+cp web/sites/default/default.settings.php web/sites/default/settings.php
+#drainpipe-end
+
+composer install
+#drainpipe-start
+rm -rf .tugboat
+mv drainpipe/.tugboat .tugboat
+#drainpipe-end

.tugboat/steps/2-update.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eux
+echo "Updating..."
+
+#drainpipe-start
+mv .tugboat .tugboat-tmp
+#drainpipe-end
+composer install
+./vendor/bin/task sync
+#drainpipe-start
+./vendor/bin/drush config:export --yes
+rm -rf .tugboat
+mv .tugboat-tmp .tugboat
+#drainpipe-end

.tugboat/steps/3-build.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eux
+echo "Building..."
+
+./vendor/bin/task build
+./vendor/bin/task drupal:update

README.md

@@ -28,6 +28,7 @@ for a Drupal site, including:
 * [GitLab CI Integration](#gitlab-ci-integration)
     + [Composer Lock Diff](#composer-lock-diff-1)
     + [Pantheon](#pantheon-2)
+* [Tugboat Integration](#tugboat)
 ---
 ## Installation
 
@@ -430,8 +431,101 @@ Requires `GITLAB_ACCESS_TOKEN` variable to be set, which is an access token with
 }
 ```
 
+- Add the following the composer.json to enable deployment of Pantheon Review Apps
+  ```json
+  "extra": {
+      "drainpipe": {
+          "github": ["PantheonReviewApps"]
+      }
+  }
+  ```
+- Run `composer install`
+- Add your Pantheon `site-name` to the last job in the new
+  workflow file at `.github/workflows/PantheonReviewApps.yml`
+- Add the following secrets to your repository:
+  - `PANTHEON_TERMINUS_TOKEN` See https://pantheon.io/docs/terminus/install#machine-token
+  - `SSH_PRIVATE_KEY` A private key of a user which can push to Pantheon
+  - `SSH_KNOWN_HOSTS` The result of running `ssh-keyscan -H codeserver.dev.$PANTHEON_SITE_ID.drush.in`
+  - `TERMINUS_PLUGINS` Comma-separated list of Terminus plugins to be available (optional)
+
 This will setup Merge Request deployment to Pantheon Multidev environments. See
 [scaffold/gitlab/gitlab-ci.example.yml] for an example. You can also just
 include which will give you helpers that you can include and reference for tasks
 such as setting up [Terminus](https://pantheon.io/docs/terminus). See
 [scaffold/gitlab/Pantheon.gitlab-ci.yml](scaffold/gitlab/Pantheon.gitlab-ci.yml).
+
+## Tugboat
+
+Add the following to `composer.json` to add Tugboat configuration:
+
+```json
+{
+    "extra": {
+        "drainpipe": {
+            "tugboat": {}
+        }
+    }
+}
+```
+
+The following will be autodetected based on your `.ddev/config.yml`:
+- Web server (nginx or apache)
+- PHP version
+- Database type and version
+- nodejs version
+- Redis (Obtained with `ddev get ddev/ddev-redis`)
+
+Additionally, Pantheon Terminus can be added:
+```json
+{
+    "extra": {
+        "drainpipe": {
+            "tugboat": {
+              "terminus": true
+            }
+        }
+    }
+}
+```
+
+It is assumed the following tasks exist:
+- `build`
+- `sync`
+
+The `build` and `sync` tasks can be overridden with a `build:tugboat` and
+`sync:tugboat` task if required (you will need to re-run `composer install` to
+regenerate the Tugboat scripts if you  are adding this task to your
+`Taskfile.yml` for the first time).
+
+```
+  sync:
+    desc: "Fetches a database from Pantheon and imports it"
+    cmds:
+      - task: pantheon:fetch-db
+      - task: drupal:import-db
+  sync:tugboat:
+    desc: "Fetches a database from Pantheon and imports it in Tugboat"
+    cmds:
+      - task: pantheon:fetch-db
+        vars:
+          DB_DIR: /var/lib/tugboat/files/db
+      - task: drupal:import-db
+        vars:
+          DB_DIR: /var/lib/tugboat/files/db
+```
+
+>>>
+💡
+`composer install` should be re-run if any changes are made to the DDEV
+configuration.
+>>>
+
+You can hook into the `init` step of images by adding them to your
+`Taskfile.yml`, e.g.
+
+```
+tugboat:php:init:
+  cmds:
+    - apt-get install -y libldap2-dev
+    - docker-php-ext-install ldap
+```

composer.json

@@ -11,9 +11,11 @@
     },
     "require": {
         "php": "^7.3||^8.0",
+        "ext-json": "*",
         "composer-plugin-api": "^2.0",
         "drush/drush": "^10|^11",
         "symfony/yaml": "^3|^4|^5|^6",
+        "twig/twig": "^2|^3",
         "vlucas/phpdotenv": "^4|^5",
         "ext-json": "*"
     },

scaffold/Taskfile.yml

@@ -35,3 +35,11 @@ tasks:
 # assets:watch:
 #   desc: "Builds assets such as CSS & JS, and watches them for changes"
 #   deps: [sass:watch, javascript:watch]
+  sync:
+    desc: "Sync a database from production and import it"
+    cmds:
+      # Replace this with a command to fetch your database.
+      - ./vendor/bin/drush site:install -y
+      - echo "🧹 Sanitising database"
+      - ./vendor/bin/drush sql:sanitize --yes
+

scaffold/github/actions/common/ddev/action.yml

@@ -53,7 +53,7 @@ runs:
           sudo apt install -y ddev
         fi
 
-        ddev config global --instrumentation-opt-in=false --omit-containers=dba,ddev-ssh-agent
+        ddev config global --instrumentation-opt-in=false --omit-containers=ddev-ssh-agent
 
         if [ "${{ inputs.composer-cache-dir }}" != "false" ]; then
           # @todo Replace /var/www/html with an environment variable.

scaffold/gitignore/common.gitignore

@@ -40,8 +40,8 @@ vendor
 ########
 # Yarn #
 ########
+.pnp.*
 .yarn/*
-!.yarn/cache
 !.yarn/patches
 !.yarn/plugins
 !.yarn/releases
@@ -54,3 +54,4 @@ node_modules
 # Tests #
 #########
 test_result
+.prettiercache

scaffold/tugboat/config.yml.twig

@@ -0,0 +1,42 @@
+services:
+  php:
+    http: false
+    image: {{ webserver_image }}
+    default: true
+
+    depends:
+      - {{ database_type }}
+{% if memory_cache_type %}
+      - {{ memory_cache_type }}
+{% endif %}
+
+    commands:
+      init: ./.tugboat/steps/1-init.sh
+      update: ./.tugboat/steps/2-update.sh
+      build: ./.tugboat/steps/3-build.sh
+
+  {{ database_type }}:
+    image: tugboatqa/{{ database_type }}:{{ database_version }}
+{% if init.mysql %}
+    checkout: true
+    commands:
+      init:
+        - sh -c "$(curl --location https://raw.githubusercontent.com/go-task/task/v{{ task_version }}/install-task.sh)" -- -d -b /usr/local/bin
+        - wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+        - yq -i 'del(."includes")' Taskfile.yml
+        - task tugboat:mysql:init
+{% endif %}
+{% if memory_cache_type %}
+
+  {{ memory_cache_type }}:
+    image: tugboatqa/{{ memory_cache_type }}:{{ memory_cache_version }}
+{% if init.redis %}
+    checkout: true
+    commands:
+      init:
+        - sh -c "$(curl --location https://raw.githubusercontent.com/go-task/task/v{{ task_version }}/install-task.sh)" -- -d -b /usr/local/bin
+        - wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+        - yq -i 'del(."includes")' Taskfile.yml
+        - task tugboat:redis:init
+{% endif %}
+{% endif %}