From df7c3552860c95a48dd19d5cb6da803cd65c3855 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 20 Jul 2022 15:54:56 -0400 Subject: [PATCH] Add MySQL 8 Support to Backup Tests (#10691) * Add support for MySQL 8.0 in backup tests Signed-off-by: Matt Lord * Add 8.0 workflow Signed-off-by: Matt Lord * whitespace Signed-off-by: Matt Lord * Use vtctldclient SetKeyspaceDurabilityPolicy to manage semi-sync This needed to be done after the shard was setup in order to satisfy the semantic assumptions related to semi-sync in the tests. Signed-off-by: Matt Lord * Remove extraneous changes Signed-off-by: Matt Lord * We need lz4 for TestXtrabackupStreamWithlz4Compression Signed-off-by: Matt Lord * Try using Percona Repo for MySQL 8 to align mysqld and xtrabackup versions Signed-off-by: Matt Lord * Specify stream type everywhere Signed-off-by: Matt Lord * Remove repeated server install Signed-off-by: Matt Lord * Moar... Signed-off-by: Matt Lord * Move vtctlbackup test to 8.0 Signed-off-by: Matt Lord * Rename vtbackup test and move to MySQL 8 Signed-off-by: Matt Lord * Split the xbstream tests so the workflow doesn't time out Otherwise it was going over the 10min limit and getting killed. Signed-off-by: Matt Lord * Use MySQL 8 compat method for setting passwords Signed-off-by: Matt Lord * Test increasing timeout at another level Signed-off-by: Matt Lord * Don't use the init passwords file with 8.0 mysqlctl doesn't start... This is likely due to the change in initialization behavior with MySQL 8.0 as it goes though two phases and you can't simply start up mysqld and pass it data, it has to initialize and restart first. Signed-off-by: Matt Lord * Fix incorrect password update statement for vt_repl user Signed-off-by: Rohit Nayak * Bump timeouts for 8.0 backup tests Signed-off-by: Matt Lord * Bump it more :( Signed-off-by: Matt Lord * Increase backup/restore timeout in backup_utils Signed-off-by: Matt Lord * Apply new 8.0 template everywhere Signed-off-by: Matt Lord * Fix bugs around how the compression flags were getting passed Signed-off-by: Matt Lord * Use 45m timeout for the workflow, 30m for the run. Signed-off-by: Matt Lord * These changes were no longer needed so limiting diff. Signed-off-by: Matt Lord * Explicitly skip new linter check Signed-off-by: Matt Lord Co-authored-by: Rohit Nayak --- .github/workflows/cluster_endtoend_19.yml | 112 ------------- .../workflows/cluster_endtoend_mysql80.yml | 15 ++ ...ter_endtoend_onlineddl_revert_mysql80.yml} | 47 +++--- ..._endtoend_onlineddl_revertible_mysql80.yml | 116 ++++++++++++++ ...r_endtoend_onlineddl_scheduler_mysql80.yml | 116 ++++++++++++++ ...r_endtoend_onlineddl_singleton_mysql80.yml | 116 ++++++++++++++ ...uster_endtoend_onlineddl_vrepl_mysql80.yml | 116 ++++++++++++++ ...ndtoend_onlineddl_vrepl_stress_mysql80.yml | 116 ++++++++++++++ ...d_onlineddl_vrepl_stress_suite_mysql80.yml | 116 ++++++++++++++ ...endtoend_onlineddl_vrepl_suite_mysql80.yml | 116 ++++++++++++++ ...ster_endtoend_schemadiff_vrepl_mysql80.yml | 116 ++++++++++++++ ...endtoend_tabletmanager_tablegc_mysql80.yml | 116 ++++++++++++++ ...dtoend_vreplication_across_db_versions.yml | 15 ++ .../cluster_endtoend_vtbackup_transform.yml | 39 ++--- ..._vtctlbackup_sharded_clustertest_heavy.yml | 49 ++++++ .../workflows/cluster_endtoend_vtorc_8.0.yml | 15 ++ .../cluster_endtoend_xb_backup_mysql80.yml | 118 ++++++++++++++ .../tools/makecolldata/contractions.go | 2 +- .../backup/mysqlctld/backup_mysqlctld_test.go | 4 +- .../backup/vtctlbackup/backup_test.go | 4 +- .../backup/vtctlbackup/backup_utils.go | 23 ++- go/test/endtoend/cluster/cluster_util.go | 43 +---- test/ci_workflow_gen.go | 15 +- test/config.json | 29 ++-- .../cluster_endtoend_test_mysql80.tpl | 150 ++++++++++++++++++ 25 files changed, 1504 insertions(+), 220 deletions(-) delete mode 100644 .github/workflows/cluster_endtoend_19.yml rename .github/workflows/{cluster_endtoend_vtgate_partial_keyspace.yml => cluster_endtoend_onlineddl_revert_mysql80.yml} (63%) create mode 100644 .github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml create mode 100644 .github/workflows/cluster_endtoend_xb_backup_mysql80.yml create mode 100644 test/templates/cluster_endtoend_test_mysql80.tpl diff --git a/.github/workflows/cluster_endtoend_19.yml b/.github/workflows/cluster_endtoend_19.yml deleted file mode 100644 index f813b345f03..00000000000 --- a/.github/workflows/cluster_endtoend_19.yml +++ /dev/null @@ -1,112 +0,0 @@ -# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" - -name: Cluster (19) -on: [push, pull_request] -concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (19)') - cancel-in-progress: true - -jobs: - build: - name: Run endtoend tests on Cluster (19) - runs-on: - group: vitess-ubuntu20 - - steps: - - name: Check if workflow needs to be skipped - id: skip-workflow - run: | - skip='false' - if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then - skip='true' - fi - echo Skip ${skip} - echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - - name: Check out code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - - name: Check for changes in relevant files - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: frouioui/paths-filter@main - id: changes - with: - token: '' - filters: | - end_to_end: - - 'go/**/*.go' - - 'test.go' - - 'Makefile' - - 'build.env' - - 'go.sum' - - 'go.mod' - - 'proto/*.proto' - - 'tools/**' - - 'config/**' - - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_19.yml' - - - name: Set up Go - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v3 - with: - go-version: 1.18.7 - - - name: Set up python - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v4 - - - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf - - - name: Get dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo apt-get update - - sudo service etcd stop - echo 'mysql version: ' - sudo mysql --version - sudo service mysql stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download - - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - - name: Run cluster endtoend test - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: nick-fields/retry@v2 - with: - timeout_minutes: 45 - max_attempts: 3 - retry_on: error - command: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 19 diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 44d7aebfbad..18d5ca0deab 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -11,6 +11,7 @@ jobs: name: Run endtoend tests on Cluster (mysql80) runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -70,6 +71,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | +<<<<<<< HEAD # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 @@ -77,11 +79,24 @@ jobs: wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* +======= + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo apt-get update # Install everything else we need, and configure +<<<<<<< HEAD sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo apt-get update +======= + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo service etcd stop echo 'mysql version: ' sudo mysql --version diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml similarity index 63% rename from .github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml rename to .github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml index 629c99d3c55..64b48f25b46 100644 --- a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml @@ -1,9 +1,9 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (vtgate_partial_keyspace) +name: Cluster (onlineddl_revert) mysql80 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_partial_keyspace)') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revert) mysql80') cancel-in-progress: true env: @@ -13,27 +13,16 @@ env: jobs: build: - name: Run endtoend tests on Cluster (vtgate_partial_keyspace) + name: Run endtoend tests on Cluster (onlineddl_revert) mysql80 runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - - name: Check if workflow needs to be skipped - id: skip-workflow - run: | - skip='false' - if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then - skip='true' - fi - echo Skip ${skip} - echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: Check for changes in relevant files - if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -49,20 +38,20 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml' + - '.github/workflows/**' - name: Set up Go - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v3 + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 with: - go-version: 1.19.3 + go-version: 1.18.3 - name: Set up python - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v4 + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio @@ -70,7 +59,7 @@ jobs: sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | # Setup Percona Server for MySQL 8.0 sudo apt-get update @@ -92,7 +81,7 @@ jobs: go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -104,8 +93,8 @@ jobs: launchable record build --name "$GITHUB_RUN_ID" --source . - name: Run cluster endtoend test - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 45 + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -115,10 +104,10 @@ jobs: set -x # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_partial_keyspace -partial-keyspace=true | tee -a output.txt | go-junit-report -set-exit-code > report.xml + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_revert | tee -a output.txt | go-junit-report -set-exit-code > report.xml - name: Print test output and Record test result in launchable - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + if: steps.changes.outputs.end_to_end == 'true' && always() run: | # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml new file mode 100644 index 00000000000..60d5146c33c --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_revertible) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revertible) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_revertible) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_revertible | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml new file mode 100644 index 00000000000..b5f4717c3b7 --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_scheduler) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_scheduler | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml new file mode 100644 index 00000000000..6c37c4a3123 --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_singleton) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_singleton) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_singleton) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_singleton | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml new file mode 100644 index 00000000000..4d753821b5e --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_vrepl) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_vrepl) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml new file mode 100644 index 00000000000..7e177cd1fa5 --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_vrepl_stress) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml new file mode 100644 index 00000000000..aa940f2d39f --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_vrepl_stress_suite) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress_suite) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml new file mode 100644 index 00000000000..912ae2aefc0 --- /dev/null +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (onlineddl_vrepl_suite) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_suite) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml new file mode 100644 index 00000000000..eee73e7f42e --- /dev/null +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (schemadiff_vrepl) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (schemadiff_vrepl) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (schemadiff_vrepl) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml new file mode 100644 index 00000000000..dbf175954f1 --- /dev/null +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml @@ -0,0 +1,116 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (tabletmanager_tablegc) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (tabletmanager_tablegc) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (tabletmanager_tablegc) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard tabletmanager_tablegc | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 2a4fc32db31..ebafb446a4c 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -11,6 +11,7 @@ jobs: name: Run endtoend tests on Cluster (vreplication_across_db_versions) runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -70,6 +71,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | +<<<<<<< HEAD # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 @@ -77,11 +79,24 @@ jobs: wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* +======= + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo apt-get update # Install everything else we need, and configure +<<<<<<< HEAD sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo apt-get update +======= + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo service etcd stop echo 'mysql version: ' sudo mysql --version diff --git a/.github/workflows/cluster_endtoend_vtbackup_transform.yml b/.github/workflows/cluster_endtoend_vtbackup_transform.yml index 7525f11137c..bcc3c5b40db 100644 --- a/.github/workflows/cluster_endtoend_vtbackup_transform.yml +++ b/.github/workflows/cluster_endtoend_vtbackup_transform.yml @@ -16,24 +16,13 @@ jobs: name: Run endtoend tests on Cluster (vtbackup_transform) runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - - name: Check if workflow needs to be skipped - id: skip-workflow - run: | - skip='false' - if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then - skip='true' - fi - echo Skip ${skip} - echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: Check for changes in relevant files - if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -49,20 +38,20 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_vtbackup_transform.yml' + - '.github/workflows/**' - name: Set up Go - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v3 + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 with: - go-version: 1.19.3 + go-version: 1.18.3 - name: Set up python - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v4 + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio @@ -70,7 +59,7 @@ jobs: sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | # Setup Percona Server for MySQL 8.0 sudo apt-get update @@ -92,7 +81,7 @@ jobs: go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + if: steps.changes.outputs.end_to_end == 'true' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -104,8 +93,8 @@ jobs: launchable record build --name "$GITHUB_RUN_ID" --source . - name: Run cluster endtoend test - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 45 + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -118,7 +107,7 @@ jobs: eatmydata -- go run test.go -docker=false -follow -shard vtbackup_transform | tee -a output.txt | go-junit-report -set-exit-code > report.xml - name: Print test output and Record test result in launchable - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + if: steps.changes.outputs.end_to_end == 'true' && always() run: | # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 72519a34f1d..4551e33582f 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -11,6 +11,7 @@ jobs: name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -70,6 +71,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | +<<<<<<< HEAD # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 @@ -82,6 +84,19 @@ jobs: sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo apt-get update +======= + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo service etcd stop echo 'mysql version: ' sudo mysql --version @@ -108,6 +123,7 @@ jobs: set -x +<<<<<<< HEAD # Increase our local ephemeral port range as we could exhaust this sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this @@ -131,3 +147,36 @@ jobs: # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy +======= + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) diff --git a/.github/workflows/cluster_endtoend_vtorc_8.0.yml b/.github/workflows/cluster_endtoend_vtorc_8.0.yml index 8a248c8eeca..b46bd36b4a2 100644 --- a/.github/workflows/cluster_endtoend_vtorc_8.0.yml +++ b/.github/workflows/cluster_endtoend_vtorc_8.0.yml @@ -11,6 +11,7 @@ jobs: name: Run endtoend tests on Cluster (vtorc_8.0) runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -70,6 +71,7 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | +<<<<<<< HEAD # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 @@ -77,11 +79,24 @@ jobs: wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* +======= + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo apt-get update # Install everything else we need, and configure +<<<<<<< HEAD sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo apt-get update +======= + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop +>>>>>>> bf13249082 (Add MySQL 8 Support to Backup Tests (#10691)) sudo service etcd stop echo 'mysql version: ' sudo mysql --version diff --git a/.github/workflows/cluster_endtoend_xb_backup_mysql80.yml b/.github/workflows/cluster_endtoend_xb_backup_mysql80.yml new file mode 100644 index 00000000000..6008c36e569 --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_backup_mysql80.yml @@ -0,0 +1,118 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_backup) mysql80 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_backup) mysql80') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (xb_backup) mysql80 + runs-on: + group: vitess-ubuntu20 + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/jstemmer/go-junit-report@latest + + sudo apt-get install percona-xtrabackup-80 lz4 + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/go/mysql/collations/tools/makecolldata/contractions.go b/go/mysql/collations/tools/makecolldata/contractions.go index 8ac5b27c328..ef8a08fe27d 100644 --- a/go/mysql/collations/tools/makecolldata/contractions.go +++ b/go/mysql/collations/tools/makecolldata/contractions.go @@ -58,7 +58,7 @@ func printContraction1(g *codegen.Generator, wa *weightarray, incont []uca.Contr trie[r] = append(trie[r], cont) } if depth == len(cont.Path) { - leaf = &cont + leaf = &cont // nolint:exportloopref } } diff --git a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go index 177c1a8b8ff..e49ece11934 100644 --- a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go +++ b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go @@ -26,7 +26,7 @@ import ( // TestBackupMysqlctld - tests the backup using mysqlctld. func TestBackupMysqlctld(t *testing.T) { - backup.TestBackup(t, backup.Mysqlctld, "", 0, nil, nil) + backup.TestBackup(t, backup.Mysqlctld, "xbstream", 0, nil, nil) } func TestBackupMysqlctldWithlz4Compression(t *testing.T) { @@ -35,7 +35,7 @@ func TestBackupMysqlctldWithlz4Compression(t *testing.T) { BuiltinCompressor: "lz4", } - backup.TestBackup(t, backup.Mysqlctld, "", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"}) + backup.TestBackup(t, backup.Mysqlctld, "xbstream", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"}) } func setDefaultCompressionFlag() { diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index 00e51c435e6..557ee57ec27 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -24,7 +24,7 @@ import ( // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - TestBackup(t, Backup, "", 0, nil, nil) + TestBackup(t, Backup, "xbstream", 0, nil, nil) } func TestBackupMainWithZstdCompression(t *testing.T) { @@ -35,7 +35,7 @@ func TestBackupMainWithZstdCompression(t *testing.T) { ExternalDecompressorCmd: "zstd -d", } - TestBackup(t, Backup, "", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"}) + TestBackup(t, Backup, "xbstream", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"}) } func setDefaultCompressionFlag() { diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index e03b578e9f9..34cbd11b3fc 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -254,6 +254,27 @@ func TearDownCluster() { // TestBackup runs all the backup tests func TestBackup(t *testing.T, setupType int, streamMode string, stripes int, cDetails *CompressionDetails, runSpecific []string) error { + verStr, err := mysqlctl.GetVersionString() + require.NoError(t, err) + _, vers, err := mysqlctl.ParseVersionString(verStr) + require.NoError(t, err) + switch streamMode { + case "xbstream": + if vers.Major < 8 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=xbstream as those are only tested on XtraBackup/MySQL 8.0+") + return nil + } + case "", "tar": // streaming method of tar is the default for the vttablet --xtrabackup_stream_mode flag + // XtraBackup 8.0 must be used with MySQL 8.0 and it no longer supports tar as a stream method: + // https://docs.percona.com/percona-xtrabackup/2.4/innobackupex/streaming_backups_innobackupex.html + // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup_bin/backup.streaming.html + if vers.Major > 5 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=tar as tar is no longer a streaming option in XtraBackup 8.0") + return nil + } + default: + require.FailNow(t, fmt.Sprintf("Unsupported xtrabackup stream mode: %s", streamMode)) + } testMethods := []struct { name string @@ -362,7 +383,7 @@ func primaryBackup(t *testing.T) { require.Nil(t, err) restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) require.Nil(t, err) // Verify that we have all the new data -- we should have 2 records now... diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index eb6e8c2b2f4..83e4f4ea243 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -317,44 +317,15 @@ func WriteDbCredentialToTmp(tmpDir string) string { func GetPasswordUpdateSQL(localCluster *LocalProcessCluster) string { pwdChangeCmd := ` # Set real passwords for all users. - UPDATE mysql.user SET %s = PASSWORD('RootPass') - WHERE User = 'root' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtDbaPass') - WHERE User = 'vt_dba' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAppPass') - WHERE User = 'vt_app' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass') - WHERE User = 'vt_allprivs' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtReplPass') - WHERE User = 'vt_repl' AND Host = '%%'; - UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass') - WHERE User = 'vt_filtered' AND Host = 'localhost'; + SET PASSWORD FOR 'root'@'localhost' = 'RootPass'; + SET PASSWORD FOR 'vt_dba'@'localhost' = 'VtDbaPass'; + SET PASSWORD FOR 'vt_app'@'localhost' = 'VtAppPass'; + SET PASSWORD FOR 'vt_allprivs'@'localhost' = 'VtAllprivsPass'; + SET PASSWORD FOR 'vt_repl'@'%' = 'VtReplPass'; + SET PASSWORD FOR 'vt_filtered'@'localhost' = 'VtFilteredPass'; FLUSH PRIVILEGES; ` - pwdCol, _ := getPasswordField(localCluster) - return fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) -} - -// getPasswordField determines which column is used for user passwords in this MySQL version. -func getPasswordField(localCluster *LocalProcessCluster) (pwdCol string, err error) { - tablet := &Vttablet{ - Type: "relpica", - TabletUID: 100, - MySQLPort: 15000, - MysqlctlProcess: *MysqlCtlProcessInstance(100, 15000, localCluster.TmpDirectory), - } - if err = tablet.MysqlctlProcess.Start(); err != nil { - return "", err - } - tablet.VttabletProcess = VttabletProcessInstance(tablet.HTTPPort, tablet.GrpcPort, tablet.TabletUID, "", "", "", 0, tablet.Type, localCluster.TopoPort, "", "", nil, false, localCluster.DefaultCharset) - result, err := tablet.VttabletProcess.QueryTablet("select password from mysql.user limit 0", "", false) - if err == nil && len(result.Rows) > 0 { - return "password", nil - } - tablet.MysqlctlProcess.Stop() - os.RemoveAll(path.Join(tablet.VttabletProcess.Directory)) - return "authentication_string", nil - + return pwdChangeCmd } // CheckSrvKeyspace confirms that the cell and keyspace contain the expected diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 65f517c7335..58d45a07e2f 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -73,8 +73,7 @@ var ( "ers_prs_newfeatures_heavy", "15", "vtgate_general_heavy", - "18", - "19", + "vtbackup_transform", "xb_backup", "21", "22", @@ -172,6 +171,18 @@ func clusterMySQLVersions(clusterName string) mysqlVersions { return allMySQLVersions case clusterName == "mysql57": return []mysqlVersion{mysql57} + case clusterName == "mysql80": + return []mysqlVersion{mysql80} + case clusterName == "vtorc_8.0": + return []mysqlVersion{mysql80} + case clusterName == "vreplication_across_db_versions": + return []mysqlVersion{mysql80} + case clusterName == "xb_backup": + return allMySQLVersions + case clusterName == "vtctlbackup_sharded_clustertest_heavy": + return []mysqlVersion{mysql80} + case clusterName == "vtbackup_transform": + return []mysqlVersion{mysql80} default: return defaultMySQLVersions } diff --git a/test/config.json b/test/config.json index c7a71830fc6..54f7c6903be 100644 --- a/test/config.json +++ b/test/config.json @@ -93,7 +93,7 @@ }, "backup": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup", "-timeout", "15m"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup", "-timeout", "30m"], "Command": [], "Manual": false, "Shard": "vtctlbackup_sharded_clustertest_heavy", @@ -102,7 +102,7 @@ }, "backup_mysqlctld": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/mysqlctld", "-timeout", "15m"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/mysqlctld", "-timeout", "30m"], "Command": [], "Manual": false, "Shard": "21", @@ -111,19 +111,19 @@ }, "backup_only": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup", "-timeout", "30m"], "Command": [], "Manual": false, - "Shard": "19", + "Shard": "vtbackup_transform", "RetryMax": 1, "Tags": ["upgrade_downgrade_backups"] }, "backup_transform": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/transform"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/transform", "-timeout", "30m"], "Command": [], "Manual": false, - "Shard": "19", + "Shard": "vtbackup_transform", "RetryMax": 1, "Tags": ["upgrade_downgrade_backups"] }, @@ -147,7 +147,16 @@ }, "backup_xtrabackup_xbstream": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream"], + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream", "-run", "XtrabackupStream", "-timeout", "30m"], + "Command": [], + "Manual": false, + "Shard": "xb_backup", + "RetryMax": 1, + "Tags": [] + }, + "backup_xtrabackup_xbstream_lz4": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/xtrabackupstream", "-run", "XtrabackupStreamWithlz4Compression", "-timeout", "30m"], "Command": [], "Manual": false, "Shard": "xb_backup", @@ -398,7 +407,7 @@ }, "sharded": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/sharded"], + "Args": ["vitess.io/vitess/go/test/endtoend/sharded", "-timeout", "30m"], "Command": [], "Manual": false, "Shard": "vtctlbackup_sharded_clustertest_heavy", @@ -1071,7 +1080,7 @@ }, "vreplication_v2": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestBasicV2Workflows"], + "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestBasicV2Workflows", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "vreplication_v2", @@ -1080,7 +1089,7 @@ }, "vreplication_across_db_versions": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestV2WorkflowsAcrossDBVersions"], + "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestV2WorkflowsAcrossDBVersions", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "vreplication_across_db_versions", diff --git a/test/templates/cluster_endtoend_test_mysql80.tpl b/test/templates/cluster_endtoend_test_mysql80.tpl new file mode 100644 index 00000000000..015a808601a --- /dev/null +++ b/test/templates/cluster_endtoend_test_mysql80.tpl @@ -0,0 +1,150 @@ +name: {{.Name}} +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{"{{"}} github.ref {{"}}"}}, '{{.Name}}') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{`{{ github.event.pull_request.head.sha }}`}}" + +jobs: + build: + name: Run endtoend tests on {{.Name}} + {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/**' + + - name: Set up Go + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.3 + + - name: Set up python + if: steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + {{if .InstallXtraBackup}} + + sudo apt-get install percona-xtrabackup-80 lz4 + + {{end}} + + {{if .MakeTools}} + + - name: Installing zookeeper and consul + if: steps.changes.outputs.end_to_end == 'true' + run: | + make tools + + {{end}} + + - name: Setup launchable dependencies + if: steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + {{if .LimitResourceUsage}} + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + {{end}} + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt