Merge branch 'main' into dependabot/github_actions/whoan/docker-build…

…-with-cache-action-8

.github/docker-images/manylinux2014-aarch64/Dockerfile

@@ -22,7 +22,7 @@ RUN /opt/python/cp39-cp39/bin/python -m pip install --upgrade setuptools virtual
 ###############################################################################
 # nodejs/npm
 ###############################################################################
-RUN curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash -
+RUN curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash -
 RUN sudo yum -y install nodejs && node --version
 
 ###############################################################################

.github/docker-images/swift-5-al2-x64/Dockerfile

@@ -1,5 +1,5 @@
 # https://hub.docker.com/_/swift
-FROM swift:5.5.3-amazonlinux2
+FROM swift:5.7.3-amazonlinux2
 
 ###############################################################################
 # Install prereqs

.github/docker-images/swift-5-centos-x64/Dockerfile

@@ -1,5 +1,5 @@
 # https://hub.docker.com/_/swift
-FROM swift:5.5.3-centos7
+FROM swift:5.7.3-centos7
 
 ###############################################################################
 # Install prereqs

.github/docker-images/swift-5-ubuntu-x64/Dockerfile

@@ -1,5 +1,5 @@
 # https://hub.docker.com/_/swift
-FROM swift:5.5.3-focal
+FROM swift:5.7.3-focal
 
 ###############################################################################
 # Install prereqs

.github/workflows/sanity-test.yml

@@ -78,7 +78,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        host: [ubuntu-20.04, macos-11, macos-12, windows-2022]
+        host: [ubuntu-22.04, macos-11, macos-12, windows-2022]
     needs: package
     runs-on: ${{ matrix.host }}
     steps:

builder/actions/setup_cross_ci_crt_environment.py

@@ -205,6 +205,8 @@ def _common_setup(self, env):
                             "ci/mqtt5/us/authorizer/signed/tokenkeyname")
         self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE",
                             "ci/mqtt5/us/authorizer/signed/signature")
+        self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED",
+                            "ci/mqtt5/us/authorizer/signed/signature/unencoded")
 
         # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM)
         if (self.is_linux == True):
@@ -291,6 +293,8 @@ def _common_setup(self, env):
                             "ci/mqtt5/us/authorizer/signed/tokenkeyname")
         self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE",
                             "ci/mqtt5/us/authorizer/signed/signature")
+        self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED",
+                            "ci/mqtt5/us/authorizer/signed/signature/unencoded")
 
         # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM)
         if (self.is_linux == True):
@@ -419,8 +423,11 @@ def _common_setup(self, env):
         pass
 
     def run(self, env):
+        # A special environment variable indicating that we want to dump test environment variables to a specified file.
+        env_dump_file = env.shell.getenv("AWS_SETUP_CRT_TEST_ENVIRONMENT_DUMP_FILE")
+
         # Bail if not running tests
-        if not env.project.needs_tests(env):
+        if not env.project.needs_tests(env) and not env_dump_file:
             print('Tests not needed for project. Skipping setting test environment variables')
             return
 
@@ -471,3 +478,10 @@ def run(self, env):
         print(f"Detected whether on Codebuild: {self.is_codebuild}")
 
         self._common_setup(env)
+
+        # Create a temporary file with all environment variables.
+        # Useful for running tests locally.
+        if env_dump_file:
+            with open(file=env_dump_file, mode='w+') as file:
+                for env_name, env_value in env.project.config['test_env'].items():
+                    file.write(f"export {env_name}={env_value}\n")

builder/actions/setup_cross_ci_helpers.py

@@ -215,7 +215,7 @@ def _get_token_slots(env):
 
 def _get_softhsm2_version(env):
     output = _exec_softhsm2_util(env, '--version').output
-    match = re.match('([0-9+])\.([0-9]+).([0-9]+)', output)
+    match = re.match(r'([0-9+])\.([0-9]+).([0-9]+)', output)
     return (int(match.group(1)), int(match.group(2)), int(match.group(3)))
 
 ################################################################################

builder/core/fetch.py

@@ -236,7 +236,25 @@ def fetch_and_extract(url, archive_path, extract_path):
     print('Extracting {} to {}'.format(archive_path, extract_path))
     if tarfile.is_tarfile(archive_path):
         with tarfile.open(archive_path) as tar:
-            tar.extractall(extract_path)
+            def is_within_directory(directory, target):
+
+                abs_directory = os.path.abspath(directory)
+                abs_target = os.path.abspath(target)
+
+                prefix = os.path.commonprefix([abs_directory, abs_target])
+
+                return prefix == abs_directory
+
+            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+
+                for member in tar.getmembers():
+                    member_path = os.path.join(path, member.name)
+                    if not is_within_directory(path, member_path):
+                        raise Exception("Attempted Path Traversal in Tar File")
+
+                tar.extractall(path, members, numeric_owner=numeric_owner)
+
+            safe_extract(tar, extract_path)
 
     elif zipfile.is_zipfile(archive_path):
         with zipfile.ZipFile(archive_path) as zip:

builder/core/toolchain.py

@@ -17,19 +17,19 @@ def _compiler_version(cc):
 
         for text in lines:
             # Apple clang
-            m = re.match('Apple (LLVM|clang) version (\d+)', text)
+            m = re.match(r'Apple (LLVM|clang) version (\d+)', text)
             if m:
                 return 'appleclang', m.group(2)
             # LLVM clang
-            m = re.match('.*(LLVM|clang) version (\d+)', text)
+            m = re.match(r'.*(LLVM|clang) version (\d+)', text)
             if m:
                 return 'clang', m.group(2)
             # GCC 4.x
-            m = re.match('gcc .+ (4\.\d+)', text)
+            m = re.match(r'gcc .+ (4\.\d+)', text)
             if m:
                 return 'gcc', m.group(1)
             # GCC 5+
-            m = re.match('gcc .+ (\d+)\.', text)
+            m = re.match(r'gcc .+ (\d+)\.', text)
             if m:
                 return 'gcc', m.group(1)
     return None, None

builder/imports/llvm.py

@@ -130,7 +130,7 @@ def install(self, env):
         sudo = ['sudo'] if sudo else []
 
         # Strip minor version info
-        version = env.toolchain.compiler_version.replace('\..+', '')
+        version = env.toolchain.compiler_version.replace(r'\..+', '')
 
         script = tempfile.NamedTemporaryFile(delete=False)
         script_path = script.name

builder/imports/nodejs.py

@@ -14,7 +14,7 @@
 import re
 
 
-NVM = """\
+NVM = r"""\
 #!/usr/bin/env bash
 export NVM_DIR="$HOME/.nvm"
 [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
@@ -126,7 +126,7 @@ def install_node_via_unofficial_build(self, env):
         # Normaliz version format, please note 12.16.3 is the last version has x86 support
         def normalize_version(v):
             append_times = 0
-            while re.match('^([0-9]+\.){2}[0-9]+$', v) == None:
+            while re.match(r'^([0-9]+\.){2}[0-9]+$', v) == None:
                 # Only try append sub version twice
                 if append_times < 2:
                     v += ".0"