diff --git a/.github/docker-images/manylinux2014-aarch64/Dockerfile b/.github/docker-images/manylinux2014-aarch64/Dockerfile index 16f63ff12..c685b585d 100644 --- a/.github/docker-images/manylinux2014-aarch64/Dockerfile +++ b/.github/docker-images/manylinux2014-aarch64/Dockerfile @@ -22,7 +22,7 @@ RUN /opt/python/cp39-cp39/bin/python -m pip install --upgrade setuptools virtual ############################################################################### # nodejs/npm ############################################################################### -RUN curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash - +RUN curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash - RUN sudo yum -y install nodejs && node --version ############################################################################### diff --git a/.github/docker-images/swift-5-al2-x64/Dockerfile b/.github/docker-images/swift-5-al2-x64/Dockerfile index 358703228..26cae9dde 100644 --- a/.github/docker-images/swift-5-al2-x64/Dockerfile +++ b/.github/docker-images/swift-5-al2-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-amazonlinux2 +FROM swift:5.7.3-amazonlinux2 ############################################################################### # Install prereqs diff --git a/.github/docker-images/swift-5-centos-x64/Dockerfile b/.github/docker-images/swift-5-centos-x64/Dockerfile index f5df6b64f..34063a068 100644 --- a/.github/docker-images/swift-5-centos-x64/Dockerfile +++ b/.github/docker-images/swift-5-centos-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-centos7 +FROM swift:5.7.3-centos7 ############################################################################### # Install prereqs diff --git a/.github/docker-images/swift-5-ubuntu-x64/Dockerfile b/.github/docker-images/swift-5-ubuntu-x64/Dockerfile index 8b03f5b71..45b2317af 100644 --- a/.github/docker-images/swift-5-ubuntu-x64/Dockerfile +++ b/.github/docker-images/swift-5-ubuntu-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-focal +FROM swift:5.7.3-focal ############################################################################### # Install prereqs diff --git a/.github/workflows/sanity-test.yml b/.github/workflows/sanity-test.yml index f8cccc2ec..ae905c121 100644 --- a/.github/workflows/sanity-test.yml +++ b/.github/workflows/sanity-test.yml @@ -78,7 +78,7 @@ jobs: strategy: fail-fast: false matrix: - host: [ubuntu-20.04, macos-11, macos-12, windows-2022] + host: [ubuntu-22.04, macos-11, macos-12, windows-2022] needs: package runs-on: ${{ matrix.host }} steps: diff --git a/builder/actions/setup_cross_ci_crt_environment.py b/builder/actions/setup_cross_ci_crt_environment.py index db3efaaa8..bdd43a074 100644 --- a/builder/actions/setup_cross_ci_crt_environment.py +++ b/builder/actions/setup_cross_ci_crt_environment.py @@ -205,6 +205,8 @@ def _common_setup(self, env): "ci/mqtt5/us/authorizer/signed/tokenkeyname") self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE", "ci/mqtt5/us/authorizer/signed/signature") + self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED", + "ci/mqtt5/us/authorizer/signed/signature/unencoded") # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM) if (self.is_linux == True): @@ -291,6 +293,8 @@ def _common_setup(self, env): "ci/mqtt5/us/authorizer/signed/tokenkeyname") self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE", "ci/mqtt5/us/authorizer/signed/signature") + self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED", + "ci/mqtt5/us/authorizer/signed/signature/unencoded") # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM) if (self.is_linux == True): @@ -419,8 +423,11 @@ def _common_setup(self, env): pass def run(self, env): + # A special environment variable indicating that we want to dump test environment variables to a specified file. + env_dump_file = env.shell.getenv("AWS_SETUP_CRT_TEST_ENVIRONMENT_DUMP_FILE") + # Bail if not running tests - if not env.project.needs_tests(env): + if not env.project.needs_tests(env) and not env_dump_file: print('Tests not needed for project. Skipping setting test environment variables') return @@ -471,3 +478,10 @@ def run(self, env): print(f"Detected whether on Codebuild: {self.is_codebuild}") self._common_setup(env) + + # Create a temporary file with all environment variables. + # Useful for running tests locally. + if env_dump_file: + with open(file=env_dump_file, mode='w+') as file: + for env_name, env_value in env.project.config['test_env'].items(): + file.write(f"export {env_name}={env_value}\n") diff --git a/builder/actions/setup_cross_ci_helpers.py b/builder/actions/setup_cross_ci_helpers.py index 1419f92ee..968426028 100644 --- a/builder/actions/setup_cross_ci_helpers.py +++ b/builder/actions/setup_cross_ci_helpers.py @@ -215,7 +215,7 @@ def _get_token_slots(env): def _get_softhsm2_version(env): output = _exec_softhsm2_util(env, '--version').output - match = re.match('([0-9+])\.([0-9]+).([0-9]+)', output) + match = re.match(r'([0-9+])\.([0-9]+).([0-9]+)', output) return (int(match.group(1)), int(match.group(2)), int(match.group(3))) ################################################################################ diff --git a/builder/core/fetch.py b/builder/core/fetch.py index d3cd615b1..96a7095b8 100644 --- a/builder/core/fetch.py +++ b/builder/core/fetch.py @@ -236,7 +236,25 @@ def fetch_and_extract(url, archive_path, extract_path): print('Extracting {} to {}'.format(archive_path, extract_path)) if tarfile.is_tarfile(archive_path): with tarfile.open(archive_path) as tar: - tar.extractall(extract_path) + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + safe_extract(tar, extract_path) elif zipfile.is_zipfile(archive_path): with zipfile.ZipFile(archive_path) as zip: diff --git a/builder/core/toolchain.py b/builder/core/toolchain.py index 97ab4020a..c33fc9815 100644 --- a/builder/core/toolchain.py +++ b/builder/core/toolchain.py @@ -17,19 +17,19 @@ def _compiler_version(cc): for text in lines: # Apple clang - m = re.match('Apple (LLVM|clang) version (\d+)', text) + m = re.match(r'Apple (LLVM|clang) version (\d+)', text) if m: return 'appleclang', m.group(2) # LLVM clang - m = re.match('.*(LLVM|clang) version (\d+)', text) + m = re.match(r'.*(LLVM|clang) version (\d+)', text) if m: return 'clang', m.group(2) # GCC 4.x - m = re.match('gcc .+ (4\.\d+)', text) + m = re.match(r'gcc .+ (4\.\d+)', text) if m: return 'gcc', m.group(1) # GCC 5+ - m = re.match('gcc .+ (\d+)\.', text) + m = re.match(r'gcc .+ (\d+)\.', text) if m: return 'gcc', m.group(1) return None, None diff --git a/builder/imports/llvm.py b/builder/imports/llvm.py index ccb932fb7..7a2372fc6 100644 --- a/builder/imports/llvm.py +++ b/builder/imports/llvm.py @@ -130,7 +130,7 @@ def install(self, env): sudo = ['sudo'] if sudo else [] # Strip minor version info - version = env.toolchain.compiler_version.replace('\..+', '') + version = env.toolchain.compiler_version.replace(r'\..+', '') script = tempfile.NamedTemporaryFile(delete=False) script_path = script.name diff --git a/builder/imports/nodejs.py b/builder/imports/nodejs.py index 0d395af35..36ff88266 100644 --- a/builder/imports/nodejs.py +++ b/builder/imports/nodejs.py @@ -14,7 +14,7 @@ import re -NVM = """\ +NVM = r"""\ #!/usr/bin/env bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" @@ -126,7 +126,7 @@ def install_node_via_unofficial_build(self, env): # Normaliz version format, please note 12.16.3 is the last version has x86 support def normalize_version(v): append_times = 0 - while re.match('^([0-9]+\.){2}[0-9]+$', v) == None: + while re.match(r'^([0-9]+\.){2}[0-9]+$', v) == None: # Only try append sub version twice if append_times < 2: v += ".0"