Skip to content

Commit

Permalink
chore: bootstrap CRT build for native platforms (#91)
Browse files Browse the repository at this point in the history
  • Loading branch information
aajtodd committed Feb 13, 2024
1 parent 1974bbe commit bbc7cf1
Show file tree
Hide file tree
Showing 31 changed files with 1,322 additions and 47 deletions.
101 changes: 63 additions & 38 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,48 +18,73 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
- uses: actions/cache@v2
uses: actions/checkout@v4
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Build and Test ${{ env.PACKAGE_NAME }}
run: |
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
chmod a+x builder.pyz
echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties
./builder.pyz build -p ${{ env.PACKAGE_NAME }}
macos-compat:
runs-on: macos-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
- uses: actions/cache@v2
submodules: true
- name: Configure JDK
uses: actions/setup-java@v3
with:
distribution: 'corretto'
java-version: 17
cache: 'gradle'
# Cache the Kotlin/Native toolchain based on the input Kotlin version from version catalog
# see https://kotlinlang.org/docs/native-improving-compilation-time.html
- uses: actions/cache@v4
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
~/.konan
key: ${{ runner.os }}-konan-${{ hashFiles('gradle/libs.versions.toml') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Build and Test ${{ env.PACKAGE_NAME }}
${{ runner.os }}-konan-
# FIXME - we will need to migrate this to ECR to avoid throttle limits from dockerhub, for now rebuild the images during CI
- name: Configure Docker Images
run: |
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
chmod a+x builder.pyz
echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties
./builder.pyz build -p ${{ env.PACKAGE_NAME }}
windows-compat:
runs-on: windows-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
./docker-images/build-all.sh
docker images
- name: Build and Test ${{ env.PACKAGE_NAME }}
run: |
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
python3 builder.pyz build -p ${{ env.PACKAGE_NAME }}
docker images
./gradlew apiCheck
./gradlew allTests
# macos-compat:
# runs-on: macos-latest
# steps:
# - name: Checkout sources
# uses: actions/checkout@v4
# with:
# submodules: true
# - name: Configure JDK
# uses: actions/setup-java@v3
# with:
# distribution: 'corretto'
# java-version: 17
# cache: 'gradle'
# - name: Configure Docker Images
# run: |
# ./docker-images/build-all.sh
# - name: Build and Test ${{ env.PACKAGE_NAME }}
# run: |
# ./gradlew apiCheck
# ./gradlew allTests
#
# windows-compat:
# runs-on: windows-latest
# steps:
# - name: Checkout sources
# uses: actions/checkout@v4
# with:
# submodules: true
# - name: Configure JDK
# uses: actions/setup-java@v3
# with:
# distribution: 'corretto'
# java-version: 17
# cache: 'gradle'
# - name: Configure Docker Images
# run: |
# ./docker-images/build-all.sh
# - name: Build and Test ${{ env.PACKAGE_NAME }}
# run: |
# ./gradlew apiCheck
# ./gradlew allTests
33 changes: 33 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
[submodule "crt/aws-c-common"]
path = crt/aws-c-common
url = https://github.com/awslabs/aws-c-common
[submodule "crt/aws-c-auth"]
path = crt/aws-c-auth
url = https://github.com/awslabs/aws-c-auth
[submodule "crt/aws-c-cal"]
path = crt/aws-c-cal
url = https://github.com/awslabs/aws-c-cal
[submodule "crt/aws-c-compression"]
path = crt/aws-c-compression
url = https://github.com/awslabs/aws-c-compression
[submodule "crt/aws-c-http"]
path = crt/aws-c-http
url = https://github.com/awslabs/aws-c-http
[submodule "crt/aws-c-io"]
path = crt/aws-c-io
url = https://github.com/awslabs/aws-c-io
[submodule "crt/aws-checksums"]
path = crt/aws-checksums
url = https://github.com/awslabs/aws-checksums
[submodule "crt/aws-c-mqtt"]
path = crt/aws-c-mqtt
url = https://github.com/awslabs/aws-c-mqtt
[submodule "crt/s2n"]
path = crt/s2n
url = https://github.com/aws/s2n-tls
[submodule "crt/aws-lc"]
path = crt/aws-lc
url = https://github.com/aws/aws-lc
[submodule "crt/aws-c-sdkutils"]
path = crt/aws-c-sdkutils
url = https://github.com/awslabs/aws-c-sdkutils
105 changes: 105 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
cmake_minimum_required(VERSION 3.1)
project(aws-crt-kotlin C)
message(STATUS "CMake ${CMAKE_VERSION}")

option(BUILD_DEPS "Builds aws common runtime dependencies as part of build. Turn off if you want to control your dependency chain." ON)
option(BUILD_SHARED_LIBS "Build shared library for FFI: default: ON" ON)

if (POLICY CMP0069)
cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler
endif()

if (DEFINED CMAKE_PREFIX_PATH)
file(TO_CMAKE_PATH "${CMAKE_PREFIX_PATH}" CMAKE_PREFIX_PATH)
endif()

if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
set(CMAKE_INSTALL_LIBDIR "lib")
endif()

if (UNIX AND NOT APPLE)
set(FIND_LIBRARY_USE_LIB64_PATHS true)
endif()

# This is required in order to append /lib/cmake to each element in CMAKE_PREFIX_PATH
set(AWS_MODULE_DIR "/${CMAKE_INSTALL_LIBDIR}/cmake")
string(REPLACE ";" "${AWS_MODULE_DIR};" AWS_MODULE_PATH "${CMAKE_PREFIX_PATH}${AWS_MODULE_DIR}")
# Append that generated list to the module search path
list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH})


if (BUILD_DEPS)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/crt/aws-c-common/cmake")

include(AwsFindPackage)

set(IN_SOURCE_BUILD ON)
set(SEARCH_LIBCRYPTO OFF CACHE BOOL "Let S2N use libcrypto from AWS-LC.")

# Don't compile tests in subdirectories.
# Turn off using `option` instead of `set`, or CTest will declare
# it as an option later and override the existing variable.
set(BUILD_TESTING OFF)

# Disable BUILD_SHARED_LIBS for all CRT libs
set(SHARED_FFI_LIB ${BUILD_SHARED_LIBS})
set(BUILD_SHARED_LIBS OFF)

# CRT Libraries
add_subdirectory(crt/aws-c-common)
if (UNIX AND NOT APPLE)
if (NOT USE_OPENSSL)
if (NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) AND
NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l))
set(DISABLE_PERL ON)
endif()
if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512")
set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
endif()
set(DISABLE_GO ON)
set(SEARCH_LIBCRYPTO OFF)
set(BUILD_LIBSSL OFF)
add_subdirectory(crt/aws-lc)
else()
set(SEARCH_LIBCRYPTO ON)
# Find the system libcrypto and propagate its location to s2n's find script
find_package(OpenSSL REQUIRED)
set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
set(LibCrypto_LIBRARY ${OPENSSL_CRYPTO_LIBRARY})
if (LibCrypto_LIBRARY MATCHES ".so$")
set(LibCrypto_SHARED_LIBRARY ${LibCrypto_LIBRARY})
else()
set(LibCrypto_STATIC_LIBRARY ${LibCrypto_LIBRARY})
endif()
endif()
add_subdirectory(crt/s2n)
endif()
add_subdirectory(crt/aws-c-sdkutils)
add_subdirectory(crt/aws-c-io)
add_subdirectory(crt/aws-c-cal)
add_subdirectory(crt/aws-c-compression)
add_subdirectory(crt/aws-c-http)
add_subdirectory(crt/aws-c-auth)
add_subdirectory(crt/aws-checksums)
else()
include(AwsFindPackage)
set(IN_SOURCE_BUILD OFF)
endif()

# Restore BUILD_SHARED_LIBS for this project
set(BUILD_SHARED_LIBS ${SHARED_FFI_LIB})

include(AwsCFlags)
include(AwsSharedLibSetup)
include(AwsSanitizers)

aws_use_package(aws-c-common)
aws_use_package(aws-c-sdkutils)
aws_use_package(aws-c-io)
aws_use_package(aws-c-cal)
aws_use_package(aws-c-compression)
aws_use_package(aws-c-http)
aws_use_package(aws-c-auth)
aws_use_package(aws-checksums)

34 changes: 33 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ This project is licensed under the Apache-2.0 License.

## Building

Kotlin Multiplatform projects are in [Alpha](https://kotlinlang.org/docs/reference/evolution/components-stability.html). The CRT interfaces are subject to change.
CRT interfaces are subject to change.

### Linux/Unix
Install some version of libcrypto on which s2n depends. See the [s2n](https://github.com/awslabs/s2n) documentation.
Expand All @@ -23,6 +23,7 @@ apt-get install libssl-dev
```

OR

```sh
yum install openssl-devel
```
Expand All @@ -32,6 +33,36 @@ Set the path to `libcrypto.a` either as a command line argument to gradle `-Plib

### OSX

#### Debugging simulator test issues

**Xcode does not support simulator tests for \<native target>**

```
* What went wrong:
Execution failed for task ':aws-crt-kotlin:iosX64Test'.
> Error while evaluating property 'device' of task ':aws-crt-kotlin:iosX64Test'.
> Failed to calculate the value of task ':aws-crt-kotlin:iosX64Test' property 'device'.
> Xcode does not support simulator tests for ios_x64. Check that requested SDK is installed.
```

Ensure that you have an appropriate simulator runtime installed.

e.g. to install `iOS` platform support including simulator runtimes:
```sh
xcodebuild -downloadPlatform iOS
```

List simulator runtimes with:

```sh
xcrun simctl list devices available
```


See also:

* https://developer.apple.com/documentation/xcode/installing-additional-simulator-runtimes
* https://www.iosdev.recipes/simctl/

### Windows

Expand Down Expand Up @@ -69,3 +100,4 @@ CRTDEBUG=trace=2 ./elasticurl/bin/macosX64/elasticurl.kexe -v trace https://aws.
Run the simple elasticurl integration test script

`./scripts/elasticurl-test.sh`

44 changes: 40 additions & 4 deletions aws-crt-kotlin/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
import aws.sdk.kotlin.gradle.crt.cmakeInstallDir
import aws.sdk.kotlin.gradle.crt.configureCrtCMakeBuild
import aws.sdk.kotlin.gradle.dsl.configurePublishing
import aws.sdk.kotlin.gradle.kmp.IDEA_ACTIVE
import aws.sdk.kotlin.gradle.kmp.configureKmpTargets
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget

plugins {
alias(libs.plugins.kotlin.multiplatform)
alias(libs.plugins.aws.kotlin.repo.tools.kmp)
id("crt-build-support")
}

val sdkVersion: String by project
Expand All @@ -24,6 +29,10 @@ configureKmpTargets()
kotlin {
explicitApi()

iosArm64()
iosSimulatorArm64()
iosX64()

jvm {
attributes {
attribute<org.gradle.api.attributes.java.TargetJvmEnvironment>(
Expand Down Expand Up @@ -60,10 +69,6 @@ kotlin {
}
}

val kotlinVersion: String by project
val coroutinesVersion: String by project
val mockServerVersion: String by project

sourceSets {
val commonMain by getting {
dependencies {
Expand Down Expand Up @@ -116,6 +121,37 @@ kotlin {
sourceSets.all {
optinAnnotations.forEach { languageSettings.optIn(it) }
}

// create a single "umbrella" cinterop will all the aws-c-* API's we want to consume
// see: https://github.com/JetBrains/kotlin-native/issues/2423#issuecomment-466300153
targets.withType<KotlinNativeTarget> {
val knTarget = this
logger.info("configuring $knTarget: ${knTarget.name}")
val cmakeInstallTask = configureCrtCMakeBuild(knTarget)
val targetInstallDir = project.cmakeInstallDir(knTarget)
val headerDir = targetInstallDir.resolve("include")
val libDir = targetInstallDir.resolve("lib")

compilations["main"].cinterops {
val interopDir = "$projectDir/native/interop"
println("configuring crt cinterop for: ${knTarget.name}")
val interopSettings = create("aws-crt") {
defFile("$interopDir/crt.def")
includeDirs(headerDir)
compilerOpts("-L${libDir.absolutePath}")
}

// cinterop tasks processes header files which requires the corresponding CMake build/install to run
val cinteropTask = tasks.named(interopSettings.interopProcessingTaskName)
cinteropTask.configure {
dependsOn(cmakeInstallTask)
}
}

compilations["test"].compilerOptions.configure {
freeCompilerArgs.addAll(listOf("-linker-options", "-L${libDir.absolutePath}"))
}
}
}

// Publishing
Expand Down
Loading

0 comments on commit bbc7cf1

Please sign in to comment.