Skip to content

Commit d373d41

Browse files
rnrornro
authored
Swift SDK installation script improvements (#3429)
Swift SDK installation script improvements ### Motivation: The SDK installation scripts had a few rough edges and didn't verify the artefacts they downloaded. ### Modifications: * Change naming to the more accurate "Swift SDK" where we can without breaking downstream adopters. * Separate Android NDK installation into its own script. * Avoid using `cd` when setting up Android NDK. * Verify checksums for SDKs and signatures for toolchains. * Specify/control the SDK directory. ### Result: * More accurate naming. * More maintainable scripts. * More secure downloads. An example of this working https://github.com/apple/swift-nio/actions/runs/18944235613/job/54090944716?pr=3429
1 parent a24771a commit d373d41

File tree

9 files changed

+167
-47
lines changed

9 files changed

+167
-47
lines changed

‎.github/workflows/android_sdk.yml‎ renamed to ‎.github/workflows/android_swift_sdk.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Android SDK
1+
name: Android Swift SDK
22

33
permissions:
44
contents: read
@@ -12,12 +12,12 @@ on:
1212
default: "{}"
1313
additional_command_arguments:
1414
type: string
15-
description: "Additional arguments passed to swift build (the Android SDK will be specified). Defaults to empty."
15+
description: "Additional arguments passed to swift build (the Android Swift SDK will be specified). Defaults to empty."
1616
default: ""
1717

1818
jobs:
1919
construct-matrix:
20-
name: Construct Android SDK matrix
20+
name: Construct Android Swift SDK matrix
2121
runs-on: ubuntu-latest
2222
outputs:
2323
android-sdk-matrix: '${{ steps.generate-matrix.outputs.android-sdk-matrix }}'
@@ -47,7 +47,7 @@ jobs:
4747
"platform":"Linux",
4848
"runner":"ubuntu-latest",
4949
"image":"ubuntu:jammy",
50-
"setup_command":"apt update -q && apt install -y -q curl jq tar && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_prerequisites.sh | bash && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_sdk.sh | INSTALL_SWIFT_BRANCH=main INSTALL_SWIFT_ARCH=x86_64 INSTALL_SWIFT_SDK=android-sdk bash && hash -r",
50+
"setup_command":"apt update -q && apt install -y -q curl jq tar && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_prerequisites.sh | bash && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_sdk.sh | INSTALL_SWIFT_BRANCH=main INSTALL_SWIFT_ARCH=x86_64 INSTALL_SWIFT_SDK=android-sdk bash && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_android_ndk.sh | bash && hash -r",
5151
"command":"curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/swift-build-with-android-sdk.sh | bash -s --",
5252
"command_arguments":"${{ inputs.additional_command_arguments }}",
5353
"env":'"$env_vars_json"'
@@ -56,11 +56,11 @@ jobs:
5656
}' | jq -c)
5757
EOM
5858
59-
android-sdk:
60-
name: Android SDK
59+
android-swift-sdk:
60+
name: Android Swift SDK
6161
needs: construct-matrix
6262
# Workaround https://github.com/nektos/act/issues/1875
6363
uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
6464
with:
65-
name: "Android SDK"
65+
name: "Android Swift SDK"
6666
matrix_string: '${{ needs.construct-matrix.outputs.android-sdk-matrix }}'

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
android-sdk:
7575
name: Android Swift SDK
7676
# Workaround https://github.com/nektos/act/issues/1875
77-
uses: apple/swift-nio/.github/workflows/android_sdk.yml@main
77+
uses: apple/swift-nio/.github/workflows/android_swift_sdk.yml@main
7878

7979
macos-tests:
8080
name: macOS tests

‎.github/workflows/pull_request.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ jobs:
108108
android-sdk:
109109
name: Android Swift SDK
110110
# Workaround https://github.com/nektos/act/issues/1875
111-
uses: apple/swift-nio/.github/workflows/android_sdk.yml@main
111+
uses: apple/swift-nio/.github/workflows/android_swift_sdk.yml@main
112112

113113
release-builds:
114114
name: Release builds

‎.github/workflows/static_sdk.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
"image":"ubuntu:jammy",
5050
"setup_command":"apt update -q && apt install -y -q curl jq tar && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_prerequisites.sh | bash && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_sdk.sh | INSTALL_SWIFT_VERSION=latest INSTALL_SWIFT_ARCH=x86_64 bash && hash -r",
5151
"command":"swift build",
52-
"command_arguments":"${{ inputs.command_arguments }}",
52+
"command_arguments":"--swift-sdks-path /tmp/swiftsdks ${{ inputs.command_arguments }}",
5353
"env":'"$env_vars_json"'
5454
},
5555
{
@@ -60,7 +60,7 @@ jobs:
6060
"image":"ubuntu:jammy",
6161
"setup_command":"apt update -q && apt install -y -q curl jq tar && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_prerequisites.sh | bash && curl -s --retry 3 https://raw.githubusercontent.com/apple/swift-nio/main/scripts/install_swift_sdk.sh | INSTALL_SWIFT_BRANCH=main INSTALL_SWIFT_ARCH=x86_64 bash && hash -r",
6262
"command":"swift build",
63-
"command_arguments":"${{ inputs.command_arguments }}",
63+
"command_arguments":"--swift-sdks-path /tmp/swiftsdks ${{ inputs.command_arguments }}",
6464
"env":'"$env_vars_json"'
6565
}
6666
]

‎scripts/install_android_ndk.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
##===----------------------------------------------------------------------===##
3+
##
4+
## This source file is part of the SwiftNIO open source project
5+
##
6+
## Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
7+
## Licensed under Apache License v2.0
8+
##
9+
## See LICENSE.txt for license information
10+
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
11+
##
12+
## SPDX-License-Identifier: Apache-2.0
13+
##
14+
##===----------------------------------------------------------------------===##
15+
16+
set -uo pipefail
17+
18+
log() { printf -- "** %s\n" "$*" >&2; }
19+
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
20+
fatal() { error "$@"; exit 1; }
21+
22+
# Parameter environment variables
23+
# see https://developer.android.com/ndk/downloads for releases and shasums
24+
android_ndk_version="${INSTALL_ANDROID_NDK:-"r27d"}"
25+
android_ndk_sha1="${ANDROID_NDK_SHA1:-"22105e410cf29afcf163760cc95522b9fb981121"}"
26+
swift_sdk_directory="${SWIFT_SDK_DIRECTORY:-"/tmp/swiftsdks"}"
27+
28+
CURL_BIN="${CURL_BIN:-$(which curl 2> /dev/null)}"; test -n "$CURL_BIN" || fatal "CURL_BIN unset and no curl on PATH"
29+
UNZIP_BIN="${UNZIP_BIN:-$(which unzip 2> /dev/null)}"; test -n "$UNZIP_BIN" || fatal "UNZIP_BIN unset and no unzip on PATH"
30+
SHASUM_BIN="${SHASUM_BIN:-$(which shasum 2> /dev/null)}"; test -n "$SHASUM_BIN" || fatal "SHASUM_BIN unset and no shasum on PATH"
31+
32+
# download and link the NDK
33+
android_ndk_url="https://dl.google.com/android/repository/android-ndk-${android_ndk_version}-$(uname -s).zip"
34+
35+
log "Android Native Development Kit URL: $android_ndk_url"
36+
"$CURL_BIN" -fsSL -o android_ndk.zip --retry 3 "$android_ndk_url"
37+
38+
# Verify checksum
39+
log "Verifying Android NDK checksum..."
40+
actual_sha1="$("$SHASUM_BIN" -a 1 android_ndk.zip | cut -d' ' -f1)"
41+
test "$actual_sha1" = "$android_ndk_sha1" || fatal "Android NDK checksum mismatch: expected $android_ndk_sha1, got $actual_sha1"
42+
43+
"$UNZIP_BIN" -d "$swift_sdk_directory" -q android_ndk.zip
44+
bundledir="$(find "$swift_sdk_directory" -maxdepth 2 -type d -name '*android*.artifactbundle' | head -n 1)"
45+
test -n "$bundledir" || fatal "Could not find Android artifact bundle directory (expected '*android*.artifactbundle')"
46+
47+
export ANDROID_NDK_HOME="${swift_sdk_directory}/android-ndk-${android_ndk_version}"
48+
"${bundledir}/swift-android/scripts/setup-android-sdk.sh"

‎scripts/install_swift_prerequisites.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ else
2828
fatal "Cannot find either 'apt' or 'yum'"
2929
fi
3030

31-
log "Installing standard Swift pre-requisites" # pre-reqs list taken from swift.org
31+
log "Installing standard Swift prerequisites" # pre-reqs list taken from swift.org
3232
DEBIAN_FRONTEND=noninteractive "$PACKAGE_MANAGER_BIN" install -y\
3333
binutils\
3434
git\

‎scripts/install_swift_sdk.sh‎

Lines changed: 81 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ error() { printf -- "** ERROR: %s\n" "$*" >&2; }
2020
fatal() { error "$@"; exit 1; }
2121

2222
# Parameter environment variables
23-
branch="${INSTALL_SWIFT_BRANCH:=""}"
24-
version="${INSTALL_SWIFT_VERSION:=""}"
25-
arch="${INSTALL_SWIFT_ARCH:="aarch64"}"
26-
os_image="${INSTALL_SWIFT_OS_IMAGE:="ubuntu22.04"}"
27-
sdk="${INSTALL_SWIFT_SDK:="static-sdk"}"
28-
android_ndk_version="${INSTALL_ANDROID_NDK:="r27d"}"
23+
branch="${INSTALL_SWIFT_BRANCH:-""}"
24+
version="${INSTALL_SWIFT_VERSION:-""}"
25+
arch="${INSTALL_SWIFT_ARCH:-"aarch64"}"
26+
os_image="${INSTALL_SWIFT_OS_IMAGE:-"ubuntu22.04"}"
27+
sdk="${INSTALL_SWIFT_SDK:-"static-sdk"}"
28+
swift_sdk_directory="${SWIFT_SDK_DIRECTORY:-"/tmp/swiftsdks"}"
2929

3030
if [[ ! ( -n "$branch" && -z "$version" ) && ! ( -z "$branch" && -n "$version") ]]; then
3131
fatal "Exactly one of build or version must be defined."
@@ -35,6 +35,9 @@ CURL_BIN="${CURL_BIN:-$(which curl 2> /dev/null)}"; test -n "$CURL_BIN" || fatal
3535
TAR_BIN="${TAR_BIN:-$(which tar 2> /dev/null)}"; test -n "$TAR_BIN" || fatal "TAR_BIN unset and no tar on PATH"
3636
JQ_BIN="${JQ_BIN:-$(which jq 2> /dev/null)}"; test -n "$JQ_BIN" || fatal "JQ_BIN unset and no jq on PATH"
3737
SED_BIN="${SED_BIN:-$(which sed 2> /dev/null)}"; test -n "$SED_BIN" || fatal "SED_BIN unset and no sed on PATH"
38+
SHASUM_BIN="${SHASUM_BIN:-$(which shasum 2> /dev/null)}"; test -n "$SHASUM_BIN" || fatal "SHASUM_BIN unset and no shasum on PATH"
39+
GPG_BIN="${GPG_BIN:-$(which gpg 2> /dev/null)}"; test -n "$GPG_BIN" || fatal "GPG_BIN unset and no gpg on PATH"
40+
ZCAT_BIN="${ZCAT_BIN:-$(which zcat 2> /dev/null)}"; test -n "$ZCAT_BIN" || fatal "ZCAT_BIN unset and no zcat on PATH"
3841

3942
case "$arch" in
4043
"aarch64")
@@ -65,6 +68,25 @@ esac
6568

6669
os_image_sanitized="${os_image//./}"
6770

71+
# Function to extract checksum from release info
72+
extract_checksum() {
73+
local release_info="$1"
74+
if [[ -z "$release_info" ]]; then
75+
log "Warning: No release information available"
76+
return
77+
fi
78+
79+
local checksum
80+
# shellcheck disable=SC2016 # Our use of JQ_BIN means that shellcheck can't tell this is a `jq` invocation
81+
checksum=$(echo "$release_info" | "$JQ_BIN" -r --arg platform "$sdk" '.platforms[] | select(.platform == $platform) | .checksum // empty')
82+
if [[ -n "$checksum" ]]; then
83+
log "Found checksum for $sdk: $checksum"
84+
echo "$checksum"
85+
else
86+
log "Warning: No checksum available for $sdk platform"
87+
fi
88+
}
89+
6890
if [[ -n "$branch" ]]; then
6991
# Some snapshots may not have all the artefacts we require
7092
log "Discovering branch snapshot for branch $branch"
@@ -95,13 +117,27 @@ if [[ -n "$branch" ]]; then
95117
elif [[ -n "$version" ]]; then
96118
if [[ "$version" == "latest" ]]; then
97119
log "Discovering latest version"
98-
version=$("$CURL_BIN" -s https://www.swift.org/api/v1/install/releases.json | "$JQ_BIN" -r '.[-1].tag' | "$SED_BIN" -E 's/swift-([0-9]+\.[0-9]+\.?[0-9]*)-RELEASE/\1/')
120+
release_info=$("$CURL_BIN" -s https://www.swift.org/api/v1/install/releases.json | "$JQ_BIN" -r '.[-1]')
121+
if [[ -z "$release_info" ]]; then
122+
log "Warning: Could not find release information for version $version"
123+
fi
124+
version=$(echo "$release_info" | "$JQ_BIN" -r '.tag' | "$SED_BIN" -E 's/swift-([0-9]+\.[0-9]+\.?[0-9]*)-RELEASE/\1/')
99125
if [[ -z "$version" ]]; then
100126
fatal "Failed to discover latest Swift version"
101127
fi
102128
log "Discovered latest Swift version: $version"
129+
130+
else
131+
# For specific versions, we need to fetch the release info
132+
log "Getting release information for version $version"
133+
# shellcheck disable=SC2016 # Our use of JQ_BIN means that shellcheck can't tell this is a `jq` invocation
134+
release_info=$("$CURL_BIN" -s https://www.swift.org/api/v1/install/releases.json | "$JQ_BIN" -r --arg ver "swift-$version-RELEASE" '.[] | select(.tag == $ver)')
135+
if [[ -z "$release_info" ]]; then
136+
log "Warning: Could not find release information for version $version"
137+
fi
103138
fi
104139

140+
expected_checksum=$(extract_checksum "$release_info")
105141
snapshot_url="https://download.swift.org/swift-${version}-release/${os_image_sanitized}${arch_suffix}/swift-${version}-RELEASE/swift-${version}-RELEASE-${os_image}${arch_suffix}.tar.gz"
106142
sdk_url="https://download.swift.org/swift-${version}-release/${sdk_dir}/swift-${version}-RELEASE/swift-${version}-RELEASE${sdk_suffix}.artifactbundle.tar.gz"
107143
fi
@@ -111,6 +147,28 @@ log "Snapshot URL: $snapshot_url"
111147
snapshot_path="/tmp/$(basename "$snapshot_url")"
112148
"$CURL_BIN" -sfL "$snapshot_url" -o "$snapshot_path" || fatal "Failed to download Swift toolchain"
113149

150+
# Import Swift's public key for GPG verification
151+
152+
keys_url="https://swift.org/keys/all-keys.asc"
153+
keys_path="/tmp/all-keys.asc"
154+
"$CURL_BIN" -sL "$keys_url" -o "$keys_path" || fatal "Failed to download Swift's public keys"
155+
"$ZCAT_BIN" -f "$keys_path" | "$GPG_BIN" --import - || fatal "Failed to import Swift's public keys"
156+
157+
# Verify GPG signature for all Swift toolchain downloads
158+
signature_url="${snapshot_url}.sig"
159+
signature_path="${snapshot_path}.sig"
160+
161+
log "Downloading signature from: $signature_url"
162+
echo "$CURL_BIN" -sfL "$signature_url" -o "$signature_path"
163+
"$CURL_BIN" -sfL "$signature_url" -o "$signature_path" || fatal "Failed to download Swift toolchain signature"
164+
165+
# Verify the signature
166+
log "Verifying Swift toolchain GPG signature..."
167+
"$GPG_BIN" --verify "$signature_path" "$snapshot_path" || fatal "Swift toolchain GPG signature verification failed"
168+
169+
# Clean up signature file
170+
rm -f "$signature_path"
171+
114172
log "Installing Swift toolchain"
115173
mkdir -p /tmp/snapshot
116174
"$TAR_BIN" xfz "$snapshot_path" --strip-components 1 -C /
@@ -120,28 +178,26 @@ log "Swift SDK URL: $sdk_url"
120178
sdk_path="/tmp/$(basename "$sdk_url")"
121179
"$CURL_BIN" -sfL "$sdk_url" -o "$sdk_path" || fatal "Failed to download Swift SDK"
122180

181+
# Verify SDK checksum if available
182+
if [[ -n "${expected_checksum:-}" ]]; then
183+
log "Verifying Swift SDK checksum..."
184+
actual_checksum=$("$SHASUM_BIN" -a 256 "$sdk_path" | cut -d' ' -f1)
185+
if [[ "$actual_checksum" = "$expected_checksum" ]]; then
186+
log "Swift SDK checksum verified successfully"
187+
else
188+
fatal "Swift SDK checksum mismatch: expected $expected_checksum, got $actual_checksum"
189+
fi
190+
else
191+
log "Skipping checksum verification (no checksum available)"
192+
fi
193+
123194
log "Looking for swift"
124195
which swift || fatal "Failed to locate installed Swift"
125196

126197
log "Checking swift"
127198
swift --version
128199

129200
log "Installing Swift SDK"
130-
swift sdk install "$sdk_path"
131-
132-
log "Swift SDK Post-install"
133-
if [[ "$sdk" == "android-sdk" ]]; then
134-
# guess some common places where the swift-sdks folder lives
135-
cd ~/Library/org.swift.swiftpm || cd ~/.config/swiftpm || cd ~/.local/swiftpm || cd ~/.swiftpm || cd /root/.swiftpm || fatal "Cannot locate swiftpm config directory"
136-
137-
# download and link the NDK
138-
android_ndk_url="https://dl.google.com/android/repository/android-ndk-${android_ndk_version}-$(uname -s).zip"
139-
log "Android Native Development Kit URL: ${android_ndk_url}"
140-
"$CURL_BIN" -fsSL -o ndk.zip --retry 3 "${android_ndk_url}"
141-
unzip -q ndk.zip
142-
rm ndk.zip
143-
export ANDROID_NDK_HOME="${PWD}/android-ndk-${android_ndk_version}"
144-
bundledir=$(find . -type d -maxdepth 2 -name '*android*.artifactbundle' | head -n 1)
145-
"${bundledir}"/swift-android/scripts/setup-android-sdk.sh
146-
cd - || fatal "Cannot cd back to previous directory"
147-
fi
201+
log "Using Swift SDK directory: $swift_sdk_directory"
202+
mkdir -p "$swift_sdk_directory/swift-sdks"
203+
swift sdk install --swift-sdks-path "$swift_sdk_directory" "$sdk_path"

‎scripts/swift-build-with-android-sdk.sh‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,20 @@
1515

1616
set -uo pipefail
1717

18+
log() { printf -- "** %s\n" "$*" >&2; }
19+
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
20+
fatal() { error "$@"; exit 1; }
21+
22+
# Parameter environment variables
23+
swift_sdk_directory="${SWIFT_SDK_DIRECTORY:-"/tmp/swiftsdks"}"
24+
25+
log "Using Swift SDK directory: $swift_sdk_directory"
26+
1827
# Select the Swift SDK for Android
19-
SWIFT_SDK="$(swift sdk list | grep android | head -n1)"
28+
SWIFT_SDK="$(swift sdk list --swift-sdks-path "$swift_sdk_directory" | grep android | head -n1)"
2029
if [[ -z "$SWIFT_SDK" ]]; then
21-
echo "No Android Swift SDK found. Please ensure you have the Android Swift SDK installed."
22-
exit 1
30+
fatal "No Android Swift SDK found. Please ensure you have the Android Swift SDK installed."
2331
fi
2432

25-
echo "Using Swift SDK: $SWIFT_SDK"
26-
swift build --swift-sdk "$SWIFT_SDK" "${@}"
33+
log "Building using Swift SDK: $SWIFT_SDK"
34+
swift build --swift-sdk "$SWIFT_SDK" --swift-sdks-path "$swift_sdk_directory" "${@}"

‎scripts/swift-build-with-wasm-sdk.sh‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,20 @@
1515

1616
set -uo pipefail
1717

18+
log() { printf -- "** %s\n" "$*" >&2; }
19+
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
20+
fatal() { error "$@"; exit 1; }
21+
22+
# Parameter environment variables
23+
swift_sdk_directory="${SWIFT_SDK_DIRECTORY:-"/tmp/swiftsdks"}"
24+
25+
log "Using Swift SDK directory: $swift_sdk_directory"
26+
1827
# Select the Swift SDK for WebAssembly, not the Embedded one
19-
SWIFT_SDK="$(swift sdk list | grep _wasm | grep -v -embedded | head -n1)"
28+
SWIFT_SDK="$(swift sdk list --swift-sdks-path "$swift_sdk_directory" | grep _wasm | grep -v -embedded | head -n1)"
2029
if [[ -z "$SWIFT_SDK" ]]; then
21-
echo "No WebAssembly Swift SDK found. Please ensure you have the WebAssembly Swift SDK installed following https://www.swift.org/documentation/articles/wasm-getting-started.html."
22-
exit 1
30+
fatal "No WebAssembly Swift SDK found. Please ensure you have the WebAssembly Swift SDK installed following https://www.swift.org/documentation/articles/wasm-getting-started.html."
2331
fi
2432

25-
echo "Using Swift SDK: $SWIFT_SDK"
26-
swift build --swift-sdk "$SWIFT_SDK" "${@}"
33+
log "Building using Swift SDK: $SWIFT_SDK"
34+
swift build --swift-sdk "$SWIFT_SDK" --swift-sdks-path "$swift_sdk_directory" "${@}"

0 commit comments

Comments
 (0)