Skip to content

Cache the apt packages and Chrome binaries #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Cache the apt packages and Chrome binaries #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
68991f5
Actually use the cache.
wycats Aug 28, 2025
47d14c4
Cache the Chrome binaries
wycats Aug 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 29 additions & 13 deletions bin/compile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ topic "Installing Chrome for Testing"
INSTALL_DIR="$BUILD_DIR/.chrome-for-testing"
mkdir -p "$INSTALL_DIR"

# Set up cache directories for Chrome downloads
CHROME_CACHE_DIR="$CACHE_DIR/chrome-for-testing"
mkdir -p "$CHROME_CACHE_DIR"

# Detect if the old config var is still used
if [ -f "$ENV_DIR/CHROMEDRIVER_VERSION" ]; then
error "This buildpack no longer supports CHROMEDRIVER_VERSION config var and must be removed to continue, because this buildpack now installs compatible versions of Chrome & Chromedriver. GOOGLE_CHROME_CHANNEL can be used to set Stable (default), Beta, Dev, or Canary."
Expand All @@ -58,19 +62,31 @@ CHANNEL="$(echo -n "$channel" | awk '{ print toupper($0) }')"
VERSION="$(curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_$CHANNEL")"
echo "Resolved $CHANNEL version $VERSION" | indent

echo "Downloading Chrome" | indent
ZIP_URL="https://storage.googleapis.com/chrome-for-testing-public/$VERSION/linux64/chrome-linux64.zip"
ZIP_LOCATION="$INSTALL_DIR/chrome.zip"
curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 -o "${ZIP_LOCATION}" "${ZIP_URL}" | indent
unzip -q -o "$ZIP_LOCATION" -d "$INSTALL_DIR" | indent
rm -f "$ZIP_LOCATION"

echo "Downloading Chromedriver" | indent
ZIP_URL="https://storage.googleapis.com/chrome-for-testing-public/$VERSION/linux64/chromedriver-linux64.zip"
ZIP_LOCATION="$INSTALL_DIR/chromedriver.zip"
curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 -o "${ZIP_LOCATION}" "${ZIP_URL}" | indent
unzip -q -o "$ZIP_LOCATION" -d "$INSTALL_DIR" | indent
rm -f "$ZIP_LOCATION"
# Check if Chrome is already cached for this version
CHROME_CACHE_FILE="$CHROME_CACHE_DIR/chrome-$VERSION.zip"
if [ -f "$CHROME_CACHE_FILE" ]; then
echo "Using cached Chrome $VERSION" | indent
else
echo "Downloading Chrome" | indent
ZIP_URL="https://storage.googleapis.com/chrome-for-testing-public/$VERSION/linux64/chrome-linux64.zip"
curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 -o "${CHROME_CACHE_FILE}" "${ZIP_URL}" | indent
fi
unzip -q -o "$CHROME_CACHE_FILE" -d "$INSTALL_DIR" | indent

# Check if Chromedriver is already cached for this version
CHROMEDRIVER_CACHE_FILE="$CHROME_CACHE_DIR/chromedriver-$VERSION.zip"
if [ -f "$CHROMEDRIVER_CACHE_FILE" ]; then
echo "Using cached Chromedriver $VERSION" | indent
else
echo "Downloading Chromedriver" | indent
ZIP_URL="https://storage.googleapis.com/chrome-for-testing-public/$VERSION/linux64/chromedriver-linux64.zip"
curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 -o "${CHROMEDRIVER_CACHE_FILE}" "${ZIP_URL}" | indent
fi
unzip -q -o "$CHROMEDRIVER_CACHE_FILE" -d "$INSTALL_DIR" | indent

# Clean up old cached versions to prevent cache bloat (keep only 3 most recent)
ls -t "$CHROME_CACHE_DIR"/chrome-*.zip 2>/dev/null | tail -n +4 | xargs rm -f 2>/dev/null || true
ls -t "$CHROME_CACHE_DIR"/chromedriver-*.zip 2>/dev/null | tail -n +4 | xargs rm -f 2>/dev/null || true

source "$BUILDPACK_DIR/bin/install-chrome-dependencies"

Expand Down
72 changes: 54 additions & 18 deletions bin/install-chrome-dependencies
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ esac
# time, so we have to match that, given this buildpack and the APT buildpack share all the
# other cached APT directories.
STACK_VERSION_FILE="${CACHE_DIR}/.apt/STACK"
CACHE_TIMESTAMP_FILE="${CACHE_DIR}/.apt/TIMESTAMP"
CACHE_MAX_AGE_DAYS=7

if [[ -d "${CACHE_DIR}/apt" ]]; then
if [[ -f "${STACK_VERSION_FILE}" ]]; then
Expand All @@ -66,21 +68,37 @@ if [[ -d "${CACHE_DIR}/apt" ]]; then
CACHED_STACK=
fi

if [[ "${CACHED_STACK}" == "${STACK}" ]]; then
echo "Reusing APT cache" | indent
elif [[ -z "${CACHED_STACK}" ]]; then
# Older versions of the buildpack won't have written the version file.
# (Plus any other broken APT-related buildpacks that don't invalidate on cache change.)
echo "Clearing APT cache since it's missing the stack version metadata file." | indent
rm -rf "${CACHE_DIR}/apt"
else
echo "Clearing APT cache since stack changed from ${CACHED_STACK} to ${STACK}" | indent
rm -rf "${CACHE_DIR}/apt"
# Check cache age
if [[ -f "${CACHE_TIMESTAMP_FILE}" ]]; then
CACHE_TIMESTAMP=$(cat "${CACHE_TIMESTAMP_FILE}")
CURRENT_TIMESTAMP=$(date +%s)
CACHE_AGE_DAYS=$(( (CURRENT_TIMESTAMP - CACHE_TIMESTAMP) / 86400 ))

if [[ ${CACHE_AGE_DAYS} -gt ${CACHE_MAX_AGE_DAYS} ]]; then
echo "Clearing APT cache since it's ${CACHE_AGE_DAYS} days old (max: ${CACHE_MAX_AGE_DAYS} days)" | indent
rm -rf "${CACHE_DIR}/apt"
fi
fi

# Check if cache still exists after potential age-based clearing
if [[ -d "${CACHE_DIR}/apt" ]]; then
if [[ "${CACHED_STACK}" == "${STACK}" ]]; then
echo "Reusing APT cache" | indent
elif [[ -z "${CACHED_STACK}" ]]; then
# Older versions of the buildpack won't have written the version file.
# (Plus any other broken APT-related buildpacks that don't invalidate on cache change.)
echo "Clearing APT cache since it's missing the stack version metadata file." | indent
rm -rf "${CACHE_DIR}/apt"
else
echo "Clearing APT cache since stack changed from ${CACHED_STACK} to ${STACK}" | indent
rm -rf "${CACHE_DIR}/apt"
fi
fi
fi

mkdir -p "${CACHE_DIR}/.apt"
echo "${STACK}" > "${STACK_VERSION_FILE}"
date +%s > "${CACHE_TIMESTAMP_FILE}"

APT_CACHE_DIR="$CACHE_DIR/apt/cache"
APT_STATE_DIR="$CACHE_DIR/apt/state"
Expand All @@ -90,9 +108,14 @@ mkdir -p "$APT_STATE_DIR/lists/partial"

APT_OPTIONS="-o debug::nolocking=true -o dir::cache=$APT_CACHE_DIR -o dir::state=$APT_STATE_DIR"

echo "Updating APT package index" | indent
apt-get $APT_OPTIONS update | indent
if [[ ! -d "${CACHE_DIR}/apt" ]] || [[ "${CACHED_STACK}" != "${STACK}" ]]; then
echo "Updating APT package index" | indent
apt-get $APT_OPTIONS update | indent
else
echo "Using cached APT package index" | indent
fi

PACKAGES_NEEDED=false
for PACKAGE in $PACKAGES; do
if [[ $PACKAGE == *deb ]]; then
PACKAGE_NAME=$(basename $PACKAGE .deb)
Expand All @@ -101,17 +124,30 @@ for PACKAGE in $PACKAGES; do
echo "Fetching $PACKAGE" | indent
curl -s -L -z $PACKAGE_FILE -o $PACKAGE_FILE $PACKAGE 2>&1 | indent
else
echo "Fetching .debs for $PACKAGE" | indent
apt-get $APT_OPTIONS -y --force-yes -d install --reinstall --no-install-recommends $PACKAGE | indent
# Check if package needs to be downloaded
OUTPUT=$(apt-get $APT_OPTIONS -y --force-yes -d install --no-install-recommends $PACKAGE 2>&1)
if echo "$OUTPUT" | grep -q "Need to get [1-9]"; then
PACKAGES_NEEDED=true
echo "Fetching .debs for $PACKAGE" | indent
echo "$OUTPUT" | indent
fi
fi
done

mkdir -p $BUILD_DIR/.apt

for DEB in $(ls -1 $APT_CACHE_DIR/archives/*.deb); do
echo "Installing $(basename $DEB)" | indent
dpkg -x $DEB $BUILD_DIR/.apt/
done
if [[ "$PACKAGES_NEEDED" == "true" ]]; then
echo "Installing packages" | indent
for DEB in $(ls -1 $APT_CACHE_DIR/archives/*.deb); do
echo " $(basename $DEB)" | indent
dpkg -x $DEB $BUILD_DIR/.apt/
done
else
echo "Extracting cached packages" | indent
for DEB in $(ls -1 $APT_CACHE_DIR/archives/*.deb); do
dpkg -x $DEB $BUILD_DIR/.apt/
done
fi

echo "Writing profile script" | indent
mkdir -p $BUILD_DIR/.profile.d
Expand Down