Skip to content

Commit 1071db5

Browse files
erichaagdeverichaagdev
committed
Add offline mode for exp3-gradle
1 parent 7c5387b commit 1071db5

File tree

11 files changed

+247
-173
lines changed

11 files changed

+247
-173
lines changed

‎build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ val mavenComponents by configurations.creating
4141
dependencies {
4242
argbash("argbash:argbash:2.10.0@zip")
4343
commonComponents(project(path = ":fetch-build-scan-data-cmdline-tool", configuration = "shadow"))
44+
commonComponents(project(path = ":mock-scan-dump-to-csv-tool", configuration = "shadow"))
4445
mavenComponents(project(":capture-build-scan-url-maven-extension"))
4546
mavenComponents("com.gradle:gradle-enterprise-maven-extension:1.15.4")
4647
mavenComponents("com.gradle:common-custom-user-data-maven-extension:1.11.1")

‎components/mock-scan-dump-to-csv-tool/build.gradle.kts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
plugins {
2+
java
3+
application
4+
id("com.github.johnrengelman.shadow") version "7.1.2"
5+
}
6+
7+
description = "Application which mocks converting a scan dump to a CSV"
8+
9+
repositories {
10+
mavenCentral()
11+
}
12+
13+
java {
14+
toolchain {
15+
languageVersion.set(JavaLanguageVersion.of(8))
16+
}
17+
}
18+
19+
application {
20+
mainClass.set("com.gradle.enterprise.Main")
21+
}
22+
23+
// See: https://github.com/gradle/gradle/issues/21364
24+
tasks.withType<JavaExec>().configureEach {
25+
if (name.endsWith("main()")) {
26+
notCompatibleWithConfigurationCache("JavaExec created by IntelliJ")
27+
}
28+
}

‎components/mock-scan-dump-to-csv-tool/src/main/java/com/gradle/enterprise/Main.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.gradle.enterprise;
2+
3+
public class Main {
4+
public static void main(String[] args) {
5+
final String headers = "Root Project Name,Gradle Enterprise Server,Build Scan,Build Scan ID,Git URL,Git Branch,Git Commit ID,Requested Tasks,Build Outcome,Remote Build Cache URL,Remote Build Cache Shard,Avoided Up To Date,Avoided up-to-date avoidance savings,Avoided from cache,Avoided from cache avoidance savings,Executed cacheable,Executed cacheable duration,Executed not cacheable,Executed not cacheable duration";
6+
final String scan1 = "java-ordered-properties,https://ge.solutions-team.gradle.com,https://ge.solutions-team.gradle.com/s/mbkuvmcoabzmw,mbkuvmcoabzmw,https://github.com/etiennestuder/java-ordered-properties,ge,b471d917952df7fcc2fab937d404365c4e3e4b4f,clean build,SUCCESS,,,1,0.000s,0,0.000s,3,1.806s,1,0.005s";
7+
final String scan2 = "java-ordered-properties,https://ge.solutions-team.gradle.com,https://ge.solutions-team.gradle.com/s/fugxcgswu2nqq,fugxcgswu2nqq,https://github.com/etiennestuder/java-ordered-properties,ge,b471d917952df7fcc2fab937d404365c4e3e4b4f,clean build,SUCCESS,,,1,0.000s,2,1.211s,9,0.565s,1,0.004s";
8+
System.out.println(String.join("\n", headers, scan1, scan2));
9+
}
10+
}

‎components/scripts/gradle/03-validate-local-build-caching-different-locations.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,12 @@ execute_second_build() {
154154
}
155155

156156
fetch_build_cache_metrics() {
157-
read_build_scan_metadata
158-
fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}"
157+
if [ "${offline_mode}" == "on" ]; then
158+
read_build_scan_from_scan_dump
159+
else
160+
read_build_scan_metadata
161+
fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}"
162+
fi
159163
}
160164

161165
# Overrides info.sh#print_performance_metrics

‎components/scripts/lib/build_scan.sh‎

Lines changed: 8 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,7 @@
11
#!/usr/bin/env bash
22

33
readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-scan-data-cmdline-tool-${SCRIPT_VERSION}-all.jar"
4-
5-
# Arrays used by callers to access the fetched build scan data
6-
project_names=()
7-
base_urls=()
8-
build_scan_urls=()
9-
build_scan_ids=()
10-
git_repos=()
11-
git_branches=()
12-
git_commit_ids=()
13-
requested_tasks=()
14-
build_outcomes=()
15-
# shellcheck disable=SC2034 # not all scripts use this data
16-
remote_build_cache_urls=()
17-
# shellcheck disable=SC2034 # not all scripts use this data
18-
remote_build_cache_shards=()
19-
20-
# Build caching performance metrics
21-
avoided_up_to_date_num_tasks=()
22-
avoided_up_to_date_avoidance_savings=()
23-
avoided_from_cache_num_tasks=()
24-
avoided_from_cache_avoidance_savings=()
25-
executed_cacheable_num_tasks=()
26-
executed_cacheable_duration=()
27-
executed_not_cacheable_num_tasks=()
28-
executed_not_cacheable_duration=()
4+
readonly MOCK_SCAN_DUMP_TO_CSV_JAR="${LIB_DIR}/export-api-clients/mock-scan-dump-to-csv-tool-${SCRIPT_VERSION}-all.jar"
295

306
read_build_scan_metadata() {
317
# This isn't the most robust way to read a CSV,
@@ -47,114 +23,10 @@ read_build_data_from_current_dir() {
4723
requested_tasks+=("${tasks}")
4824
}
4925

50-
fetch_build_scan_data() {
51-
# OS specific support (must be 'true' or 'false').
52-
cygwin=false
53-
msys=false
54-
case "$(uname)" in
55-
CYGWIN* )
56-
cygwin=true
57-
;;
58-
MINGW* )
59-
msys=true
60-
;;
61-
esac
62-
63-
# Determine the Java command to use to start the JVM.
64-
if [ -n "$JAVA_HOME" ] ; then
65-
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
66-
# IBM's JDK on AIX uses strange locations for the executables
67-
JAVACMD="$JAVA_HOME/jre/sh/java"
68-
else
69-
JAVACMD="$JAVA_HOME/bin/java"
70-
fi
71-
if [ ! -x "$JAVACMD" ] ; then
72-
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
73-
74-
Please set the JAVA_HOME variable in your environment to match the
75-
location of your Java installation."
76-
fi
77-
else
78-
JAVACMD="java"
79-
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
80-
81-
Please set the JAVA_HOME variable in your environment to match the
82-
location of your Java installation."
83-
fi
84-
85-
# For Cygwin or MSYS, switch paths to Windows format before running java
86-
if [ "$cygwin" = "true" ] || [ "$msys" = "true" ] ; then
87-
APP_HOME=$(cygpath --path --mixed "$APP_HOME")
88-
CLASSPATH=$(cygpath --path --mixed "$CLASSPATH")
89-
JAVACMD=$(cygpath --unix "$JAVACMD")
90-
91-
# We build the pattern for arguments to be converted via cygpath
92-
ROOTDIRSRAW=$(find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null)
93-
SEP=""
94-
for dir in $ROOTDIRSRAW ; do
95-
ROOTDIRS="$ROOTDIRS$SEP$dir"
96-
SEP="|"
97-
done
98-
OURCYGPATTERN="(^($ROOTDIRS))"
99-
# Add a user-defined pattern to the cygpath arguments
100-
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
101-
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
102-
fi
103-
# Now convert the arguments - kludge to limit ourselves to /bin/sh
104-
i=0
105-
for arg in "$@" ; do
106-
CHECK=$(echo "$arg"|grep -E -c "$OURCYGPATTERN" -)
107-
CHECK2=$(echo "$arg"|grep -E -c "^-") ### Determine if an option
108-
109-
# shellcheck disable=SC2046 # we actually want word splitting
110-
# shellcheck disable=SC2116 # using echo to expand globs
111-
if [ "$CHECK" -ne 0 ] && [ "$CHECK2" -eq 0 ] ; then ### Added a condition
112-
eval $(echo args$i)=$(cygpath --path --ignore --mixed "$arg")
113-
else
114-
eval $(echo args$i)="\"$arg\""
115-
fi
116-
# shellcheck disable=SC2003
117-
i=$(expr $i + 1)
118-
done
119-
# shellcheck disable=SC2154
120-
case $i in
121-
0) set -- ;;
122-
1) set -- "$args0" ;;
123-
2) set -- "$args0" "$args1" ;;
124-
3) set -- "$args0" "$args1" "$args2" ;;
125-
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
126-
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
127-
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
128-
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
129-
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
130-
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
131-
esac
132-
fi
133-
134-
# Escape application args
135-
save () {
136-
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
137-
echo " "
138-
}
139-
APP_ARGS=$(save "$@")
140-
141-
CLASSPATH="${FETCH_BUILD_SCAN_DATA_JAR}"
142-
# Collect all arguments for the java command, following the shell quoting and substitution rules
143-
eval set -- -Dpicocli.ansi=true -jar "\"$CLASSPATH\"" "$APP_ARGS"
144-
145-
# shellcheck disable=SC2154
146-
if [[ "$_arg_debug" == "on" ]]; then
147-
debug "$JAVACMD $*" 1>&2
148-
print_bl 1>&2
149-
fi
150-
151-
exec "$JAVACMD" "$@"
152-
}
153-
15426
fetch_and_read_build_scan_data() {
15527
# This isn't the most robust way to read a CSV,
15628
# but we control the CSV so we don't have to worry about various CSV edge cases
157-
local args build_cache_metrics_only fetched_data header_row_read idx
29+
local args build_cache_metrics_only
15830
args=()
15931

16032
if [[ "$_arg_debug" == "on" ]]; then
@@ -179,49 +51,14 @@ fetch_and_read_build_scan_data() {
17951
fi
18052
shift
18153
args+=( "$@" )
182-
fetched_data="$(fetch_build_scan_data "${args[@]}")"
183-
184-
debug "Raw fetched build scan data"
185-
debug "---------------------------"
186-
debug "${fetched_data}"
187-
debug ""
188-
189-
header_row_read=false
190-
idx=0
19154

192-
# shellcheck disable=SC2034 # not all scripts use all of the fetched data
193-
while IFS=, read -r field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19; do
194-
if [[ "$header_row_read" == "false" ]]; then
195-
header_row_read=true
196-
continue;
197-
fi
198-
199-
if [[ "${build_cache_metrics_only}" != "true" ]]; then
200-
project_names[idx]="$field_1"
201-
base_urls[idx]="$field_2"
202-
build_scan_urls[idx]="$field_3"
203-
build_scan_ids[idx]="$field_4"
204-
git_repos[idx]="$field_5"
205-
git_branches[idx]="$field_6"
206-
git_commit_ids[idx]="$field_7"
207-
requested_tasks[idx]="$(remove_clean_task "${field_8}")"
208-
build_outcomes[idx]="$field_9"
209-
remote_build_cache_urls[idx]="${field_10}"
210-
remote_build_cache_shards[idx]="${field_11}"
211-
fi
212-
213-
# Build caching performance metrics
214-
avoided_up_to_date_num_tasks[idx]="${field_12}"
215-
avoided_up_to_date_avoidance_savings[idx]="${field_13}"
216-
avoided_from_cache_num_tasks[idx]="${field_14}"
217-
avoided_from_cache_avoidance_savings[idx]="${field_15}"
218-
executed_cacheable_num_tasks[idx]="${field_16}"
219-
executed_cacheable_duration[idx]="${field_17}"
220-
executed_not_cacheable_num_tasks[idx]="${field_18}"
221-
executed_not_cacheable_duration[idx]="${field_19}"
55+
raw_build_scan_data="$(invoke_java "$FETCH_BUILD_SCAN_DATA_JAR" "${args[@]}")"
56+
parse_raw_build_scan_data "$raw_build_scan_data" "$build_cache_metrics_only"
57+
}
22258

223-
((idx++))
224-
done <<< "${fetched_data}"
59+
read_build_scan_from_scan_dump() {
60+
raw_build_scan_data="$(invoke_java "$MOCK_SCAN_DUMP_TO_CSV_JAR")"
61+
parse_raw_build_scan_data "$raw_build_scan_data"
22562
}
22663

22764
detect_warnings_from_build_scans() {

‎components/scripts/lib/cli-parsers/gradle/03-cli-parser.m4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Created by argbash-init v2.10.0
44
# ARG_OPTIONAL_BOOLEAN([fail-if-not-fully-cacheable],[f],[])
55
# ARG_HELP([This function is overridden later on.])
6+
# ARG_OPTIONAL_BOOLEAN([offline],[],[],[off])
67
# ARG_VERSION([print_version],[v],[version],[])
78
# ARGBASH_WRAP([common])
89
# ARGBASH_SET_INDENT([ ])

‎components/scripts/lib/config.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ process_arguments() {
6262
if [ -n "${_arg_interactive+x}" ]; then
6363
interactive_mode="${_arg_interactive}"
6464
fi
65+
66+
if [ -n "${_arg_offline+x}" ]; then
67+
offline_mode="${_arg_offline}"
68+
fi
6569
}
6670

6771
validate_required_config() {

‎components/scripts/lib/java.sh‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/env bash
2+
3+
invoke_java() {
4+
local classpath="$1"
5+
shift
6+
local args=( "$@" )
7+
8+
# OS specific support (must be 'true' or 'false').
9+
cygwin=false
10+
msys=false
11+
case "$(uname)" in
12+
CYGWIN*)
13+
cygwin=true
14+
;;
15+
MINGW*)
16+
msys=true
17+
;;
18+
esac
19+
20+
# Determine the Java command to use to start the JVM.
21+
if [ -n "$JAVA_HOME" ]; then
22+
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
23+
# IBM's JDK on AIX uses strange locations for the executables
24+
JAVACMD="$JAVA_HOME/jre/sh/java"
25+
else
26+
JAVACMD="$JAVA_HOME/bin/java"
27+
fi
28+
if [ ! -x "$JAVACMD" ]; then
29+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
30+
31+
Please set the JAVA_HOME variable in your environment to match the
32+
location of your Java installation."
33+
fi
34+
else
35+
JAVACMD="java"
36+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
37+
38+
Please set the JAVA_HOME variable in your environment to match the
39+
location of your Java installation."
40+
fi
41+
42+
# For Cygwin or MSYS, switch paths to Windows format before running java
43+
if [ "$cygwin" = "true" ] || [ "$msys" = "true" ]; then
44+
APP_HOME=$(cygpath --path --mixed "$APP_HOME")
45+
CLASSPATH=$(cygpath --path --mixed "$CLASSPATH")
46+
JAVACMD=$(cygpath --unix "$JAVACMD")
47+
48+
# We build the pattern for arguments to be converted via cygpath
49+
ROOTDIRSRAW=$(find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null)
50+
SEP=""
51+
for dir in $ROOTDIRSRAW; do
52+
ROOTDIRS="$ROOTDIRS$SEP$dir"
53+
SEP="|"
54+
done
55+
OURCYGPATTERN="(^($ROOTDIRS))"
56+
# Add a user-defined pattern to the cygpath arguments
57+
if [ "$GRADLE_CYGPATTERN" != "" ]; then
58+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
59+
fi
60+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
61+
i=0
62+
for arg in "${args[@]}"; do
63+
CHECK=$(echo "$arg" | grep -E -c "$OURCYGPATTERN" -)
64+
CHECK2=$(echo "$arg" | grep -E -c "^-") ### Determine if an option
65+
66+
# shellcheck disable=SC2046 # we actually want word splitting
67+
# shellcheck disable=SC2116 # using echo to expand globs
68+
if [ "$CHECK" -ne 0 ] && [ "$CHECK2" -eq 0 ]; then ### Added a condition
69+
eval $(echo args$i)=$(cygpath --path --ignore --mixed "$arg")
70+
else
71+
eval $(echo args$i)="\"$arg\""
72+
fi
73+
# shellcheck disable=SC2003
74+
i=$(expr $i + 1)
75+
done
76+
# shellcheck disable=SC2154
77+
case $i in
78+
0) set -- ;;
79+
1) set -- "$args0" ;;
80+
2) set -- "$args0" "$args1" ;;
81+
3) set -- "$args0" "$args1" "$args2" ;;
82+
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
83+
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
84+
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
85+
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
86+
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
87+
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
88+
esac
89+
fi
90+
91+
# Escape application args
92+
save() {
93+
for i; do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/"; done
94+
echo " "
95+
}
96+
97+
# Collect all arguments for the java command, following the shell quoting and substitution rules
98+
eval set -- -Dpicocli.ansi=true -jar "\"$classpath\"" "$(save "${args[@]}")"
99+
100+
# shellcheck disable=SC2154
101+
if [[ "$_arg_debug" == "on" ]]; then
102+
debug "$JAVACMD $*" 1>&2
103+
print_bl 1>&2
104+
fi
105+
106+
exec "$JAVACMD" "$@"
107+
}

0 commit comments

Comments
 (0)