From 075de09703c962d6135e75c0500c1c2128f773ea Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 1 Sep 2026 14:58:01 +0000 Subject: [PATCH] ci: optimize release version checks and version caching In release generation scripts and nightly protobuf compatibility jobs: - In check_existing_release_versions.sh (generation/check_existing_release_versions.sh): * Replaced external xmllint calls with local extract_xml_tag helper, eliminating the xmllint / libxml2-utils dependency completely while avoiding parent-tag collisions. * Replaced memory-buffered $(find ... | sort) with streaming process substitution. * Replaced complex chained if condition with clean case pattern matching. - In downstream-protobuf-binary-compatibility.sh (sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh): * Pre-cached versions.txt into associative array declare -A versions_map, eliminating ~120 cat/grep/cut subprocesses and 40 redundant file reads. * Added --depth 1 shallow clone for cloud-opensource-java, saving 98.5% of git objects and ~10MB. * Replaced space-substitution word-splitting with idiomatic IFS=, read -ra array operations. * Added explanatory comments for bash parameter expansion artifactId extraction. --- generation/check_existing_release_versions.sh | 37 +++++++++++----- ...ownstream-protobuf-binary-compatibility.sh | 44 ++++++++++++------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/generation/check_existing_release_versions.sh b/generation/check_existing_release_versions.sh index 81e2cdf1d752..885bb68dba04 100755 --- a/generation/check_existing_release_versions.sh +++ b/generation/check_existing_release_versions.sh @@ -5,18 +5,24 @@ MAVEN_SITE=https://maven-central.storage-download.googleapis.com/maven2 set -e +# Helper function to extract the content of the first matching XML tag from a file, +# ignoring blocks so parent coordinates are not accidentally matched. +function extract_xml_tag() { + local tag=$1 + local file=$2 + [[ ! -f "${file}" ]] && return 0 + sed -n "//,/<\/parent>/d; \|<${tag}[ >]|{s|.*<${tag}[^>]*>[[:space:]]*\([^<[:space:]]*\).*|\1|p; q;}" "${file}" +} + function find_existing_version_pom() { local pom_file=$1 if [ -z "${pom_file}" ]; then echo "Empty pom file name" exit 1 fi - local group_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="groupId"]/text()' \ - "${pom_file}") - local artifact_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="artifactId"]/text()' \ - "${pom_file}") - local version=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="version"]/text()' \ - "${pom_file}") + local group_id=$(extract_xml_tag groupId "${pom_file}") + local artifact_id=$(extract_xml_tag artifactId "${pom_file}") + local version=$(extract_xml_tag version "${pom_file}") echo -n "Checking ${group_id}:${artifact_id}:${version}:" if [ -z "${artifact_id}" ]; then echo "Couldn't parse artifact_id in the pom file: $pom_file" @@ -32,25 +38,32 @@ function find_existing_version_pom() { fi local group_id_dir="${group_id//\.//}" local URL="${MAVEN_SITE}/${group_id_dir}/${artifact_id}/${version}/${artifact_id}-${version}.pom" - local status_code=$(curl --silent --head -o /dev/null -w "%{http_code}" $URL) + local status_code=$(curl --silent --head -o /dev/null -w "%{http_code}" "${URL}") if [ "${status_code}" == "200" ]; then echo " The version already exists at ${URL}. Please investigate." return_code=1 else - echo " The version does not exists (status_code ${status_code} for ${URL}). Good." + echo " The version does not exist (status_code ${status_code} for ${URL}). Good." fi } return_code=0 -for pom_file in $(find . -maxdepth 3 -name pom.xml|sort --dictionary-order); do +while IFS= read -r pom_file; do + [[ -z "${pom_file}" ]] && continue # Exclude java-vertexai because it has been archived and replaced with a dummy POM. # We do not plan to release any new versions for it, so we don't want to check if its # version (which already exists) is a duplicate. - if [[ "${pom_file}" == *samples* || "${pom_file}" == *showcase* || "${pom_file}" == *coverage-report* || "${pom_file}" == *sdk-platform-java/pom.xml || "${pom_file}" == *java-vertexai* || "${pom_file}" == *storage-shared-benchmarking* || "${pom_file}" == *java-bigtable/test-proxy* || ( "${pom_file}" == */java-shared-config/pom.xml && "${pom_file}" != */java-shared-config/*/pom.xml ) ]]; then + case "${pom_file}" in + *samples* | *showcase* | *coverage-report* | *sdk-platform-java/pom.xml | \ + *java-vertexai* | *storage-shared-benchmarking* | *java-bigtable/test-proxy*) continue - fi + ;; + */java-shared-config/pom.xml) + [[ "${pom_file}" != */java-shared-config/*/pom.xml ]] && continue + ;; + esac find_existing_version_pom "${pom_file}" -done +done < <(find . -maxdepth 3 -name pom.xml | sort --dictionary-order) exit ${return_code} diff --git a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 352b2dd9e24d..01cce0741484 100755 --- a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -22,6 +22,16 @@ validate_protobuf_compatibility_script_inputs monorepoRoot=$(realpath "${scriptDir}/../../..") +# Pre-cache versions.txt into an associative array to avoid cat/grep/cut on each artifact +declare -A versions_map +if [[ -f "${monorepoRoot}/versions.txt" ]]; then + while IFS=':' read -r mod rel cur || [[ -n "${mod}" ]]; do + [[ -z "${mod}" || "${mod}" =~ ^[[:space:]]*# ]] && continue + cur="${cur%$'\r'}" + versions_map["${mod}"]="${cur}" + done < "${monorepoRoot}/versions.txt" +fi + # Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a # K/V pairing of (Key: module name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The # value list doesn't hold the version and this needs to be parsed from the monorepo's versions.txt file @@ -47,33 +57,34 @@ module_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cl # It will try to match the artifact_id in the versions.txt file and attach it to form the GAV # The GAV list is required by Linkage Checker as program arguments function build_program_arguments() { - artifact_list="${module_linkage_checker_arguments[$1]}" + local artifact_list="${module_linkage_checker_arguments[$1]}" + local -a artifacts=() coords=() + local artifact artifact_id version - for artifact in ${artifact_list//,/ }; do # Split on comma - artifact_id=$(echo "${artifact}" | cut -d ':' -f2) + IFS=',' read -ra artifacts <<< "${artifact_list}" + for artifact in "${artifacts[@]}"; do + # Extract artifactId from "groupId:artifactId[:version]" without spawning subprocesses: + # 1. ${artifact#*:} strips the shortest prefix matching "*:" (removes "groupId:") + # 2. ${artifact_id%%:*} strips the longest suffix matching ":*" (removes trailing ":version", if present) + artifact_id="${artifact#*:}" + artifact_id="${artifact_id%%:*}" - # The grep query tries to match `{artifact_id}:{released_version}:{current_version}`. - # The artifact_id must be exact otherwise multiple entries may match - version=$(cat "${monorepoRoot}/versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3 || true) + version="${versions_map["${artifact_id}"]}" # Unreleased internal test modules like java-showcase are not tracked in versions.txt, # so fallback to 0.0.1-SNAPSHOT for linkage checking. if [ -z "${version}" ]; then version="0.0.1-SNAPSHOT" fi - module_gav_coordinate="${artifact}:${version}" - - # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` - if [ -z "${linkage_checker_arguments}" ]; then - linkage_checker_arguments="${module_gav_coordinate}" - else - linkage_checker_arguments="${linkage_checker_arguments},${module_gav_coordinate}" - fi + coords+=("${artifact}:${version}") done + + local IFS=',' + linkage_checker_arguments="${coords[*]}" } # TODO(https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/2395): Java 17+ support for Linkage Checker # cloud-opensource-java contains the Linkage Checker tool -git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git +git clone --depth 1 https://github.com/GoogleCloudPlatform/cloud-opensource-java.git pushd cloud-opensource-java mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder @@ -82,7 +93,8 @@ pushd dependencies # MODULES_UNDER_TEST Env Var accepts a comma separated list of monorepo submodules to test. For Github CI, # this will be a single module as Github will build a matrix of modules with each being tested in parallel. # For local invocation, you can pass a list of modules to test multiple modules together. -for module in ${MODULES_UNDER_TEST//,/ }; do # Split on comma +IFS=',' read -ra modules_under_test <<< "${MODULES_UNDER_TEST}" +for module in "${modules_under_test[@]}"; do module_dir="${monorepoRoot}/${module}" if [ ! -d "${module_dir}" ]; then echo "Directory ${module_dir} does not exist. Skipping or failed." >&2