Skip to content

ci: optimize release version checks and version caching #14229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lqiu96 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ci: optimize release version checks and version caching #14229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions generation/check_existing_release_versions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,24 @@ MAVEN_SITE=https://maven-central.storage-download.googleapis.com/maven2

set -e

# Helper function to extract the content of the first matching XML tag from a file,
# ignoring <parent> blocks so parent coordinates are not accidentally matched.
function extract_xml_tag() {
local tag=$1
local file=$2
[[ ! -f "${file}" ]] && return 0
sed -n "/<parent>/,/<\/parent>/d; \|<${tag}[ >]|{s|.*<${tag}[^>]*>[[:space:]]*\([^<[:space:]]*\).*|\1|p; q;}" "${file}"
}

function find_existing_version_pom() {
local pom_file=$1
if [ -z "${pom_file}" ]; then
echo "Empty pom file name"
exit 1
fi
local group_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="groupId"]/text()' \
"${pom_file}")
local artifact_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="artifactId"]/text()' \
"${pom_file}")
local version=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="version"]/text()' \
"${pom_file}")
local group_id=$(extract_xml_tag groupId "${pom_file}")
local artifact_id=$(extract_xml_tag artifactId "${pom_file}")
local version=$(extract_xml_tag version "${pom_file}")
echo -n "Checking ${group_id}:${artifact_id}:${version}:"
if [ -z "${artifact_id}" ]; then
echo "Couldn't parse artifact_id in the pom file: $pom_file"
Expand All @@ -32,25 +38,32 @@ function find_existing_version_pom() {
fi
local group_id_dir="${group_id//\.//}"
local URL="${MAVEN_SITE}/${group_id_dir}/${artifact_id}/${version}/${artifact_id}-${version}.pom"
local status_code=$(curl --silent --head -o /dev/null -w "%{http_code}" $URL)
local status_code=$(curl --silent --head -o /dev/null -w "%{http_code}" "${URL}")
if [ "${status_code}" == "200" ]; then
echo " The version already exists at ${URL}. Please investigate."
return_code=1
else
echo " The version does not exists (status_code ${status_code} for ${URL}). Good."
echo " The version does not exist (status_code ${status_code} for ${URL}). Good."
fi
}

return_code=0

for pom_file in $(find . -maxdepth 3 -name pom.xml|sort --dictionary-order); do
while IFS= read -r pom_file; do
[[ -z "${pom_file}" ]] && continue
# Exclude java-vertexai because it has been archived and replaced with a dummy POM.
# We do not plan to release any new versions for it, so we don't want to check if its
# version (which already exists) is a duplicate.
if [[ "${pom_file}" == *samples* || "${pom_file}" == *showcase* || "${pom_file}" == *coverage-report* || "${pom_file}" == *sdk-platform-java/pom.xml || "${pom_file}" == *java-vertexai* || "${pom_file}" == *storage-shared-benchmarking* || "${pom_file}" == *java-bigtable/test-proxy* || ( "${pom_file}" == */java-shared-config/pom.xml && "${pom_file}" != */java-shared-config/*/pom.xml ) ]]; then
case "${pom_file}" in
*samples* | *showcase* | *coverage-report* | *sdk-platform-java/pom.xml | \
*java-vertexai* | *storage-shared-benchmarking* | *java-bigtable/test-proxy*)
continue
fi
;;
*/java-shared-config/pom.xml)
[[ "${pom_file}" != */java-shared-config/*/pom.xml ]] && continue
;;
esac
find_existing_version_pom "${pom_file}"
done
done < <(find . -maxdepth 3 -name pom.xml | sort --dictionary-order)

exit ${return_code}
44 changes: 28 additions & 16 deletions sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh
View file
Open in desktop

@blakeli0 blakeli0 Sep 3, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the changes in this file. Can we repurpose this PR to only include versions_map changes and remove the xml parsing part?

Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@ validate_protobuf_compatibility_script_inputs

monorepoRoot=$(realpath "${scriptDir}/../../..")

# Pre-cache versions.txt into an associative array to avoid cat/grep/cut on each artifact
declare -A versions_map
if [[ -f "${monorepoRoot}/versions.txt" ]]; then
while IFS=':' read -r mod rel cur || [[ -n "${mod}" ]]; do
[[ -z "${mod}" || "${mod}" =~ ^[[:space:]]*# ]] && continue
cur="${cur%$'\r'}"
versions_map["${mod}"]="${cur}"
done < "${monorepoRoot}/versions.txt"
fi

# Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a
# K/V pairing of (Key: module name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The
# value list doesn't hold the version and this needs to be parsed from the monorepo's versions.txt file
Expand All @@ -47,33 +57,34 @@ module_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cl
# It will try to match the artifact_id in the versions.txt file and attach it to form the GAV
# The GAV list is required by Linkage Checker as program arguments
function build_program_arguments() {
artifact_list="${module_linkage_checker_arguments[$1]}"
local artifact_list="${module_linkage_checker_arguments[$1]}"
local -a artifacts=() coords=()
local artifact artifact_id version

for artifact in ${artifact_list//,/ }; do # Split on comma
artifact_id=$(echo "${artifact}" | cut -d ':' -f2)
IFS=',' read -ra artifacts <<< "${artifact_list}"
for artifact in "${artifacts[@]}"; do
# Extract artifactId from "groupId:artifactId[:version]" without spawning subprocesses:
# 1. ${artifact#*:} strips the shortest prefix matching "*:" (removes "groupId:")
# 2. ${artifact_id%%:*} strips the longest suffix matching ":*" (removes trailing ":version", if present)
artifact_id="${artifact#*:}"
artifact_id="${artifact_id%%:*}"

# The grep query tries to match `{artifact_id}:{released_version}:{current_version}`.
# The artifact_id must be exact otherwise multiple entries may match
version=$(cat "${monorepoRoot}/versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3 || true)
version="${versions_map["${artifact_id}"]}"
# Unreleased internal test modules like java-showcase are not tracked in versions.txt,
# so fallback to 0.0.1-SNAPSHOT for linkage checking.
if [ -z "${version}" ]; then
version="0.0.1-SNAPSHOT"
fi
module_gav_coordinate="${artifact}:${version}"

# The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}`
if [ -z "${linkage_checker_arguments}" ]; then
linkage_checker_arguments="${module_gav_coordinate}"
else
linkage_checker_arguments="${linkage_checker_arguments},${module_gav_coordinate}"
fi
coords+=("${artifact}:${version}")
done

local IFS=','
linkage_checker_arguments="${coords[*]}"
}

# TODO(https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/2395): Java 17+ support for Linkage Checker
# cloud-opensource-java contains the Linkage Checker tool
git clone https://github.com/GoogleCloudPlatform/cloud-opensource-java.git
git clone --depth 1 https://github.com/GoogleCloudPlatform/cloud-opensource-java.git
pushd cloud-opensource-java
mvn -B -ntp clean compile -T 1C
# Linkage Checker tool resides in the /dependencies subfolder
Expand All @@ -82,7 +93,8 @@ pushd dependencies
# MODULES_UNDER_TEST Env Var accepts a comma separated list of monorepo submodules to test. For Github CI,
# this will be a single module as Github will build a matrix of modules with each being tested in parallel.
# For local invocation, you can pass a list of modules to test multiple modules together.
for module in ${MODULES_UNDER_TEST//,/ }; do # Split on comma
IFS=',' read -ra modules_under_test <<< "${MODULES_UNDER_TEST}"
for module in "${modules_under_test[@]}"; do
module_dir="${monorepoRoot}/${module}"
if [ ! -d "${module_dir}" ]; then
echo "Directory ${module_dir} does not exist. Skipping or failed." >&2
Expand Down
Loading