From 3be491aff9fd9b3b5fb84c74003e68cdb9300547 Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Mon, 10 Aug 2026 16:54:00 -0500 Subject: [PATCH] Skip syncing refs if files are unchanged Historically for modules set to sync by commits (instead of tags), we've synced every upstream commit even if the calculated digest is identical to the last ref. A calculated digest is unchanged if the .proto, buf.md, LICENSE or other files haven't changed. This is frequently the case with large repositories where we only sync a small subset of files (googleapis/cloud-run and googleapis/googleapis). Looking at state.json for googleapis/googleapis, there are 5974 refs but only 41 unique digests across them. Syncing these commits is of low value (the label is immediately archived in the BSR) and leads to more maintenance on this repo than necessary. Update the fetch script and companion utilities to only write refs to state.json when the digest changes. --- README.md | 12 +++++ cmd/modprocessor/main.go | 56 ++++++++++++++++----- private/bufpkg/bufstate/state.go | 58 ++++++++++++++++------ private/bufpkg/bufstate/state_test.go | 45 +++++++++++++++++ scripts/fetch.sh | 70 +++++++++++++++++++++++---- 5 files changed, 204 insertions(+), 37 deletions(-) create mode 100644 private/bufpkg/bufstate/state_test.go diff --git a/README.md b/README.md index 3bdc4989..71f899f7 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,18 @@ We currently sync automatically the following modules: | prometheus/client-model | https://github.com/prometheus/client_model | | | protocolbuffers/wellknowntypes | https://github.com/protocolbuffers/protobuf | | +### How we handle references + +Each module is synced either from its source repository's releases or from its commits. Modules +synced from releases get a reference for every release tag, even when the tagged contents are +identical to the previous tag. + +Modules synced from commits only get a reference when the commit changes the files we sync. We sync +a curated subset of each source repository (see the `rsync.incl` file in the module's directory +under `modules/static`), and in large repositories most commits leave that subset untouched. A +reference for such a commit would point at contents that were already published under an earlier +reference, so we skip it. + ### How we handle dependencies Dependencies are an essential part of these community modules as they help developers reuse well diff --git a/cmd/modprocessor/main.go b/cmd/modprocessor/main.go index 88c4ae47..a886567e 100644 --- a/cmd/modprocessor/main.go +++ b/cmd/modprocessor/main.go @@ -36,6 +36,10 @@ const ( ownerFlagName = "owner" repoFlagName = "repo" refFlagName = "ref" + dryRunFlagName = "dry-run" + + changedStatus = "changed" + unchangedStatus = "unchanged" ) type command struct { @@ -44,6 +48,7 @@ type command struct { owner string repo string ref string + dryRun bool } func newCmd( @@ -52,6 +57,7 @@ func newCmd( owner string, repo string, modRef string, + dryRun bool, ) (*command, error) { var err error if len(rootSyncDir) == 0 { @@ -78,6 +84,7 @@ func newCmd( owner: owner, repo: repo, ref: modRef, + dryRun: dryRun, }, nil } @@ -88,6 +95,10 @@ func main() { owner = flag.String(ownerFlagName, "", "Managed module owner name.") repo = flag.String(repoFlagName, "", "Managed module repository name.") ref = flag.String(refFlagName, "", "Managed module reference that matches the contents in the source directory.") + dryRun = flag.Bool(dryRunFlagName, false, fmt.Sprintf( + "Print %q or %q depending on whether the source files differ from the module's latest reference, without writing any blobs or state.", + changedStatus, unchangedStatus, + )) ) flag.Parse() cmd, err := newCmd( @@ -96,6 +107,7 @@ func main() { *owner, *repo, *ref, + *dryRun, ) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "cannot run mod processor: %v\n\nusage: modprocessor [flags]\n\n", err) @@ -111,7 +123,7 @@ func main() { func (c *command) run() error { ctx := context.Background() - manifestDigest, err := c.convertToCAS(ctx) + fileSet, manifestBlob, err := c.newFileSet(ctx) if err != nil { return fmt.Errorf("convert module to CAS: %w", err) } @@ -119,43 +131,63 @@ func (c *command) run() error { if err != nil { return fmt.Errorf("new state read writer: %w", err) } - manifestHexDigest := hex.EncodeToString(manifestDigest.Value()) + manifestHexDigest := hex.EncodeToString(manifestBlob.Digest().Value()) + if c.dryRun { + latestDigest, err := stateRW.LatestModuleDigest(c.rootSyncDir, c.owner, c.repo) + if err != nil { + return fmt.Errorf("latest module digest: %w", err) + } + status := changedStatus + if latestDigest == manifestHexDigest { + status = unchangedStatus + } + _, _ = fmt.Fprintln(os.Stdout, status) + return nil + } + if err := c.writeCAS(fileSet, manifestBlob); err != nil { + return fmt.Errorf("write module CAS: %w", err) + } if err := stateRW.AppendModuleReference(c.rootSyncDir, c.owner, c.repo, c.ref, manifestHexDigest); err != nil { return fmt.Errorf("update mod reference: %w", err) } return nil } -// convertToCAS converts all files in the source directory to blobs, and a saves -// them in the module destination directory using its manifest digest hex string -// as filenames. -func (c *command) convertToCAS(ctx context.Context) (cas.Digest, error) { +// newFileSet converts all files in the source directory to blobs, returning them +// along with the blob of their manifest. +func (c *command) newFileSet(ctx context.Context) (cas.FileSet, cas.Blob, error) { storageosProvider := storageos.NewProvider() bucket, err := storageosProvider.NewReadWriteBucket(c.srcDir) if err != nil { - return nil, fmt.Errorf("new bucket from buf dir: %w", err) + return nil, nil, fmt.Errorf("new bucket from buf dir: %w", err) } fileSet, err := cas.NewFileSetForBucket(ctx, bucket, cas.DigestTypeShake256) if err != nil { - return nil, fmt.Errorf("new file set from bucket: %w", err) + return nil, nil, fmt.Errorf("new file set from bucket: %w", err) } manifestBlob, err := cas.ManifestToBlob(fileSet.Manifest(), cas.DigestTypeShake256) if err != nil { - return nil, fmt.Errorf("manifest to blob: %w", err) + return nil, nil, fmt.Errorf("manifest to blob: %w", err) } + return fileSet, manifestBlob, nil +} + +// writeCAS saves the file set blobs in the module destination directory using +// their digest hex string as filenames. +func (c *command) writeCAS(fileSet cas.FileSet, manifestBlob cas.Blob) error { modSyncDir := filepath.Join(c.rootSyncDir, c.owner, c.repo, "cas") // mkdir directory in case this is the first time a reference is being synced for this module. if err := os.MkdirAll(modSyncDir, 0755); err != nil { - return nil, fmt.Errorf("make module sync cas dir: %w", err) + return fmt.Errorf("make module sync cas dir: %w", err) } // TODO: parallelize for _, blob := range append([]cas.Blob{manifestBlob}, fileSet.BlobSet().Blobs()...) { if err := writeBlobInDir(blob, modSyncDir); err != nil { hexDigest := hex.EncodeToString(blob.Digest().Value()) - return nil, fmt.Errorf("write blob %q to file: %w", hexDigest, err) + return fmt.Errorf("write blob %q to file: %w", hexDigest, err) } } - return manifestBlob.Digest(), nil + return nil } // writeBlobInDir takes a blob and writes its content to a file named as its diff --git a/private/bufpkg/bufstate/state.go b/private/bufpkg/bufstate/state.go index 9302410e..4ddc4334 100644 --- a/private/bufpkg/bufstate/state.go +++ b/private/bufpkg/bufstate/state.go @@ -24,6 +24,26 @@ import ( const SyncRoot = "modules/sync" +// LatestModuleDigest returns the digest of the last reference appended to the +// module state, or an empty string if the module has no state file yet or its +// state file has no references. It assumes the same sync dir structure as +// AppendModuleReference. +func (rw *ReadWriter) LatestModuleDigest( + rootSyncDir string, + ownerName string, + repoName string, +) (string, error) { + modState, err := rw.readModuleState(filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName)) + if err != nil { + return "", err + } + references := modState.GetReferences() + if len(references) == 0 { + return "", nil + } + return references[len(references)-1].GetDigest(), nil +} + // AppendModuleReference appends a reference-digest pair at the end of the module // state, and updates the module's latest reference in the global state. It // assumes the structure of the sync dir is @@ -37,21 +57,9 @@ func (rw *ReadWriter) AppendModuleReference( digest string, ) error { modFilePath := filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName) - var modState *statev1alpha1.ModuleState - if _, err := os.Stat(modFilePath); err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("stat file: %w", err) - } - modState = &statev1alpha1.ModuleState{} - } else { - modStateFile, err := os.Open(modFilePath) - if err != nil { - return fmt.Errorf("open file: %w", err) - } - modState, err = rw.ReadModStateFile(modStateFile) - if err != nil { - return fmt.Errorf("read module state file: %w", err) - } + modState, err := rw.readModuleState(modFilePath) + if err != nil { + return err } modState.SetReferences(append(modState.GetReferences(), statev1alpha1.ModuleReference_builder{Name: reference, Digest: digest}.Build())) // As the state file read/write functions both close after their operations, @@ -112,3 +120,23 @@ func (rw *ReadWriter) AppendModuleReference( } return nil } + +// readModuleState reads the module state file at the given path, returning an +// empty state if the file does not exist yet. +func (rw *ReadWriter) readModuleState(modFilePath string) (*statev1alpha1.ModuleState, error) { + if _, err := os.Stat(modFilePath); err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat file: %w", err) + } + return &statev1alpha1.ModuleState{}, nil + } + modStateFile, err := os.Open(modFilePath) + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + modState, err := rw.ReadModStateFile(modStateFile) + if err != nil { + return nil, fmt.Errorf("read module state file: %w", err) + } + return modState, nil +} diff --git a/private/bufpkg/bufstate/state_test.go b/private/bufpkg/bufstate/state_test.go new file mode 100644 index 00000000..4525d2dc --- /dev/null +++ b/private/bufpkg/bufstate/state_test.go @@ -0,0 +1,45 @@ +// Copyright 2021-2025 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufstate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLatestModuleDigest(t *testing.T) { + t.Parallel() + readWriter, err := NewReadWriter() + require.NoError(t, err) + t.Run("noStateFile", func(t *testing.T) { + t.Parallel() + latestDigest, err := readWriter.LatestModuleDigest(t.TempDir(), "acme", "widgets") + require.NoError(t, err) + require.Empty(t, latestDigest) + }) + t.Run("appendedReferences", func(t *testing.T) { + t.Parallel() + rootSyncDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(rootSyncDir, "acme", "widgets"), 0755)) + require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit1", "digest1")) + require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit2", "digest2")) + latestDigest, err := readWriter.LatestModuleDigest(rootSyncDir, "acme", "widgets") + require.NoError(t, err) + require.Equal(t, "digest2", latestDigest) + }) +} diff --git a/scripts/fetch.sh b/scripts/fetch.sh index a75ef27e..e54dbc4f 100755 --- a/scripts/fetch.sh +++ b/scripts/fetch.sh @@ -11,14 +11,19 @@ all_mods_sync_path="${repo_root}/modules/sync" all_mods_static_path="${repo_root}/modules/static" cd "${repo_root}" +# Build the module processor once up front. It runs at least once per reference walked, and `go run` +# relinks it on every invocation. +modprocessor_bin="${repo_root}/.tmp/bin/modprocessor" +go build -o "${modprocessor_bin}" "${repo_root}/cmd/modprocessor" + log() { >&2 echo "$@" } -# process_ref should be called within the appropriate proto src directory where files will be copied -# from. Rsync file should be relative to this dir. This func process a reference, checks out to it, -# copies relevant files and stores them in CAS format. It updates references in state files. -process_ref() { +# prepare_ref should be called within the appropriate proto src directory where files will be copied +# from. Rsync file should be relative to this dir. This func checks out to a reference and copies the +# relevant files to a fresh temporary directory, setting its path in ${prepared_ref_path}. +prepare_ref() { local -r mod_ref="${1}" local -r mod_tmp_path="$(mktemp -d)" dirs_to_delete+=("${mod_tmp_path}") @@ -60,6 +65,43 @@ process_ref() { rsync "${rsync_args[@]}" . "${mod_tmp_path}" [ ! -e "${module_static_path}/buf.md" ] || cp "${module_static_path}/buf.md" "${mod_tmp_path}" [ ! -e "${module_static_path}/buf.yaml" ] || cp "${module_static_path}/buf.yaml" "${mod_tmp_path}" + prepared_ref_path="${mod_tmp_path}" +} + +# ref_digest_status prints "changed" or "unchanged" depending on whether the contents prepared in the +# given directory differ from the digest of the module's latest reference in its state file. +ref_digest_status() { + local -r mod_tmp_path="${1}" + local -r mod_ref="${2}" + "${modprocessor_bin}" \ + --dry-run \ + --root-sync-dir="${all_mods_sync_path}" \ + --src-dir="${mod_tmp_path}" \ + --owner="${owner}" \ + --repo="${repo}" \ + --ref="${mod_ref}" +} + +# process_ref should be called within the appropriate proto src directory where files will be copied +# from. This func processes a reference, checks it out, copies relevant files in CAS format, and +# updates references in state files. +process_ref() { + local -r mod_ref="${1}" + prepare_ref "${mod_ref}" + local -r mod_tmp_path="${prepared_ref_path}" + + # Modules synced by commit mirror every commit of their source repository, but we only sync a + # curated subset of its files. Appending a reference whose contents are identical to the previous + # one publishes a BSR label pointing at identical content, so drop it. + if [ "${sync_strategy}" == "commits" ]; then + local digest_status + digest_status="$(ref_digest_status "${mod_tmp_path}" "${mod_ref}")" + if [ "${digest_status}" == "unchanged" ]; then + echo "skipping reference ${owner}/${repo}:${mod_ref}, contents unchanged" + rm -rf "${mod_tmp_path}" + return + fi + fi # If the source of the module that we are syncing is from a v2 workspace with another module # we are syncing, e.g. protovalidate and protovalidate-testing, then it is possible that @@ -88,14 +130,12 @@ process_ref() { # process the prepared module: convert it to CAS from the tmp mod directory and put blob files in # the cas path in the repo, and update the state file. - pushd "${repo_root}" > /dev/null - go run "${repo_root}/cmd/modprocessor" \ + "${modprocessor_bin}" \ --root-sync-dir="${all_mods_sync_path}" \ --src-dir="${mod_tmp_path}" \ --owner="${owner}" \ --repo="${repo}" \ --ref="${mod_ref}" - popd > /dev/null } # sync_references ${sync_strategy} ${owner} ${repo} ${git_remote} ${opt_proto_subdir} @@ -127,6 +167,16 @@ sync_references() { local -r module_root=$(pwd) pushd "${git_owner}/${git_repo}/${proto_subdir}" > /dev/null + # Resolve the tip of the cloned branch from the remote ref instead of HEAD: processing a reference + # leaves the work tree on a detached HEAD, and a single clone can back more than one managed module + # (googleapis/googleapis and googleapis/cloud-run). + local git_origin_head_ref="refs/remotes/origin/HEAD" + if ! git rev-parse --verify --quiet "${git_origin_head_ref}" > /dev/null; then + git_origin_head_ref="$(git for-each-ref --count=1 --format='%(refname)' refs/remotes/origin)" + fi + local git_origin_head + git_origin_head="$(git rev-parse "${git_origin_head_ref}")" + local rev_list if [ "${sync_strategy}" == "releases" ]; then rev_list=$(get_release_revlist) @@ -158,15 +208,15 @@ get_commit_revlist() { if [ -f "${mod_state_file}" ]; then mod_latest_ref="$(cat "${mod_state_file}" | jq -r '.references | last.name')" log "latest reference for module ${owner}/${repo}: ${mod_latest_ref}" - # revisions from initial latest_ref...HEAD (excluding latest_ref) - commit_rev_list=$(git rev-list "${mod_latest_ref}"...HEAD --first-parent --reverse) + # revisions from initial latest_ref...git_origin_head (excluding latest_ref) + commit_rev_list=$(git rev-list "${mod_latest_ref}..${git_origin_head}" --first-parent --reverse) elif [ -f "${mod_initref_file}" ]; then log "state file not found: ${mod_state_file}" mod_init_ref="$(cat "${mod_initref_file}")" log "falling back to initializing reference for module ${owner}/${repo}: ${mod_init_ref}" # Prints revisions on the main branch, stopping when ${mod_init_ref} is # encountered, and using tac to reverse the revisions (includes init_ref). - commit_rev_list=$(git rev-list HEAD --first-parent | sed "/${mod_init_ref}/q" | tac) + commit_rev_list=$(git rev-list "${git_origin_head}" --first-parent | sed "/${mod_init_ref}/q" | tac) else log "module ${owner}/${repo} has no initializing reference" exit 2