Skip syncing refs if files are unchanged

README.md

@@ -34,6 +34,18 @@ We currently sync automatically the following modules:
 | prometheus/client-model | https://github.com/prometheus/client_model |  |
 | protocolbuffers/wellknowntypes | https://github.com/protocolbuffers/protobuf |  |
 
+### How we handle references
+
+Each module is synced either from its source repository's releases or from its commits. Modules
+synced from releases get a reference for every release tag, even when the tagged contents are
+identical to the previous tag.
+
+Modules synced from commits only get a reference when the commit changes the files we sync. We sync
+a curated subset of each source repository (see the `rsync.incl` file in the module's directory
+under `modules/static`), and in large repositories most commits leave that subset untouched. A
+reference for such a commit would point at contents that were already published under an earlier
+reference, so we skip it.
+
 ### How we handle dependencies
 
 Dependencies are an essential part of these community modules as they help developers reuse well

cmd/modprocessor/main.go

@@ -36,6 +36,10 @@ const (
 	ownerFlagName       = "owner"
 	repoFlagName        = "repo"
 	refFlagName         = "ref"
+	dryRunFlagName      = "dry-run"
+
+	changedStatus   = "changed"
+	unchangedStatus = "unchanged"
 )
 
 type command struct {
@@ -44,6 +48,7 @@ type command struct {
 	owner       string
 	repo        string
 	ref         string
+	dryRun      bool
 }
 
 func newCmd(
@@ -52,6 +57,7 @@ func newCmd(
 	owner string,
 	repo string,
 	modRef string,
+	dryRun bool,
 ) (*command, error) {
 	var err error
 	if len(rootSyncDir) == 0 {
@@ -78,6 +84,7 @@ func newCmd(
 		owner:       owner,
 		repo:        repo,
 		ref:         modRef,
+		dryRun:      dryRun,
 	}, nil
 }
 
@@ -88,6 +95,10 @@ func main() {
 		owner       = flag.String(ownerFlagName, "", "Managed module owner name.")
 		repo        = flag.String(repoFlagName, "", "Managed module repository name.")
 		ref         = flag.String(refFlagName, "", "Managed module reference that matches the contents in the source directory.")
+		dryRun      = flag.Bool(dryRunFlagName, false, fmt.Sprintf(
+			"Print %q or %q depending on whether the source files differ from the module's latest reference, without writing any blobs or state.",
+			changedStatus, unchangedStatus,
+		))
 	)
 	flag.Parse()
 	cmd, err := newCmd(
@@ -96,6 +107,7 @@ func main() {
 		*owner,
 		*repo,
 		*ref,
+		*dryRun,
 	)
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "cannot run mod processor: %v\n\nusage: modprocessor [flags]\n\n", err)
@@ -111,51 +123,71 @@ func main() {
 
 func (c *command) run() error {
 	ctx := context.Background()
-	manifestDigest, err := c.convertToCAS(ctx)
+	fileSet, manifestBlob, err := c.newFileSet(ctx)
 	if err != nil {
 		return fmt.Errorf("convert module to CAS: %w", err)
 	}
 	stateRW, err := bufstate.NewReadWriter()
 	if err != nil {
 		return fmt.Errorf("new state read writer: %w", err)
 	}
-	manifestHexDigest := hex.EncodeToString(manifestDigest.Value())
+	manifestHexDigest := hex.EncodeToString(manifestBlob.Digest().Value())
+	if c.dryRun {
+		latestDigest, err := stateRW.LatestModuleDigest(c.rootSyncDir, c.owner, c.repo)
+		if err != nil {
+			return fmt.Errorf("latest module digest: %w", err)
+		}
+		status := changedStatus
+		if latestDigest == manifestHexDigest {
+			status = unchangedStatus
+		}
+		_, _ = fmt.Fprintln(os.Stdout, status)
+		return nil
+	}
+	if err := c.writeCAS(fileSet, manifestBlob); err != nil {
+		return fmt.Errorf("write module CAS: %w", err)
+	}
 	if err := stateRW.AppendModuleReference(c.rootSyncDir, c.owner, c.repo, c.ref, manifestHexDigest); err != nil {
 		return fmt.Errorf("update mod reference: %w", err)
 	}
 	return nil
 }
 
-// convertToCAS converts all files in the source directory to blobs, and a saves
-// them in the module destination directory using its manifest digest hex string
-// as filenames.
-func (c *command) convertToCAS(ctx context.Context) (cas.Digest, error) {
+// newFileSet converts all files in the source directory to blobs, returning them
+// along with the blob of their manifest.
+func (c *command) newFileSet(ctx context.Context) (cas.FileSet, cas.Blob, error) {
 	storageosProvider := storageos.NewProvider()
 	bucket, err := storageosProvider.NewReadWriteBucket(c.srcDir)
 	if err != nil {
-		return nil, fmt.Errorf("new bucket from buf dir: %w", err)
+		return nil, nil, fmt.Errorf("new bucket from buf dir: %w", err)
 	}
 	fileSet, err := cas.NewFileSetForBucket(ctx, bucket, cas.DigestTypeShake256)
 	if err != nil {
-		return nil, fmt.Errorf("new file set from bucket: %w", err)
+		return nil, nil, fmt.Errorf("new file set from bucket: %w", err)
 	}
 	manifestBlob, err := cas.ManifestToBlob(fileSet.Manifest(), cas.DigestTypeShake256)
 	if err != nil {
-		return nil, fmt.Errorf("manifest to blob: %w", err)
+		return nil, nil, fmt.Errorf("manifest to blob: %w", err)
 	}
+	return fileSet, manifestBlob, nil
+}
+
+// writeCAS saves the file set blobs in the module destination directory using
+// their digest hex string as filenames.
+func (c *command) writeCAS(fileSet cas.FileSet, manifestBlob cas.Blob) error {
 	modSyncDir := filepath.Join(c.rootSyncDir, c.owner, c.repo, "cas")
 	// mkdir directory in case this is the first time a reference is being synced for this module.
 	if err := os.MkdirAll(modSyncDir, 0755); err != nil {
-		return nil, fmt.Errorf("make module sync cas dir: %w", err)
+		return fmt.Errorf("make module sync cas dir: %w", err)
 	}
 	// TODO: parallelize
 	for _, blob := range append([]cas.Blob{manifestBlob}, fileSet.BlobSet().Blobs()...) {
 		if err := writeBlobInDir(blob, modSyncDir); err != nil {
 			hexDigest := hex.EncodeToString(blob.Digest().Value())
-			return nil, fmt.Errorf("write blob %q to file: %w", hexDigest, err)
+			return fmt.Errorf("write blob %q to file: %w", hexDigest, err)
 		}
 	}
-	return manifestBlob.Digest(), nil
+	return nil
 }
 
 // writeBlobInDir takes a blob and writes its content to a file named as its

private/bufpkg/bufstate/state.go

@@ -24,6 +24,26 @@ import (
 
 const SyncRoot = "modules/sync"
 
+// LatestModuleDigest returns the digest of the last reference appended to the
+// module state, or an empty string if the module has no state file yet or its
+// state file has no references. It assumes the same sync dir structure as
+// AppendModuleReference.
+func (rw *ReadWriter) LatestModuleDigest(
+	rootSyncDir string,
+	ownerName string,
+	repoName string,
+) (string, error) {
+	modState, err := rw.readModuleState(filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName))
+	if err != nil {
+		return "", err
+	}
+	references := modState.GetReferences()
+	if len(references) == 0 {
+		return "", nil
+	}
+	return references[len(references)-1].GetDigest(), nil
+}
+
 // AppendModuleReference appends a reference-digest pair at the end of the module
 // state, and updates the module's latest reference in the global state. It
 // assumes the structure of the sync dir is
@@ -37,21 +57,9 @@ func (rw *ReadWriter) AppendModuleReference(
 	digest string,
 ) error {
 	modFilePath := filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName)
-	var modState *statev1alpha1.ModuleState
-	if _, err := os.Stat(modFilePath); err != nil {
-		if !os.IsNotExist(err) {
-			return fmt.Errorf("stat file: %w", err)
-		}
-		modState = &statev1alpha1.ModuleState{}
-	} else {
-		modStateFile, err := os.Open(modFilePath)
-		if err != nil {
-			return fmt.Errorf("open file: %w", err)
-		}
-		modState, err = rw.ReadModStateFile(modStateFile)
-		if err != nil {
-			return fmt.Errorf("read module state file: %w", err)
-		}
+	modState, err := rw.readModuleState(modFilePath)
+	if err != nil {
+		return err
 	}
 	modState.SetReferences(append(modState.GetReferences(), statev1alpha1.ModuleReference_builder{Name: reference, Digest: digest}.Build()))
 	// As the state file read/write functions both close after their operations,
@@ -112,3 +120,23 @@ func (rw *ReadWriter) AppendModuleReference(
 	}
 	return nil
 }
+
+// readModuleState reads the module state file at the given path, returning an
+// empty state if the file does not exist yet.
+func (rw *ReadWriter) readModuleState(modFilePath string) (*statev1alpha1.ModuleState, error) {
+	if _, err := os.Stat(modFilePath); err != nil {
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("stat file: %w", err)
+		}
+		return &statev1alpha1.ModuleState{}, nil
+	}
+	modStateFile, err := os.Open(modFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("open file: %w", err)
+	}
+	modState, err := rw.ReadModStateFile(modStateFile)
+	if err != nil {
+		return nil, fmt.Errorf("read module state file: %w", err)
+	}
+	return modState, nil
+}

private/bufpkg/bufstate/state_test.go

@@ -0,0 +1,45 @@
+// Copyright 2021-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufstate
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLatestModuleDigest(t *testing.T) {
+	t.Parallel()
+	readWriter, err := NewReadWriter()
+	require.NoError(t, err)
+	t.Run("noStateFile", func(t *testing.T) {
+		t.Parallel()
+		latestDigest, err := readWriter.LatestModuleDigest(t.TempDir(), "acme", "widgets")
+		require.NoError(t, err)
+		require.Empty(t, latestDigest)
+	})
+	t.Run("appendedReferences", func(t *testing.T) {
+		t.Parallel()
+		rootSyncDir := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(rootSyncDir, "acme", "widgets"), 0755))
+		require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit1", "digest1"))
+		require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit2", "digest2"))
+		latestDigest, err := readWriter.LatestModuleDigest(rootSyncDir, "acme", "widgets")
+		require.NoError(t, err)
+		require.Equal(t, "digest2", latestDigest)
+	})
+}

scripts/fetch.sh

@@ -11,14 +11,19 @@ all_mods_sync_path="${repo_root}/modules/sync"
 all_mods_static_path="${repo_root}/modules/static"
 cd "${repo_root}"
 
+# Build the module processor once up front. It runs at least once per reference walked, and `go run`
+# relinks it on every invocation.
+modprocessor_bin="${repo_root}/.tmp/bin/modprocessor"
+go build -o "${modprocessor_bin}" "${repo_root}/cmd/modprocessor"
+
 log() {
   >&2 echo "$@"
 }
 
-# process_ref should be called within the appropriate proto src directory where files will be copied
-# from. Rsync file should be relative to this dir. This func process a reference, checks out to it,
-# copies relevant files and stores them in CAS format. It updates references in state files.
-process_ref() {
+# prepare_ref should be called within the appropriate proto src directory where files will be copied
+# from. Rsync file should be relative to this dir. This func checks out to a reference and copies the
+# relevant files to a fresh temporary directory, setting its path in ${prepared_ref_path}.
+prepare_ref() {
   local -r mod_ref="${1}"
   local -r mod_tmp_path="$(mktemp -d)"
   dirs_to_delete+=("${mod_tmp_path}")
@@ -60,6 +65,43 @@ process_ref() {
   rsync "${rsync_args[@]}" . "${mod_tmp_path}"
   [ ! -e "${module_static_path}/buf.md" ] || cp "${module_static_path}/buf.md" "${mod_tmp_path}"
   [ ! -e "${module_static_path}/buf.yaml" ] || cp "${module_static_path}/buf.yaml" "${mod_tmp_path}"
+  prepared_ref_path="${mod_tmp_path}"
+}
+
+# ref_digest_status prints "changed" or "unchanged" depending on whether the contents prepared in the
+# given directory differ from the digest of the module's latest reference in its state file.
+ref_digest_status() {
+  local -r mod_tmp_path="${1}"
+  local -r mod_ref="${2}"
+  "${modprocessor_bin}" \
+    --dry-run \
+    --root-sync-dir="${all_mods_sync_path}" \
+    --src-dir="${mod_tmp_path}" \
+    --owner="${owner}" \
+    --repo="${repo}" \
+    --ref="${mod_ref}"
+}
+
+# process_ref should be called within the appropriate proto src directory where files will be copied
+# from. This func processes a reference, checks it out, copies relevant files in CAS format, and
+# updates references in state files.
+process_ref() {
+  local -r mod_ref="${1}"
+  prepare_ref "${mod_ref}"
+  local -r mod_tmp_path="${prepared_ref_path}"
+
+  # Modules synced by commit mirror every commit of their source repository, but we only sync a
+  # curated subset of its files. Appending a reference whose contents are identical to the previous
+  # one publishes a BSR label pointing at identical content, so drop it.
+  if [ "${sync_strategy}" == "commits" ]; then
+    local digest_status
+    digest_status="$(ref_digest_status "${mod_tmp_path}" "${mod_ref}")"
+    if [ "${digest_status}" == "unchanged" ]; then
+      echo "skipping reference ${owner}/${repo}:${mod_ref}, contents unchanged"
+      rm -rf "${mod_tmp_path}"
+      return
+    fi
+  fi
 
   # If the source of the module that we are syncing is from a v2 workspace with another module
   # we are syncing, e.g. protovalidate and protovalidate-testing, then it is possible that
@@ -88,14 +130,12 @@ process_ref() {
 
   # process the prepared module: convert it to CAS from the tmp mod directory and put blob files in
   # the cas path in the repo, and update the state file.
-  pushd "${repo_root}" > /dev/null
-  go run "${repo_root}/cmd/modprocessor" \
+  "${modprocessor_bin}" \
     --root-sync-dir="${all_mods_sync_path}" \
     --src-dir="${mod_tmp_path}" \
     --owner="${owner}" \
     --repo="${repo}" \
     --ref="${mod_ref}"
-  popd > /dev/null
 }
 
 # sync_references ${sync_strategy} ${owner} ${repo} ${git_remote} ${opt_proto_subdir}
@@ -127,6 +167,16 @@ sync_references() {
   local -r module_root=$(pwd)
   pushd "${git_owner}/${git_repo}/${proto_subdir}" > /dev/null
 
+  # Resolve the tip of the cloned branch from the remote ref instead of HEAD: processing a reference
+  # leaves the work tree on a detached HEAD, and a single clone can back more than one managed module
+  # (googleapis/googleapis and googleapis/cloud-run).
+  local git_origin_head_ref="refs/remotes/origin/HEAD"
+  if ! git rev-parse --verify --quiet "${git_origin_head_ref}" > /dev/null; then
+    git_origin_head_ref="$(git for-each-ref --count=1 --format='%(refname)' refs/remotes/origin)"
+  fi
+  local git_origin_head
+  git_origin_head="$(git rev-parse "${git_origin_head_ref}")"
+
   local rev_list
   if [ "${sync_strategy}" == "releases" ]; then
     rev_list=$(get_release_revlist)
@@ -158,15 +208,15 @@ get_commit_revlist() {
   if [ -f "${mod_state_file}" ]; then
     mod_latest_ref="$(cat "${mod_state_file}" | jq -r '.references | last.name')"
     log "latest reference for module ${owner}/${repo}: ${mod_latest_ref}"
-    # revisions from initial latest_ref...HEAD (excluding latest_ref)
-    commit_rev_list=$(git rev-list "${mod_latest_ref}"...HEAD --first-parent --reverse)
+    # revisions from initial latest_ref...git_origin_head (excluding latest_ref)
+    commit_rev_list=$(git rev-list "${mod_latest_ref}..${git_origin_head}" --first-parent --reverse)
   elif [ -f "${mod_initref_file}" ]; then
     log "state file not found: ${mod_state_file}"
     mod_init_ref="$(cat "${mod_initref_file}")"
     log "falling back to initializing reference for module ${owner}/${repo}: ${mod_init_ref}"
     # Prints revisions on the main branch, stopping when ${mod_init_ref} is
     # encountered, and using tac to reverse the revisions (includes init_ref).
-    commit_rev_list=$(git rev-list HEAD --first-parent | sed "/${mod_init_ref}/q" | tac)
+    commit_rev_list=$(git rev-list "${git_origin_head}" --first-parent | sed "/${mod_init_ref}/q" | tac)
   else
     log "module ${owner}/${repo} has no initializing reference"
     exit 2