diff --git a/README.md b/README.md index 3bdc4989..71f899f7 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,18 @@ We currently sync automatically the following modules: | prometheus/client-model | https://github.com/prometheus/client_model | | | protocolbuffers/wellknowntypes | https://github.com/protocolbuffers/protobuf | | +### How we handle references + +Each module is synced either from its source repository's releases or from its commits. Modules +synced from releases get a reference for every release tag, even when the tagged contents are +identical to the previous tag. + +Modules synced from commits only get a reference when the commit changes the files we sync. We sync +a curated subset of each source repository (see the `rsync.incl` file in the module's directory +under `modules/static`), and in large repositories most commits leave that subset untouched. A +reference for such a commit would point at contents that were already published under an earlier +reference, so we skip it. + ### How we handle dependencies Dependencies are an essential part of these community modules as they help developers reuse well diff --git a/cmd/modprocessor/main.go b/cmd/modprocessor/main.go index 88c4ae47..a886567e 100644 --- a/cmd/modprocessor/main.go +++ b/cmd/modprocessor/main.go @@ -36,6 +36,10 @@ const ( ownerFlagName = "owner" repoFlagName = "repo" refFlagName = "ref" + dryRunFlagName = "dry-run" + + changedStatus = "changed" + unchangedStatus = "unchanged" ) type command struct { @@ -44,6 +48,7 @@ type command struct { owner string repo string ref string + dryRun bool } func newCmd( @@ -52,6 +57,7 @@ func newCmd( owner string, repo string, modRef string, + dryRun bool, ) (*command, error) { var err error if len(rootSyncDir) == 0 { @@ -78,6 +84,7 @@ func newCmd( owner: owner, repo: repo, ref: modRef, + dryRun: dryRun, }, nil } @@ -88,6 +95,10 @@ func main() { owner = flag.String(ownerFlagName, "", "Managed module owner name.") repo = flag.String(repoFlagName, "", "Managed module repository name.") ref = flag.String(refFlagName, "", "Managed module reference that matches the contents in the source directory.") + dryRun = flag.Bool(dryRunFlagName, false, fmt.Sprintf( + "Print %q or %q depending on whether the source files differ from the module's latest reference, without writing any blobs or state.", + changedStatus, unchangedStatus, + )) ) flag.Parse() cmd, err := newCmd( @@ -96,6 +107,7 @@ func main() { *owner, *repo, *ref, + *dryRun, ) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "cannot run mod processor: %v\n\nusage: modprocessor [flags]\n\n", err) @@ -111,7 +123,7 @@ func main() { func (c *command) run() error { ctx := context.Background() - manifestDigest, err := c.convertToCAS(ctx) + fileSet, manifestBlob, err := c.newFileSet(ctx) if err != nil { return fmt.Errorf("convert module to CAS: %w", err) } @@ -119,43 +131,63 @@ func (c *command) run() error { if err != nil { return fmt.Errorf("new state read writer: %w", err) } - manifestHexDigest := hex.EncodeToString(manifestDigest.Value()) + manifestHexDigest := hex.EncodeToString(manifestBlob.Digest().Value()) + if c.dryRun { + latestDigest, err := stateRW.LatestModuleDigest(c.rootSyncDir, c.owner, c.repo) + if err != nil { + return fmt.Errorf("latest module digest: %w", err) + } + status := changedStatus + if latestDigest == manifestHexDigest { + status = unchangedStatus + } + _, _ = fmt.Fprintln(os.Stdout, status) + return nil + } + if err := c.writeCAS(fileSet, manifestBlob); err != nil { + return fmt.Errorf("write module CAS: %w", err) + } if err := stateRW.AppendModuleReference(c.rootSyncDir, c.owner, c.repo, c.ref, manifestHexDigest); err != nil { return fmt.Errorf("update mod reference: %w", err) } return nil } -// convertToCAS converts all files in the source directory to blobs, and a saves -// them in the module destination directory using its manifest digest hex string -// as filenames. -func (c *command) convertToCAS(ctx context.Context) (cas.Digest, error) { +// newFileSet converts all files in the source directory to blobs, returning them +// along with the blob of their manifest. +func (c *command) newFileSet(ctx context.Context) (cas.FileSet, cas.Blob, error) { storageosProvider := storageos.NewProvider() bucket, err := storageosProvider.NewReadWriteBucket(c.srcDir) if err != nil { - return nil, fmt.Errorf("new bucket from buf dir: %w", err) + return nil, nil, fmt.Errorf("new bucket from buf dir: %w", err) } fileSet, err := cas.NewFileSetForBucket(ctx, bucket, cas.DigestTypeShake256) if err != nil { - return nil, fmt.Errorf("new file set from bucket: %w", err) + return nil, nil, fmt.Errorf("new file set from bucket: %w", err) } manifestBlob, err := cas.ManifestToBlob(fileSet.Manifest(), cas.DigestTypeShake256) if err != nil { - return nil, fmt.Errorf("manifest to blob: %w", err) + return nil, nil, fmt.Errorf("manifest to blob: %w", err) } + return fileSet, manifestBlob, nil +} + +// writeCAS saves the file set blobs in the module destination directory using +// their digest hex string as filenames. +func (c *command) writeCAS(fileSet cas.FileSet, manifestBlob cas.Blob) error { modSyncDir := filepath.Join(c.rootSyncDir, c.owner, c.repo, "cas") // mkdir directory in case this is the first time a reference is being synced for this module. if err := os.MkdirAll(modSyncDir, 0755); err != nil { - return nil, fmt.Errorf("make module sync cas dir: %w", err) + return fmt.Errorf("make module sync cas dir: %w", err) } // TODO: parallelize for _, blob := range append([]cas.Blob{manifestBlob}, fileSet.BlobSet().Blobs()...) { if err := writeBlobInDir(blob, modSyncDir); err != nil { hexDigest := hex.EncodeToString(blob.Digest().Value()) - return nil, fmt.Errorf("write blob %q to file: %w", hexDigest, err) + return fmt.Errorf("write blob %q to file: %w", hexDigest, err) } } - return manifestBlob.Digest(), nil + return nil } // writeBlobInDir takes a blob and writes its content to a file named as its diff --git a/private/bufpkg/bufstate/state.go b/private/bufpkg/bufstate/state.go index 9302410e..4ddc4334 100644 --- a/private/bufpkg/bufstate/state.go +++ b/private/bufpkg/bufstate/state.go @@ -24,6 +24,26 @@ import ( const SyncRoot = "modules/sync" +// LatestModuleDigest returns the digest of the last reference appended to the +// module state, or an empty string if the module has no state file yet or its +// state file has no references. It assumes the same sync dir structure as +// AppendModuleReference. +func (rw *ReadWriter) LatestModuleDigest( + rootSyncDir string, + ownerName string, + repoName string, +) (string, error) { + modState, err := rw.readModuleState(filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName)) + if err != nil { + return "", err + } + references := modState.GetReferences() + if len(references) == 0 { + return "", nil + } + return references[len(references)-1].GetDigest(), nil +} + // AppendModuleReference appends a reference-digest pair at the end of the module // state, and updates the module's latest reference in the global state. It // assumes the structure of the sync dir is @@ -37,21 +57,9 @@ func (rw *ReadWriter) AppendModuleReference( digest string, ) error { modFilePath := filepath.Join(rootSyncDir, ownerName, repoName, ModStateFileName) - var modState *statev1alpha1.ModuleState - if _, err := os.Stat(modFilePath); err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("stat file: %w", err) - } - modState = &statev1alpha1.ModuleState{} - } else { - modStateFile, err := os.Open(modFilePath) - if err != nil { - return fmt.Errorf("open file: %w", err) - } - modState, err = rw.ReadModStateFile(modStateFile) - if err != nil { - return fmt.Errorf("read module state file: %w", err) - } + modState, err := rw.readModuleState(modFilePath) + if err != nil { + return err } modState.SetReferences(append(modState.GetReferences(), statev1alpha1.ModuleReference_builder{Name: reference, Digest: digest}.Build())) // As the state file read/write functions both close after their operations, @@ -112,3 +120,23 @@ func (rw *ReadWriter) AppendModuleReference( } return nil } + +// readModuleState reads the module state file at the given path, returning an +// empty state if the file does not exist yet. +func (rw *ReadWriter) readModuleState(modFilePath string) (*statev1alpha1.ModuleState, error) { + if _, err := os.Stat(modFilePath); err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat file: %w", err) + } + return &statev1alpha1.ModuleState{}, nil + } + modStateFile, err := os.Open(modFilePath) + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + modState, err := rw.ReadModStateFile(modStateFile) + if err != nil { + return nil, fmt.Errorf("read module state file: %w", err) + } + return modState, nil +} diff --git a/private/bufpkg/bufstate/state_test.go b/private/bufpkg/bufstate/state_test.go new file mode 100644 index 00000000..4525d2dc --- /dev/null +++ b/private/bufpkg/bufstate/state_test.go @@ -0,0 +1,45 @@ +// Copyright 2021-2025 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufstate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLatestModuleDigest(t *testing.T) { + t.Parallel() + readWriter, err := NewReadWriter() + require.NoError(t, err) + t.Run("noStateFile", func(t *testing.T) { + t.Parallel() + latestDigest, err := readWriter.LatestModuleDigest(t.TempDir(), "acme", "widgets") + require.NoError(t, err) + require.Empty(t, latestDigest) + }) + t.Run("appendedReferences", func(t *testing.T) { + t.Parallel() + rootSyncDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(rootSyncDir, "acme", "widgets"), 0755)) + require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit1", "digest1")) + require.NoError(t, readWriter.AppendModuleReference(rootSyncDir, "acme", "widgets", "commit2", "digest2")) + latestDigest, err := readWriter.LatestModuleDigest(rootSyncDir, "acme", "widgets") + require.NoError(t, err) + require.Equal(t, "digest2", latestDigest) + }) +} diff --git a/scripts/fetch.sh b/scripts/fetch.sh index a75ef27e..e54dbc4f 100755 --- a/scripts/fetch.sh +++ b/scripts/fetch.sh @@ -11,14 +11,19 @@ all_mods_sync_path="${repo_root}/modules/sync" all_mods_static_path="${repo_root}/modules/static" cd "${repo_root}" +# Build the module processor once up front. It runs at least once per reference walked, and `go run` +# relinks it on every invocation. +modprocessor_bin="${repo_root}/.tmp/bin/modprocessor" +go build -o "${modprocessor_bin}" "${repo_root}/cmd/modprocessor" + log() { >&2 echo "$@" } -# process_ref should be called within the appropriate proto src directory where files will be copied -# from. Rsync file should be relative to this dir. This func process a reference, checks out to it, -# copies relevant files and stores them in CAS format. It updates references in state files. -process_ref() { +# prepare_ref should be called within the appropriate proto src directory where files will be copied +# from. Rsync file should be relative to this dir. This func checks out to a reference and copies the +# relevant files to a fresh temporary directory, setting its path in ${prepared_ref_path}. +prepare_ref() { local -r mod_ref="${1}" local -r mod_tmp_path="$(mktemp -d)" dirs_to_delete+=("${mod_tmp_path}") @@ -60,6 +65,43 @@ process_ref() { rsync "${rsync_args[@]}" . "${mod_tmp_path}" [ ! -e "${module_static_path}/buf.md" ] || cp "${module_static_path}/buf.md" "${mod_tmp_path}" [ ! -e "${module_static_path}/buf.yaml" ] || cp "${module_static_path}/buf.yaml" "${mod_tmp_path}" + prepared_ref_path="${mod_tmp_path}" +} + +# ref_digest_status prints "changed" or "unchanged" depending on whether the contents prepared in the +# given directory differ from the digest of the module's latest reference in its state file. +ref_digest_status() { + local -r mod_tmp_path="${1}" + local -r mod_ref="${2}" + "${modprocessor_bin}" \ + --dry-run \ + --root-sync-dir="${all_mods_sync_path}" \ + --src-dir="${mod_tmp_path}" \ + --owner="${owner}" \ + --repo="${repo}" \ + --ref="${mod_ref}" +} + +# process_ref should be called within the appropriate proto src directory where files will be copied +# from. This func processes a reference, checks it out, copies relevant files in CAS format, and +# updates references in state files. +process_ref() { + local -r mod_ref="${1}" + prepare_ref "${mod_ref}" + local -r mod_tmp_path="${prepared_ref_path}" + + # Modules synced by commit mirror every commit of their source repository, but we only sync a + # curated subset of its files. Appending a reference whose contents are identical to the previous + # one publishes a BSR label pointing at identical content, so drop it. + if [ "${sync_strategy}" == "commits" ]; then + local digest_status + digest_status="$(ref_digest_status "${mod_tmp_path}" "${mod_ref}")" + if [ "${digest_status}" == "unchanged" ]; then + echo "skipping reference ${owner}/${repo}:${mod_ref}, contents unchanged" + rm -rf "${mod_tmp_path}" + return + fi + fi # If the source of the module that we are syncing is from a v2 workspace with another module # we are syncing, e.g. protovalidate and protovalidate-testing, then it is possible that @@ -88,14 +130,12 @@ process_ref() { # process the prepared module: convert it to CAS from the tmp mod directory and put blob files in # the cas path in the repo, and update the state file. - pushd "${repo_root}" > /dev/null - go run "${repo_root}/cmd/modprocessor" \ + "${modprocessor_bin}" \ --root-sync-dir="${all_mods_sync_path}" \ --src-dir="${mod_tmp_path}" \ --owner="${owner}" \ --repo="${repo}" \ --ref="${mod_ref}" - popd > /dev/null } # sync_references ${sync_strategy} ${owner} ${repo} ${git_remote} ${opt_proto_subdir} @@ -127,6 +167,16 @@ sync_references() { local -r module_root=$(pwd) pushd "${git_owner}/${git_repo}/${proto_subdir}" > /dev/null + # Resolve the tip of the cloned branch from the remote ref instead of HEAD: processing a reference + # leaves the work tree on a detached HEAD, and a single clone can back more than one managed module + # (googleapis/googleapis and googleapis/cloud-run). + local git_origin_head_ref="refs/remotes/origin/HEAD" + if ! git rev-parse --verify --quiet "${git_origin_head_ref}" > /dev/null; then + git_origin_head_ref="$(git for-each-ref --count=1 --format='%(refname)' refs/remotes/origin)" + fi + local git_origin_head + git_origin_head="$(git rev-parse "${git_origin_head_ref}")" + local rev_list if [ "${sync_strategy}" == "releases" ]; then rev_list=$(get_release_revlist) @@ -158,15 +208,15 @@ get_commit_revlist() { if [ -f "${mod_state_file}" ]; then mod_latest_ref="$(cat "${mod_state_file}" | jq -r '.references | last.name')" log "latest reference for module ${owner}/${repo}: ${mod_latest_ref}" - # revisions from initial latest_ref...HEAD (excluding latest_ref) - commit_rev_list=$(git rev-list "${mod_latest_ref}"...HEAD --first-parent --reverse) + # revisions from initial latest_ref...git_origin_head (excluding latest_ref) + commit_rev_list=$(git rev-list "${mod_latest_ref}..${git_origin_head}" --first-parent --reverse) elif [ -f "${mod_initref_file}" ]; then log "state file not found: ${mod_state_file}" mod_init_ref="$(cat "${mod_initref_file}")" log "falling back to initializing reference for module ${owner}/${repo}: ${mod_init_ref}" # Prints revisions on the main branch, stopping when ${mod_init_ref} is # encountered, and using tac to reverse the revisions (includes init_ref). - commit_rev_list=$(git rev-list HEAD --first-parent | sed "/${mod_init_ref}/q" | tac) + commit_rev_list=$(git rev-list "${git_origin_head}" --first-parent | sed "/${mod_init_ref}/q" | tac) else log "module ${owner}/${repo} has no initializing reference" exit 2