diff --git a/README.md b/README.md index dd107435c..904755733 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,6 @@ Tapioca makes it easy to work with [Sorbet](https://sorbet.org) in your codebase * [Writing custom DSL extensions](#writing-custom-dsl-extensions) * [Rewriting RBS comments to Sorbet signatures](#rewriting-rbs-comments-to-sorbet-signatures) * [Caching rewrites with Bootsnap](#caching-rewrites-with-bootsnap) - * [Priming the cache from CI](#priming-the-cache-from-ci) * [RBI files for missing constants and methods](#rbi-files-for-missing-constants-and-methods) * [Configuration](#configuration) * [Editor Integration](#editor-integration) @@ -492,37 +491,35 @@ Usage: tapioca dsl [constant...] Options: - --out, -o, [--outdir=directory] # The output directory for generated DSL RBI files - # Default: sorbet/rbi/dsl - [--file-header], [--no-file-header], [--skip-file-header] # Add a "This file is generated" header on top of each generated RBI file - # Default: true - [--only=compiler [compiler ...]] # Only run supplied DSL compiler(s) - [--exclude=compiler [compiler ...]] # Exclude supplied DSL compiler(s) - [--verify], [--no-verify], [--skip-verify] # Verifies RBIs are up-to-date - # Default: false - [--only-bootsnap-rbs-cache], [--no-only-bootsnap-rbs-cache], [--skip-only-bootsnap-rbs-cache] # Only boot the application and load DSL extensions/compilers to populate the bootsnap iseq cache, then exit. Skips compiler execution and RBI generation. Mutually exclusive with --verify and --list-compilers. - # Default: false - -q, [--quiet], [--no-quiet], [--skip-quiet] # Suppresses file creation output - # Default: false - -w, [--workers=N] # Number of parallel workers to use when generating RBIs (default: auto) - [--rbi-max-line-length=N] # Set the max line length of generated RBIs. Signatures longer than the max line length will be wrapped - # Default: 120 - [--max-diff-lines=N] # Max number of diff lines to include in the `dsl --verify` output - # Default: 250 - -e, [--environment=ENVIRONMENT] # The Rack/Rails environment to use when generating RBIs - # Default: development - -l, [--list-compilers], [--no-list-compilers], [--skip-list-compilers] # List all loaded compilers - # Default: false - [--app-root=APP_ROOT] # The path to the Rails application - # Default: . - [--halt-upon-load-error], [--no-halt-upon-load-error], [--skip-halt-upon-load-error] # Halt upon a load error while loading the Rails application - # Default: true - [--skip-constant=constant [constant ...]] # Do not generate RBI definitions for the given application constant(s) - [--compiler-options=key:value] # Options to pass to the DSL compilers - -c, [--config=] # Path to the Tapioca configuration file - # Default: sorbet/tapioca/config.yml - -V, [--verbose], [--no-verbose], [--skip-verbose] # Verbose output for debugging purposes - # Default: false + --out, -o, [--outdir=directory] # The output directory for generated DSL RBI files + # Default: sorbet/rbi/dsl + [--file-header], [--no-file-header], [--skip-file-header] # Add a "This file is generated" header on top of each generated RBI file + # Default: true + [--only=compiler [compiler ...]] # Only run supplied DSL compiler(s) + [--exclude=compiler [compiler ...]] # Exclude supplied DSL compiler(s) + [--verify], [--no-verify], [--skip-verify] # Verifies RBIs are up-to-date + # Default: false + -q, [--quiet], [--no-quiet], [--skip-quiet] # Suppresses file creation output + # Default: false + -w, [--workers=N] # Number of parallel workers to use when generating RBIs (default: auto) + [--rbi-max-line-length=N] # Set the max line length of generated RBIs. Signatures longer than the max line length will be wrapped + # Default: 120 + [--max-diff-lines=N] # Max number of diff lines to include in the `dsl --verify` output + # Default: 250 + -e, [--environment=ENVIRONMENT] # The Rack/Rails environment to use when generating RBIs + # Default: development + -l, [--list-compilers], [--no-list-compilers], [--skip-list-compilers] # List all loaded compilers + # Default: false + [--app-root=APP_ROOT] # The path to the Rails application + # Default: . + [--halt-upon-load-error], [--no-halt-upon-load-error], [--skip-halt-upon-load-error] # Halt upon a load error while loading the Rails application + # Default: true + [--skip-constant=constant [constant ...]] # Do not generate RBI definitions for the given application constant(s) + [--compiler-options=key:value] # Options to pass to the DSL compilers + -c, [--config=] # Path to the Tapioca configuration file + # Default: sorbet/tapioca/config.yml + -V, [--verbose], [--no-verbose], [--skip-verbose] # Verbose output for debugging purposes + # Default: false Generate RBIs for dynamic methods ``` @@ -857,7 +854,13 @@ The rewriting is automatic on every `tapioca` invocation: [`require-hooks`](http $ TAPIOCA_RBS_CACHE=1 bin/tapioca dsl ``` -Tapioca configures Bootsnap's iseq cache against a dedicated directory (`tmp/cache/bootsnap-tapioca-rbs` by default; override with `TAPIOCA_BOOTSNAP_CACHE_DIR`). The first run is slower because every file is rewritten and the result is baked into the iseq cache; subsequent runs against the same directory skip the rewrite entirely. +Tapioca configures Bootsnap's iseq cache against a dedicated directory (`tmp/cache/bootsnap-tapioca-rbs` by +default; override with `TAPIOCA_BOOTSNAP_CACHE_DIR`). + +Tapioca writes the current `Gemfile.lock` digest to `.gemfile-lock-digest` inside that cache directory. When the +lockfile changes, Tapioca sees the digest mismatch and resets Bootsnap's cache payload before configuring Bootsnap. +This lets gem bumps that affect rewriting, such as `tapioca`, start from a fresh cache without accumulating old cache +directories. `Bootsnap.setup` mutates a process-wide singleton, and a second call would overwrite Tapioca's dedicated cache directory and start writing rewritten iseqs into the host's normal cache. Tapioca enforces this under `TAPIOCA_RBS_CACHE=1`: after its own setup runs, any subsequent `Bootsnap.setup` raises a clear error pointing at the fix. Gate your host's `Bootsnap.setup` on the same env var. Rails apps do this in `config/boot.rb`: @@ -866,19 +869,6 @@ Tapioca configures Bootsnap's iseq cache against a dedicated directory (`tmp/cac require "bootsnap/setup" unless ENV["TAPIOCA_RBS_CACHE"] == "1" ``` -#### Priming the cache from CI - -For CI pipelines that want to populate the cache once and have downstream jobs read from a warm copy, use `--only-bootsnap-rbs-cache`. This pattern lets you scope cache writes to a single job (the prime) so PR-side jobs read from it without uploading on every successful build: - -```shell -# Prime: populate the cache. -$ TAPIOCA_RBS_CACHE=1 bin/tapioca dsl --only-bootsnap-rbs-cache - -# Consumer: read from the populated cache. -# BOOTSNAP_READONLY=1 prevents bootsnap from writing back to a read-only mount. -$ TAPIOCA_RBS_CACHE=1 BOOTSNAP_READONLY=1 bin/tapioca dsl -``` - ### RBI files for missing constants and methods Even after generating the RBIs, it is possible that some constants or methods are still undefined for Sorbet. @@ -1000,7 +990,6 @@ dsl: only: [] exclude: [] verify: false - only_bootsnap_rbs_cache: false quiet: false workers: 1 rbi_max_line_length: 120 diff --git a/lib/tapioca/cli.rb b/lib/tapioca/cli.rb index a46468f27..ab5a605f9 100644 --- a/lib/tapioca/cli.rb +++ b/lib/tapioca/cli.rb @@ -103,10 +103,6 @@ def todo type: :boolean, default: false, desc: "Verifies RBIs are up-to-date" - option :only_bootsnap_rbs_cache, - type: :boolean, - default: false, - desc: "Only boot the application and load DSL extensions/compilers to populate the bootsnap iseq cache, then exit. Skips compiler execution and RBI generation. Mutually exclusive with --verify and --list-compilers." option :quiet, aliases: ["-q"], type: :boolean, @@ -154,12 +150,6 @@ def todo def dsl(*constant_or_paths) set_environment(options) - if options[:only_bootsnap_rbs_cache] && (options[:verify] || options[:list_compilers]) - conflicting = options[:verify] ? "--verify" : "--list-compilers" - raise MalformattedArgumentError, - "Options '--only-bootsnap-rbs-cache' and '#{conflicting}' are mutually exclusive" - end - # Assume anything starting with a capital letter or colon is a class, otherwise a path constants, paths = constant_or_paths.partition { |c| c =~ /\A[A-Z:]/ } @@ -192,7 +182,7 @@ def dsl(*constant_or_paths) elsif options[:list_compilers] Commands::DslCompilerList.new(**command_args) else - Commands::DslGenerate.new(**command_args, only_bootsnap_rbs_cache: options[:only_bootsnap_rbs_cache]) + Commands::DslGenerate.new(**command_args) end command.run diff --git a/lib/tapioca/commands/dsl_generate.rb b/lib/tapioca/commands/dsl_generate.rb index f1c8d0971..3114bebd6 100644 --- a/lib/tapioca/commands/dsl_generate.rb +++ b/lib/tapioca/commands/dsl_generate.rb @@ -4,12 +4,6 @@ module Tapioca module Commands class DslGenerate < AbstractDsl - #: (?only_bootsnap_rbs_cache: bool, **untyped) -> void - def initialize(only_bootsnap_rbs_cache: false, **kwargs) - @only_bootsnap_rbs_cache = only_bootsnap_rbs_cache - super(**T.unsafe(kwargs)) - end - private # @override @@ -17,15 +11,6 @@ def initialize(only_bootsnap_rbs_cache: false, **kwargs) def execute load_application - if @only_bootsnap_rbs_cache - if ENV["TAPIOCA_RBS_CACHE"] == "1" - say("Bootsnap RBS cache populated, exiting before RBI generation.", :green) - else - say_error("Warning: --only-bootsnap-rbs-cache requires TAPIOCA_RBS_CACHE=1 to populate the cache", :yellow) - end - return - end - say("Compiling DSL RBI files...") say("") diff --git a/lib/tapioca/rbs/bootsnap_cache.rb b/lib/tapioca/rbs/bootsnap_cache.rb new file mode 100644 index 000000000..f7bfcb8a6 --- /dev/null +++ b/lib/tapioca/rbs/bootsnap_cache.rb @@ -0,0 +1,52 @@ +# typed: strict +# frozen_string_literal: true + +require "bundler" +require "digest" +require "fileutils" + +module Tapioca + module RBS + # Prepares the Bootsnap iseq cache used for RBS rewrite output. + # + # RBS rewrite output can change when the lockfile changes, even if the + # source files are unchanged. + # To account for this, we store the current Gemfile.lock SHA256 in a + # `.gemfile-lock-digest` file. + # A digest mismatch deletes Bootsnap's cache payload and records the new + # digest, so this run rebuilds the cache from scratch. + module BootsnapCache + DIGEST_FILE = ".gemfile-lock-digest" #: String + + class << self + #: (String) -> void + def prepare_for_setup(cache_dir) + digest = gemfile_lock_digest + return if digest_matches?(cache_dir, digest) + + FileUtils.rm_rf(File.join(cache_dir, "bootsnap")) + FileUtils.mkdir_p(cache_dir) + File.write(digest_path(cache_dir), digest) + end + + private + + #: -> String + def gemfile_lock_digest + Digest::SHA256.file(Bundler.default_lockfile).hexdigest + end + + #: (String, String) -> bool + def digest_matches?(cache_dir, digest) + path = digest_path(cache_dir) + File.file?(path) && File.read(path).chomp == digest + end + + #: (String) -> String + def digest_path(cache_dir) + File.join(cache_dir, DIGEST_FILE) + end + end + end + end +end diff --git a/lib/tapioca/rbs/rewriter.rb b/lib/tapioca/rbs/rewriter.rb index ac5bb1974..def5bd00e 100644 --- a/lib/tapioca/rbs/rewriter.rb +++ b/lib/tapioca/rbs/rewriter.rb @@ -1,6 +1,8 @@ # typed: strict # frozen_string_literal: true +require "tapioca/rbs/bootsnap_cache" + # This code rewrites RBS comments back into Sorbet's signatures as the files are being loaded. # This will allow `sorbet-runtime` to wrap the methods as if they were originally written with the `sig{}` blocks. # This will in turn allow Tapioca to use this signatures to generate typed RBI files. @@ -29,32 +31,41 @@ def setup(**_kwargs) MSG end end + + module BootsnapIntegration + class << self + extend T::Sig + + sig { void } + def setup + require "bootsnap" + + cache_dir = ENV.fetch("TAPIOCA_BOOTSNAP_CACHE_DIR", File.join(Dir.pwd, "tmp/cache/bootsnap-tapioca-rbs")) + Tapioca::RBS::BootsnapCache.prepare_for_setup(cache_dir) + + Bootsnap.setup( + cache_dir: cache_dir, + development_mode: true, + load_path_cache: true, + compile_cache_iseq: true, + compile_cache_yaml: true, + readonly: false, + revalidation: true, + ) + Bootsnap.log_stats! + + Bootsnap.singleton_class.prepend(Tapioca::RBS::BootsnapGuard) + end + end + end end end -# When TAPIOCA_RBS_CACHE=1, set up bootsnap with a dedicated cache directory -# and load require-hooks so the RBS-rewritten iseqs get cached. Subsequent -# runs read the rewritten iseq directly and skip the rewrite. -# -# After our setup, BootsnapGuard is prepended so the host application can't -# replace our cache directory. +# When TAPIOCA_RBS_CACHE=1, use a dedicated Bootsnap cache directory for +# RBS-rewritten iseqs. if ENV["TAPIOCA_RBS_CACHE"] == "1" begin - require "bootsnap" - # Respect BOOTSNAP_READONLY for consumers reading a pre-populated cache - # (e.g. a CI prime step). - readonly = !["0", "false", false].include?(ENV.fetch("BOOTSNAP_READONLY") { false }) - Bootsnap.setup( - cache_dir: ENV.fetch("TAPIOCA_BOOTSNAP_CACHE_DIR", File.join(Dir.pwd, "tmp/cache/bootsnap-tapioca-rbs")), - development_mode: true, - load_path_cache: true, - compile_cache_iseq: true, - compile_cache_yaml: true, - readonly: readonly, - revalidation: true, - ) - Bootsnap.log_stats! - Bootsnap.singleton_class.prepend(Tapioca::RBS::BootsnapGuard) + Tapioca::RBS::BootsnapIntegration.setup rescue LoadError # Bootsnap is not in the bundle, skip iseq caching. end diff --git a/spec/tapioca/cli/dsl_spec.rb b/spec/tapioca/cli/dsl_spec.rb index b8ae36369..246dbbe33 100644 --- a/spec/tapioca/cli/dsl_spec.rb +++ b/spec/tapioca/cli/dsl_spec.rb @@ -659,7 +659,7 @@ class Post assert_success_status(result) end - it "exits before RBI generation when --only-bootsnap-rbs-cache is set" do + it "resets the bootsnap cache when Gemfile.lock changes" do @project.write!("lib/post.rb", <<~RB) require "smart_properties" @@ -669,36 +669,22 @@ class Post end RB - result = @project.tapioca("dsl --only-bootsnap-rbs-cache Post", env: { "TAPIOCA_RBS_CACHE" => "1" }) + env = { + "TAPIOCA_RBS_CACHE" => "1", + "TAPIOCA_BOOTSNAP_CACHE_DIR" => "tmp/cache/test-bootsnap-tapioca-rbs", + } - assert_stdout_includes(result, <<~OUT) - Bootsnap RBS cache populated, exiting before RBI generation. - OUT + result = @project.tapioca("dsl Post", env: env) - assert_stderr_includes(result, "bootsnap miss:") - refute_project_file_exist("sorbet/rbi/dsl/post.rbi") assert_success_status(result) - end - - it "warns when --only-bootsnap-rbs-cache is set without TAPIOCA_RBS_CACHE=1" do - @project.write!("lib/post.rb", <<~RB) - require "smart_properties" - - class Post - include SmartProperties - property :title, accepts: String - end - RB + @project.write!("tmp/cache/test-bootsnap-tapioca-rbs/bootsnap/stale-cache-entry", "stale") - result = @project.tapioca("dsl --only-bootsnap-rbs-cache Post") + @project.write!("Gemfile.lock", "#{@gemfile_lock}\n") + result = @project.tapioca("dsl Post", env: env) - assert_stderr_includes( - result, - "Warning: --only-bootsnap-rbs-cache requires TAPIOCA_RBS_CACHE=1 to populate the cache", - ) - refute_includes(result.out, "Bootsnap RBS cache populated") - refute_project_file_exist("sorbet/rbi/dsl/post.rbi") assert_success_status(result) + refute_project_file_exist("tmp/cache/test-bootsnap-tapioca-rbs/bootsnap/stale-cache-entry") + assert_project_file_exist("tmp/cache/test-bootsnap-tapioca-rbs/.gemfile-lock-digest") end it "preserves RBS comment rewriting when the host sets up Bootsnap without TAPIOCA_RBS_CACHE" do @@ -2145,26 +2131,6 @@ class Post assert_success_status(result) end - it "rejects --only-bootsnap-rbs-cache combined with --verify" do - result = @project.tapioca("dsl --verify --only-bootsnap-rbs-cache") - - assert_stderr_includes( - result, - "Options '--only-bootsnap-rbs-cache' and '--verify' are mutually exclusive", - ) - refute_success_status(result) - end - - it "rejects --only-bootsnap-rbs-cache combined with --list-compilers" do - result = @project.tapioca("dsl --list-compilers --only-bootsnap-rbs-cache") - - assert_stderr_includes( - result, - "Options '--only-bootsnap-rbs-cache' and '--list-compilers' are mutually exclusive", - ) - refute_success_status(result) - end - it "rejects negative --max-diff-lines values" do ["0", "-1"].each do |value| result = @project.tapioca("dsl --verify --max-diff-lines=#{value}")