feat: Add the override store, overlay, and data system wiring

lib/ldclient-rb/config.rb

@@ -736,14 +736,18 @@ class DataSystemConfig
     # @param data_store [LaunchDarkly::Interfaces::FeatureStore, nil] The (optional) data store
     # @param fdv1_fallback_synchronizer [#build(String, Config), nil]
     #   The (optional) builder for FDv1-compatible fallback synchronizer
+    # @param overrides [#build(String, Config), nil] The (optional) builder for an override source.
+    #   Flag overrides are currently experimental and subject to change.
     #
     def initialize(initializers: nil, synchronizers: nil,
-                   data_store_mode: LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY, data_store: nil, fdv1_fallback_synchronizer: nil)
+                   data_store_mode: LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY, data_store: nil, fdv1_fallback_synchronizer: nil,
+                   overrides: nil)
       @initializers = initializers
       @synchronizers = synchronizers
       @data_store_mode = data_store_mode
       @data_store = data_store
       @fdv1_fallback_synchronizer = fdv1_fallback_synchronizer
+      @overrides = overrides
     end
 
     # The initializer builders for the data system. Each builder responds to build(sdk_key, config) and returns an Initializer.
@@ -765,5 +769,11 @@ def initialize(initializers: nil, synchronizers: nil,
     # The FDv1-compatible fallback synchronizer builder. Responds to build(sdk_key, config) and returns a Synchronizer.
     # @return [#build(String, Config), nil]
     attr_reader :fdv1_fallback_synchronizer
+
+    # The override source builder. Responds to build(sdk_key, config) and returns an
+    # {LaunchDarkly::Interfaces::Overrides::OverrideSource}. Flag overrides are currently
+    # experimental and subject to change.
+    # @return [#build(String, Config), nil]
+    attr_reader :overrides
   end
 end

lib/ldclient-rb/data_system/config_builder.rb

@@ -23,6 +23,7 @@ def initialize
         @fdv1_fallback_synchronizer = nil
         @data_store_mode = LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY
         @data_store = nil
+        @overrides = nil
       end
 
       #
@@ -85,6 +86,38 @@ def data_store(data_store, store_mode)
         self
       end
 
+      #
+      # Configures the SDK with an override source. Flag overrides are currently experimental and
+      # subject to change.
+      #
+      # Overrides are flag and segment definitions that take precedence over data received from
+      # LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an
+      # incident. Overrides let an operator force one or more flags to a known state on a running
+      # client, whether or not the client can reach LaunchDarkly. Flags not present in the override
+      # data are completely unaffected.
+      #
+      # The override source is not a data source. It has no effect on the client's initialization
+      # status, data availability, or data source status. Without a configured source, or while
+      # the source supplies no overrides, the SDK behaves as it would without this feature.
+      #
+      # At most one override source can be configured. A later call replaces an earlier one.
+      #
+      # When the client is offline, the source is not built or started: no overrides are served,
+      # and a configuration error in the source is not reported. The SDK logs a warning instead.
+      #
+      # @example
+      #   overrides = LaunchDarkly::Integrations::FileData.override_source(paths: ["/etc/launchdarkly/overrides.json"])
+      #   config = LaunchDarkly::Config.new(data_system_config: LaunchDarkly::DataSystem.default.overrides(overrides).build)
+      #
+      # @param source [#build(String, Config), nil] a builder that responds to build(sdk_key, config)
+      #   and returns a {LaunchDarkly::Interfaces::Overrides::OverrideSource}, or nil to remove one
+      # @return [ConfigBuilder] self for chaining
+      #
+      def overrides(source)
+        @overrides = source
+        self
+      end
+
       #
       # Builds the data system configuration.
       #
@@ -96,7 +129,8 @@ def build
           synchronizers: @synchronizers,
           data_store_mode: @data_store_mode,
           data_store: @data_store,
-          fdv1_fallback_synchronizer: @fdv1_fallback_synchronizer
+          fdv1_fallback_synchronizer: @fdv1_fallback_synchronizer,
+          overrides: @overrides
         )
       end
     end

lib/ldclient-rb/impl/data_store/store.rb

@@ -42,6 +42,10 @@ def initialize(flag_change_broadcaster, change_set_broadcaster, logger)
           # the form in which the data model names prerequisites and segments.
           @dependency_tracker = LaunchDarkly::Impl::DependencyTracker.new(logger)
 
+          # A second dependency graph, such as the override layer's, whose items may depend on
+          # items in this store. Set once when the data system is built, before any data.
+          @external_dependencies = nil
+
           # Broadcasters for events
           @flag_change_broadcaster = flag_change_broadcaster
           @change_set_broadcaster = change_set_broadcaster
@@ -180,6 +184,41 @@ def initialized?
           get_active_store.initialized?
         end
 
+        #
+        # Retrieves an item from the active store.
+        #
+        # @param kind [LaunchDarkly::Impl::DataStore::DataKind]
+        # @param key [String, Symbol]
+        # @return [Object, nil] the item, or nil if it is unknown or deleted
+        #
+        def get(kind, key)
+          get_active_store.get(kind, key)
+        end
+
+        #
+        # Retrieves all items of a kind from the active store.
+        #
+        # @param kind [LaunchDarkly::Impl::DataStore::DataKind]
+        # @return [Hash{Symbol => Object}]
+        #
+        def all(kind)
+          get_active_store.all(kind)
+        end
+
+        #
+        # Registers a second dependency graph whose items may depend on items in this store. A
+        # change to an item in this store then also notifies the external items that depend on
+        # it, directly or through any chain of dependencies in either graph.
+        #
+        # @param dependencies [#add_affected_items, nil] a graph answering `add_affected_items(items_out, { kind:, key: })`
+        # @return [void]
+        #
+        def set_external_dependencies(dependencies)
+          @lock.synchronize do
+            @external_dependencies = dependencies
+          end
+        end
+
         #
         # Get the data store status provider for the persistent store, if configured.
         #
@@ -257,7 +296,7 @@ def get_data_store_status_provider
             collection.each do |key, item|
               @dependency_tracker.update_dependencies_from(kind, key.to_s, item)
               if has_listeners
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key.to_s })
+                add_affected_items(affected_items, { kind: kind, key: key })
               end
             end
           end
@@ -344,6 +383,44 @@ def get_data_store_status_provider
           end
         end
 
+        #
+        # Adds the item and everything that depends on it to the set, following this store's own
+        # dependency graph and the external one. Every item reached through either graph is
+        # expanded through both, so a dependency chain may cross between them any number of
+        # times. Keys are compared as strings, so a change keyed by symbol reaches the same
+        # dependents as one keyed by string. Without an external graph, the store's own tracker
+        # expands the item in one traversal, as it did before external graphs existed.
+        #
+        # @param items_out [Set<Hash>] Set of {kind:, key:} hashes
+        # @param initial_modified_item [Hash] {kind:, key:} hash
+        # @return [void]
+        #
+        private def add_affected_items(items_out, initial_modified_item)
+          start = { kind: initial_modified_item[:kind], key: initial_modified_item[:key].to_s }
+          if @external_dependencies.nil?
+            @dependency_tracker.add_affected_items(items_out, start)
+            return
+          end
+
+          pending = [start]
+          until pending.empty?
+            current = pending.pop
+            # Each graph gets its own working set: a tracker does not expand an item that is
+            # already present, so a shared set would stop the second graph at the start.
+            via_store = Set.new
+            @dependency_tracker.add_affected_items(via_store, current)
+            via_external = Set.new
+            @external_dependencies&.add_affected_items(via_external, current)
+            (via_store | via_external).each do |reached|
+              item = { kind: reached[:kind], key: reached[:key].to_s }
+              next if items_out.include?(item)
+
+              items_out.add(item)
+              pending << item
+            end
+          end
+        end
+
         #
         # Send flag change events for affected items.
         #
@@ -381,9 +458,9 @@ def get_data_store_status_provider
 
               # If either is missing or versions differ, it's a change
               if old_item.nil? || new_item.nil?
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key.to_s })
+                add_affected_items(affected_items, { kind: kind, key: key })
               elsif old_item[:version] != new_item[:version]
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key.to_s })
+                add_affected_items(affected_items, { kind: kind, key: key })
               end
             end
           end

lib/ldclient-rb/impl/data_system.rb

@@ -108,6 +108,19 @@ def store
         raise NotImplementedError, "#{self.class} must implement #store"
       end
 
+      #
+      # Returns the override store's entries of the given kind, without LaunchDarkly data. The
+      # result is empty when no override source is configured or the store holds none. The
+      # returned hash must not be modified. Flag overrides are currently experimental and subject
+      # to change.
+      #
+      # @param kind [LaunchDarkly::Impl::DataStore::DataKind]
+      # @return [Hash{Symbol => Object}]
+      #
+      def override_items(kind)
+        raise NotImplementedError, "#{self.class} must implement #override_items"
+      end
+
       #
       # Sets the diagnostic accumulator for streaming initialization metrics.
       # This should be called before start() to ensure metrics are collected.

lib/ldclient-rb/impl/data_system/fdv1.rb

@@ -101,6 +101,11 @@ def store
           @store_wrapper
         end
 
+        # (see DataSystem#override_items)
+        def override_items(_kind)
+          {}
+        end
+
         # (see DataSystem#set_diagnostic_accumulator)
         def set_diagnostic_accumulator(diagnostic_accumulator)
           @diagnostic_accumulator = diagnostic_accumulator

lib/ldclient-rb/impl/data_system/fdv2.rb

@@ -8,7 +8,9 @@
 require "ldclient-rb/impl/data_source/status_provider"
 require "ldclient-rb/impl/data_store/status_provider"
 require "ldclient-rb/impl/broadcaster"
+require "ldclient-rb/impl/overrides"
 require "ldclient-rb/impl/repeating_task"
+require "ldclient-rb/impl/util"
 require "ldclient-rb/interfaces/data_system"
 
 module LaunchDarkly
@@ -118,6 +120,28 @@ def initialize(sdk_key, config, data_system_config)
           # Track configuration
           @configured_with_data_sources = (@data_system_config.initializers && !@data_system_config.initializers.empty?) ||
             !@synchronizer_builders.empty?
+
+          # The following are nil unless an override source is configured. The layer holds override
+          # entries. The overlay serves them in preference to the store's data at the store read
+          # boundary. The source populates the layer at runtime. The override system is separate
+          # from the initializer and synchronizer pipeline and never affects data availability.
+          @override_layer = nil
+          @overlay = nil
+          @override_source = nil
+          @override_sink = nil
+          if @data_system_config.overrides && @disabled
+            @logger.warn { "[LDClient] Offline mode is enabled; the override source is not started and no overrides are served" }
+          elsif @data_system_config.overrides
+            # An invalid override source configuration raises here, so LDClient.new reports it
+            # the same way as other invalid component configuration.
+            @override_source = @data_system_config.overrides.build(@sdk_key, @config)
+            @override_layer = LaunchDarkly::Impl::Overrides::Layer.new
+            @overlay = LaunchDarkly::Impl::Overrides::Overlay.new(@store, @override_layer)
+            # A LaunchDarkly change to a flag or segment that an override entry depends on must
+            # notify that entry's flag as well. The store reaches those entries through the
+            # layer's dependency graph.
+            @store.set_external_dependencies(@override_layer)
+          end
         end
 
         # (see DataSystem#start)
@@ -131,6 +155,10 @@ def start
           @stop_event.reset
           @ready_event.reset
 
+          # The override source starts before the run loop, so a source that loads synchronously has
+          # its overrides in place before the client begins evaluating.
+          start_override_source if @override_source
+
           # Start the main coordination thread
           main_thread = Thread.new { run_main_loop }
           main_thread.name = "FDv2-main"
@@ -141,6 +169,14 @@ def start
 
         # (see DataSystem#stop)
         def stop
+          if @override_source
+            # The sink stops accepting snapshots first, so a snapshot that the source supplies while
+            # it stops cannot update the layer or notify listeners while the system shuts down. The
+            # sink does not exist if the system was never started.
+            @override_sink&.close
+            stop_override_source("Error stopping override source")
+          end
+
           @stop_event.set
 
           @lock.synchronize do
@@ -175,9 +211,18 @@ def set_diagnostic_accumulator(diagnostic_accumulator)
 
         # (see DataSystem#store)
         def store
+          return @overlay if @overlay
+
           @store.get_active_store
         end
 
+        # (see DataSystem#override_items)
+        def override_items(kind)
+          return {} if @override_layer.nil?
+
+          @override_layer.all(kind)
+        end
+
         # (see DataSystem#data_source_status_provider)
         def data_source_status_provider
           @data_source_status_provider
@@ -215,6 +260,45 @@ def target_availability
 
         private
 
+        #
+        # Starts the override source. The source has no effect on initialization, and a source that
+        # fails to start does not prevent the client from operating: the failure is logged, the
+        # source is stopped, and the client runs without overrides. The SDK's own file-based source
+        # does not raise from start; it logs a failed load and retries it. A custom source could
+        # raise, and an application in an incident is better served by a client that runs without
+        # its overrides than by one that does not start.
+        #
+        def start_override_source
+          @override_sink = LaunchDarkly::Impl::Overrides::Sink.new(@override_layer, @store, @flag_change_broadcaster, @logger)
+          begin
+            @override_source.start(@override_sink)
+          rescue StandardError, ScriptError => e
+            # ScriptError covers the NotImplementedError that the interface's default start raises
+            # for a source that does not implement it.
+            LaunchDarkly::Impl::Util.log_exception(@logger, "Unable to start the override source; the client runs without overrides", e)
+            stop_override_source("Error stopping override source after it failed to start")
+            # The source may have supplied a snapshot before it failed. Clearing the layer removes
+            # it, so the client runs without overrides as logged, and a late snapshot from the
+            # stopped source is ignored.
+            @override_sink.close_and_clear
+          end
+        end
+
+        #
+        # Stops the override source and forgets it. A custom source can raise from stop, including
+        # the NotImplementedError of the interface's default stop; that is logged and not raised,
+        # so the rest of the system still stops, or still starts after a failed start.
+        #
+        def stop_override_source(failure_message)
+          source = @override_source
+          @override_source = nil
+          begin
+            source.stop
+          rescue StandardError, ScriptError => e
+            @logger.error { "[LDClient] #{failure_message}: #{e.message}" }
+          end
+        end
+
         #
         # Main coordination loop that manages initializers and synchronizers.
         #

lib/ldclient-rb/impl/overrides.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "ldclient-rb/impl/overrides/layer"
+require "ldclient-rb/impl/overrides/overlay"
+require "ldclient-rb/impl/overrides/sink"