HBASE-30329 Add topology-backed cache diagnostics and migrate single tier cache

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java

@@ -17,16 +17,20 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
+import java.util.Objects;
 import java.util.Optional;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
 import org.apache.hadoop.hbase.conf.ConfigurationManager;
 import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
 import org.apache.hadoop.hbase.io.ByteBuffAllocator;
 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
-import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheEngine;
 import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
 import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheEngine;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheTier;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheTopology;
 import org.apache.hadoop.hbase.io.hfile.cache.CacheTopologyType;
 import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessService;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -221,12 +225,52 @@ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, CacheAcces
     initFromConf(conf, family);
     this.byteBuffAllocator = byteBuffAllocator;
     this.cacheAccessService = service != null ? service : CacheAccessServices.disabled();
-    this.blockCache = service instanceof BlockCacheBackedCacheAccessService
-      ? ((BlockCacheBackedCacheAccessService) service).getBlockCache()
-      : null;
+    /*
+     * Preserve the legacy BlockCache reference only when the supplied service is a direct
+     * single-tier BlockCache adapter. Multi-tier combined caches are represented by
+     * TopologyBackedCacheAccessService and intentionally do not expose a single legacy BlockCache
+     * through this field. Callers that need cache behavior or diagnostics should use
+     * cacheAccessService-level capabilities instead of relying on this legacy field.
+     */
+    this.blockCache = unwrapSingleLegacyBlockCache(this.cacheAccessService);
 
   }
 
+  /**
+   * Extracts the legacy {@link BlockCache} from a cache access service when that service is backed
+   * by a single {@link BlockCacheBackedCacheEngine}.
+   * <p>
+   * This method exists only to preserve compatibility with legacy {@link CacheConfig} callers that
+   * still use the {@link BlockCache} field. The active cache access path remains
+   * {@link CacheAccessService}. Code that needs cache behavior or diagnostics should use
+   * {@link CacheAccessService} capabilities instead of depending on the legacy block cache field.
+   * </p>
+   * @param cacheAccessService cache access service to inspect
+   * @return wrapped legacy block cache when available; otherwise {@code null}
+   * @throws NullPointerException if {@code cacheAccessService} is {@code null}
+   */
+  private static BlockCache unwrapSingleLegacyBlockCache(CacheAccessService cacheAccessService) {
+    Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null");
+
+    if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) {
+      return null;
+    }
+
+    TopologyBackedCacheAccessService topologyBackedService =
+      (TopologyBackedCacheAccessService) cacheAccessService;
+    CacheTopology topology = topologyBackedService.getTopology();
+
+    if (topology.getType() != CacheTopologyType.SINGLE_TIER) {
+      return null;
+    }
+
+    Optional<CacheEngine> engine = topology.getEngine(CacheTier.SINGLE);
+    if (!engine.isPresent() || !(engine.get() instanceof BlockCacheBackedCacheEngine)) {
+      return null;
+    }
+    return ((BlockCacheBackedCacheEngine) engine.get()).getBlockCache();
+  }
+
   /**
    * Create a cache configuration using the specified configuration object and family descriptor.
    * @param conf   hbase configuration

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java

@@ -40,4 +40,16 @@ public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize {
    * @throws IllegalArgumentException if the victim cache had already been set
    */
   void setVictimCache(BlockCache victimCache);
+
+  /**
+   * Removes the configured victim cache from this first-level cache.
+   * <p>
+   * This method is used during the block cache migration when an existing first-level cache is
+   * moved under topology-backed orchestration. Legacy combined-cache construction wires L1 to L2
+   * through a victim cache. Once L1 and L2 are represented as independent topology engines, the
+   * topology must control L2 lookup and promotion directly, so the old victim-cache delegation must
+   * be disabled.
+   * </p>
+   */
+  void unsetVictimCache();
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruAdaptiveBlockCache.java

@@ -437,6 +437,11 @@ public void setVictimCache(BlockCache victimCache) {
     victimHandler = requireNonNull(victimCache);
   }
 
+  @Override
+  public void unsetVictimCache() {
+    this.victimHandler = null;
+  }
+
   @Override
   public void setMaxSize(long maxSize) {
     this.maxSize = maxSize;

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java

@@ -346,6 +346,11 @@ public void setVictimCache(BlockCache victimCache) {
     victimHandler = requireNonNull(victimCache);
   }
 
+  @Override
+  public void unsetVictimCache() {
+    this.victimHandler = null;
+  }
+
   @Override
   public void setMaxSize(long maxSize) {
     this.maxSize = maxSize;

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java

@@ -417,4 +417,9 @@ public long getCurrentDataSize() {
   public long getDataBlockCount() {
     return getBlockCount();
   }
+
+  @Override
+  public void unsetVictimCache() {
+    this.victimCache = null;
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheEngine.java

@@ -172,6 +172,11 @@ public long getCurrentDataSize() {
     return blockCache.getCurrentDataSize();
   }
 
+  @Override
+  public long getCurrentSize() {
+    return blockCache.getCurrentSize();
+  }
+
   @Override
   public long getBlockCount() {
     return blockCache.getBlockCount();

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
 import org.apache.hadoop.hbase.io.hfile.CachedBlock;
 import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
+import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -47,26 +48,37 @@ private CacheAccessServices() {
   }
 
   /**
-   * Creates a cache access service backed by an existing block cache.
+   * Creates a {@link CacheAccessService} for the supplied legacy {@link BlockCache}.
    * <p>
-   * For regular {@link BlockCache} implementations, this returns a legacy
-   * {@link BlockCacheBackedCacheAccessService}. For {@link CombinedBlockCache}, this returns a
-   * topology-backed service using {@link TieredExclusiveTopology}. This moves combined L1/L2
-   * orchestration to the new topology layer while keeping the existing combined block cache object
-   * available for legacy {@link BlockCache}-facing APIs.
+   * All legacy block caches are adapted through {@link TopologyBackedCacheAccessService}. Plain
+   * single-tier block caches are represented by {@link SingleTierTopology}. Exclusive combined
+   * caches are represented by {@link TieredExclusiveTopology}. Inclusive combined caches are
+   * represented by {@link TieredInclusiveTopology}.
    * </p>
-   * @param blockCache block cache to expose through {@link CacheAccessService}
-   * @return cache access service
+   * <p>
+   * {@link InclusiveCombinedBlockCache} is checked before {@link CombinedBlockCache} because the
+   * inclusive variant has different residency, promotion, and eviction semantics. Routing it
+   * through the exclusive topology would be incorrect.
+   * </p>
+   * @param blockCache legacy block cache to adapt
+   * @return topology-backed cache access service for the supplied block cache
+   * @throws NullPointerException if {@code blockCache} is {@code null}
    */
-
   public static CacheAccessService fromBlockCache(BlockCache blockCache) {
     Objects.requireNonNull(blockCache, "blockCache must not be null");
+
+    if (blockCache instanceof InclusiveCombinedBlockCache) {
+      return TopologyBackedCacheAccessServices
+        .fromInclusiveCombinedBlockCache((InclusiveCombinedBlockCache) blockCache);
+    }
+
     if (blockCache instanceof CombinedBlockCache) {
       return TopologyBackedCacheAccessServices
         .fromCombinedBlockCache((CombinedBlockCache) blockCache);
     }
-    return new BlockCacheBackedCacheAccessService(blockCache);
 
+    return TopologyBackedCacheAccessServices.fromSingleBlockCache("single", blockCache,
+      DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE);
   }
 
   /**
@@ -80,7 +92,7 @@ public static CacheAccessService fromBlockCache(BlockCache blockCache) {
    * The method delegates block-cache construction to
    * {@link BlockCacheFactory#createBlockCache(Configuration)}. If the legacy factory creates a
    * {@link BlockCache}, the returned service is backed by that cache through
-   * {@link BlockCacheBackedCacheAccessService}. If the legacy factory does not create a cache, this
+   * {@link TopologyBackedCacheAccessService}. If the legacy factory does not create a cache, this
    * method returns the disabled/no-op cache access service.
    * </p>
    * <p>
@@ -140,27 +152,13 @@ public static CacheAccessService disabled() {
    * @return optional iterable cached-block view
    * @throws NullPointerException if {@code cacheAccessService} is {@code null}
    */
-  @SuppressWarnings("unchecked")
-  // public static Optional<Iterable<CachedBlock>>
-  // asCachedBlockIterable(CacheAccessService cacheAccessService) {
-  // Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null");
-  // if (cacheAccessService instanceof Iterable) {
-  // return Optional.of((Iterable<CachedBlock>) cacheAccessService);
-  // }
-  // return Optional.empty();
-  // }
 
   public static Optional<Iterable<CachedBlock>> asCachedBlockIterable(CacheAccessService service) {
     Objects.requireNonNull(service, "service must not be null");
 
     if (service instanceof TopologyBackedCacheAccessService) {
       return ((TopologyBackedCacheAccessService) service).asCachedBlockIterable();
     }
-
-    if (service instanceof BlockCacheBackedCacheAccessService) {
-      return Optional.of(((BlockCacheBackedCacheAccessService) service).getBlockCache());
-    }
-
     return Optional.empty();
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheEngine.java

@@ -215,6 +215,19 @@ default int evictBlocksByRegionName(String regionName) {
    */
   long getCurrentDataSize();
 
+  /**
+   * Returns the current total size of this cache engine.
+   * <p>
+   * The current size may include cache metadata, allocator overhead, index structures, or other
+   * implementation-specific memory that is not part of the cached block payload. This value is
+   * intentionally distinct from {@link #getCurrentDataSize()}, which reports only cached data size.
+   * </p>
+   * @return current total cache size in bytes
+   */
+  default long getCurrentSize() {
+    return getCurrentDataSize();
+  }
+
   /**
    * Returns the total number of cached blocks.
    * @return total block count

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTier.java

@@ -30,7 +30,12 @@
 public enum CacheTier {
 
   /**
-   * Single-tier topology engine.
+   * Single cache tier.
+   * <p>
+   * This tier is used by {@link SingleTierTopology}. It represents a topology with one active cache
+   * engine without assigning L1 or L2 semantics to that engine. A single-tier cache may be backed
+   * by an in-memory cache, a bucket cache, or another cache engine.
+   * </p>
    */
   SINGLE,
 

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java

@@ -28,7 +28,7 @@ public enum CacheTopologyType {
   /**
    * A topology with a single cache engine.
    */
-  SINGLE,
+  SINGLE_TIER,
 
   /**
    * A tiered topology where a block normally resides in only one tier at a time.

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.io.hfile.cache;
 
+import java.util.Arrays;
 import java.util.Objects;
 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
 import org.apache.hadoop.hbase.io.hfile.BlockType;
@@ -36,6 +37,9 @@
 @InterfaceAudience.Private
 public class DefaultHBaseCachePlacementAdmissionPolicy implements CachePlacementAdmissionPolicy {
 
+  public static final DefaultHBaseCachePlacementAdmissionPolicy INSTANCE =
+    new DefaultHBaseCachePlacementAdmissionPolicy();
+
   @Override
   public AdmissionDecision shouldAdmit(BlockCacheKey cacheKey, Cacheable block,
     CacheWriteContext context, AdmissionPriority priority, CacheTopologyView topologyView) {
@@ -48,22 +52,39 @@ public AdmissionDecision shouldAdmit(BlockCacheKey cacheKey, Cacheable block,
     return AdmissionDecision.admit();
   }
 
+  /**
+   * Selects the cache tier or tiers for an admitted block.
+   * <p>
+   * Single-tier topology has only one active cache engine and therefore selects
+   * {@link CacheTier#SINGLE}. Tiered inclusive topology writes to both L1 and L2 because inclusive
+   * residency allows the same block to be present in multiple tiers. Tiered exclusive topology
+   * keeps the legacy combined-cache placement behavior where data blocks and non-data blocks may be
+   * placed in different tiers.
+   * </p>
+   * @param cacheKey     cache key identifying the block
+   * @param block        block being cached
+   * @param context      cache write context
+   * @param topologyView topology view available to the placement policy
+   * @return tier decision for the admitted block
+   * @throws NullPointerException if {@code cacheKey}, {@code block}, {@code context}, or
+   *                              {@code topologyView} is {@code null}
+   */
+
   @Override
   public TierDecision selectTier(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context,
     CacheTopologyView topologyView) {
     Objects.requireNonNull(cacheKey, "cacheKey must not be null");
     Objects.requireNonNull(block, "block must not be null");
     Objects.requireNonNull(context, "context must not be null");
     Objects.requireNonNull(topologyView, "topologyView must not be null");
-    /*
-     * Default compatibility placement prefers metadata/index/bloom blocks in L1 and data blocks in
-     * L2 when both tiers are available, but falls back to any available tier rather than rejecting
-     * placement.
-     */
-    if (topologyView.getType() == CacheTopologyType.SINGLE) {
+    if (topologyView.getType() == CacheTopologyType.SINGLE_TIER) {
       return TierDecision.single(CacheTier.SINGLE);
     }
 
+    if (topologyView.getType() == CacheTopologyType.TIERED_INCLUSIVE) {
+      return TierDecision.multiple(Arrays.asList(CacheTier.L1, CacheTier.L2));
+    }
+
     if (isMetaOrIndexBlock(block)) {
       if (topologyView.getEngine(CacheTier.L1).isPresent()) {
         return TierDecision.single(CacheTier.L1);