From e08d00dcac0fa861f95af608489e796b51fe9c07 Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Tue, 25 Aug 2026 20:21:13 -0700 Subject: [PATCH] HBASE-30329 Add topology-backed cache diagnostics --- .../hadoop/hbase/io/hfile/CacheConfig.java | 52 ++- .../hbase/io/hfile/FirstLevelBlockCache.java | 12 + .../hbase/io/hfile/LruAdaptiveBlockCache.java | 5 + .../hadoop/hbase/io/hfile/LruBlockCache.java | 5 + .../hbase/io/hfile/TinyLfuBlockCache.java | 5 + .../cache/BlockCacheBackedCacheEngine.java | 5 + .../io/hfile/cache/CacheAccessServices.java | 48 +- .../hbase/io/hfile/cache/CacheEngine.java | 13 + .../hbase/io/hfile/cache/CacheTier.java | 7 +- .../io/hfile/cache/CacheTopologyType.java | 2 +- ...ultHBaseCachePlacementAdmissionPolicy.java | 33 +- .../io/hfile/cache/SingleEngineTopology.java | 93 ---- .../io/hfile/cache/SingleTierTopology.java | 180 ++++++++ .../TopologyBackedCacheAccessService.java | 149 +++++- .../TopologyBackedCacheAccessServices.java | 189 +++++++- .../hbase/io/hfile/TestCacheConfig.java | 20 +- .../hbase/io/hfile/TestCacheOnWrite.java | 10 +- .../cache/CacheAccessServiceTestFactory.java | 12 +- ...estBlockCacheBackedCacheAccessService.java | 246 ---------- .../hfile/cache/TestCacheAccessServices.java | 2 +- ...tibleTopologyBackedCacheAccessService.java | 45 ++ ...tibleTopologyBackedCacheAccessService.java | 433 ++++++++++++++++++ ...eTierTopologyBackedCacheAccessService.java | 425 +++++++++++++++++ 23 files changed, 1551 insertions(+), 440 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index fa69b77f5ecb..9ab39b00c90d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; +import java.util.Objects; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -24,9 +25,12 @@ import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheEngine; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices; +import org.apache.hadoop.hbase.io.hfile.cache.CacheEngine; +import org.apache.hadoop.hbase.io.hfile.cache.CacheTier; +import org.apache.hadoop.hbase.io.hfile.cache.CacheTopology; import org.apache.hadoop.hbase.io.hfile.cache.CacheTopologyType; import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessService; import org.apache.yetus.audience.InterfaceAudience; @@ -221,12 +225,52 @@ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, CacheAcces initFromConf(conf, family); this.byteBuffAllocator = byteBuffAllocator; this.cacheAccessService = service != null ? service : CacheAccessServices.disabled(); - this.blockCache = service instanceof BlockCacheBackedCacheAccessService - ? ((BlockCacheBackedCacheAccessService) service).getBlockCache() - : null; + /* + * Preserve the legacy BlockCache reference only when the supplied service is a direct + * single-tier BlockCache adapter. Multi-tier combined caches are represented by + * TopologyBackedCacheAccessService and intentionally do not expose a single legacy BlockCache + * through this field. Callers that need cache behavior or diagnostics should use + * cacheAccessService-level capabilities instead of relying on this legacy field. + */ + this.blockCache = unwrapSingleLegacyBlockCache(this.cacheAccessService); } + /** + * Extracts the legacy {@link BlockCache} from a cache access service when that service is backed + * by a single {@link BlockCacheBackedCacheEngine}. + *

+ * This method exists only to preserve compatibility with legacy {@link CacheConfig} callers that + * still use the {@link BlockCache} field. The active cache access path remains + * {@link CacheAccessService}. Code that needs cache behavior or diagnostics should use + * {@link CacheAccessService} capabilities instead of depending on the legacy block cache field. + *

+ * @param cacheAccessService cache access service to inspect + * @return wrapped legacy block cache when available; otherwise {@code null} + * @throws NullPointerException if {@code cacheAccessService} is {@code null} + */ + private static BlockCache unwrapSingleLegacyBlockCache(CacheAccessService cacheAccessService) { + Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); + + if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) { + return null; + } + + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) cacheAccessService; + CacheTopology topology = topologyBackedService.getTopology(); + + if (topology.getType() != CacheTopologyType.SINGLE_TIER) { + return null; + } + + Optional engine = topology.getEngine(CacheTier.SINGLE); + if (!engine.isPresent() || !(engine.get() instanceof BlockCacheBackedCacheEngine)) { + return null; + } + return ((BlockCacheBackedCacheEngine) engine.get()).getBlockCache(); + } + /** * Create a cache configuration using the specified configuration object and family descriptor. * @param conf hbase configuration diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java index 34d6c8d926b6..0c96adaeff9c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java @@ -40,4 +40,16 @@ public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize { * @throws IllegalArgumentException if the victim cache had already been set */ void setVictimCache(BlockCache victimCache); + + /** + * Removes the configured victim cache from this first-level cache. + *

+ * This method is used during the block cache migration when an existing first-level cache is + * moved under topology-backed orchestration. Legacy combined-cache construction wires L1 to L2 + * through a victim cache. Once L1 and L2 are represented as independent topology engines, the + * topology must control L2 lookup and promotion directly, so the old victim-cache delegation must + * be disabled. + *

+ */ + void unsetVictimCache(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruAdaptiveBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruAdaptiveBlockCache.java index 87932074bff1..f5048bb69bed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruAdaptiveBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruAdaptiveBlockCache.java @@ -437,6 +437,11 @@ public void setVictimCache(BlockCache victimCache) { victimHandler = requireNonNull(victimCache); } + @Override + public void unsetVictimCache() { + this.victimHandler = null; + } + @Override public void setMaxSize(long maxSize) { this.maxSize = maxSize; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 1e6205c58e5a..dcad4a1d0202 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -346,6 +346,11 @@ public void setVictimCache(BlockCache victimCache) { victimHandler = requireNonNull(victimCache); } + @Override + public void unsetVictimCache() { + this.victimHandler = null; + } + @Override public void setMaxSize(long maxSize) { this.maxSize = maxSize; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java index 7852f19bd63e..d8901e833287 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java @@ -417,4 +417,9 @@ public long getCurrentDataSize() { public long getDataBlockCount() { return getBlockCount(); } + + @Override + public void unsetVictimCache() { + this.victimCache = null; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheEngine.java index 840fd9380c0d..c4f7f9ce6116 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheEngine.java @@ -172,6 +172,11 @@ public long getCurrentDataSize() { return blockCache.getCurrentDataSize(); } + @Override + public long getCurrentSize() { + return blockCache.getCurrentSize(); + } + @Override public long getBlockCount() { return blockCache.getBlockCount(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java index a020323737a4..344cfc19b501 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; import org.apache.hadoop.hbase.io.hfile.CachedBlock; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; import org.apache.yetus.audience.InterfaceAudience; /** @@ -47,26 +48,37 @@ private CacheAccessServices() { } /** - * Creates a cache access service backed by an existing block cache. + * Creates a {@link CacheAccessService} for the supplied legacy {@link BlockCache}. *

- * For regular {@link BlockCache} implementations, this returns a legacy - * {@link BlockCacheBackedCacheAccessService}. For {@link CombinedBlockCache}, this returns a - * topology-backed service using {@link TieredExclusiveTopology}. This moves combined L1/L2 - * orchestration to the new topology layer while keeping the existing combined block cache object - * available for legacy {@link BlockCache}-facing APIs. + * All legacy block caches are adapted through {@link TopologyBackedCacheAccessService}. Plain + * single-tier block caches are represented by {@link SingleTierTopology}. Exclusive combined + * caches are represented by {@link TieredExclusiveTopology}. Inclusive combined caches are + * represented by {@link TieredInclusiveTopology}. *

- * @param blockCache block cache to expose through {@link CacheAccessService} - * @return cache access service + *

+ * {@link InclusiveCombinedBlockCache} is checked before {@link CombinedBlockCache} because the + * inclusive variant has different residency, promotion, and eviction semantics. Routing it + * through the exclusive topology would be incorrect. + *

+ * @param blockCache legacy block cache to adapt + * @return topology-backed cache access service for the supplied block cache + * @throws NullPointerException if {@code blockCache} is {@code null} */ - public static CacheAccessService fromBlockCache(BlockCache blockCache) { Objects.requireNonNull(blockCache, "blockCache must not be null"); + + if (blockCache instanceof InclusiveCombinedBlockCache) { + return TopologyBackedCacheAccessServices + .fromInclusiveCombinedBlockCache((InclusiveCombinedBlockCache) blockCache); + } + if (blockCache instanceof CombinedBlockCache) { return TopologyBackedCacheAccessServices .fromCombinedBlockCache((CombinedBlockCache) blockCache); } - return new BlockCacheBackedCacheAccessService(blockCache); + return TopologyBackedCacheAccessServices.fromSingleBlockCache("single", blockCache, + DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE); } /** @@ -80,7 +92,7 @@ public static CacheAccessService fromBlockCache(BlockCache blockCache) { * The method delegates block-cache construction to * {@link BlockCacheFactory#createBlockCache(Configuration)}. If the legacy factory creates a * {@link BlockCache}, the returned service is backed by that cache through - * {@link BlockCacheBackedCacheAccessService}. If the legacy factory does not create a cache, this + * {@link TopologyBackedCacheAccessService}. If the legacy factory does not create a cache, this * method returns the disabled/no-op cache access service. *

*

@@ -140,15 +152,6 @@ public static CacheAccessService disabled() { * @return optional iterable cached-block view * @throws NullPointerException if {@code cacheAccessService} is {@code null} */ - @SuppressWarnings("unchecked") - // public static Optional> - // asCachedBlockIterable(CacheAccessService cacheAccessService) { - // Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); - // if (cacheAccessService instanceof Iterable) { - // return Optional.of((Iterable) cacheAccessService); - // } - // return Optional.empty(); - // } public static Optional> asCachedBlockIterable(CacheAccessService service) { Objects.requireNonNull(service, "service must not be null"); @@ -156,11 +159,6 @@ public static Optional> asCachedBlockIterable(CacheAccessS if (service instanceof TopologyBackedCacheAccessService) { return ((TopologyBackedCacheAccessService) service).asCachedBlockIterable(); } - - if (service instanceof BlockCacheBackedCacheAccessService) { - return Optional.of(((BlockCacheBackedCacheAccessService) service).getBlockCache()); - } - return Optional.empty(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheEngine.java index 270cbc24e1b9..04020725b570 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheEngine.java @@ -215,6 +215,19 @@ default int evictBlocksByRegionName(String regionName) { */ long getCurrentDataSize(); + /** + * Returns the current total size of this cache engine. + *

+ * The current size may include cache metadata, allocator overhead, index structures, or other + * implementation-specific memory that is not part of the cached block payload. This value is + * intentionally distinct from {@link #getCurrentDataSize()}, which reports only cached data size. + *

+ * @return current total cache size in bytes + */ + default long getCurrentSize() { + return getCurrentDataSize(); + } + /** * Returns the total number of cached blocks. * @return total block count diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTier.java index 69f1d51a49a6..c473bb29482f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTier.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTier.java @@ -30,7 +30,12 @@ public enum CacheTier { /** - * Single-tier topology engine. + * Single cache tier. + *

+ * This tier is used by {@link SingleTierTopology}. It represents a topology with one active cache + * engine without assigning L1 or L2 semantics to that engine. A single-tier cache may be backed + * by an in-memory cache, a bucket cache, or another cache engine. + *

*/ SINGLE, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java index 50ff46d5bde7..6a8cac5032e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheTopologyType.java @@ -28,7 +28,7 @@ public enum CacheTopologyType { /** * A topology with a single cache engine. */ - SINGLE, + SINGLE_TIER, /** * A tiered topology where a block normally resides in only one tier at a time. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java index 039c620a33d3..f9137f2e642e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/DefaultHBaseCachePlacementAdmissionPolicy.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile.cache; +import java.util.Arrays; import java.util.Objects; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.BlockType; @@ -36,6 +37,9 @@ @InterfaceAudience.Private public class DefaultHBaseCachePlacementAdmissionPolicy implements CachePlacementAdmissionPolicy { + public static final DefaultHBaseCachePlacementAdmissionPolicy INSTANCE = + new DefaultHBaseCachePlacementAdmissionPolicy(); + @Override public AdmissionDecision shouldAdmit(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context, AdmissionPriority priority, CacheTopologyView topologyView) { @@ -48,6 +52,24 @@ public AdmissionDecision shouldAdmit(BlockCacheKey cacheKey, Cacheable block, return AdmissionDecision.admit(); } + /** + * Selects the cache tier or tiers for an admitted block. + *

+ * Single-tier topology has only one active cache engine and therefore selects + * {@link CacheTier#SINGLE}. Tiered inclusive topology writes to both L1 and L2 because inclusive + * residency allows the same block to be present in multiple tiers. Tiered exclusive topology + * keeps the legacy combined-cache placement behavior where data blocks and non-data blocks may be + * placed in different tiers. + *

+ * @param cacheKey cache key identifying the block + * @param block block being cached + * @param context cache write context + * @param topologyView topology view available to the placement policy + * @return tier decision for the admitted block + * @throws NullPointerException if {@code cacheKey}, {@code block}, {@code context}, or + * {@code topologyView} is {@code null} + */ + @Override public TierDecision selectTier(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context, CacheTopologyView topologyView) { @@ -55,15 +77,14 @@ public TierDecision selectTier(BlockCacheKey cacheKey, Cacheable block, CacheWri Objects.requireNonNull(block, "block must not be null"); Objects.requireNonNull(context, "context must not be null"); Objects.requireNonNull(topologyView, "topologyView must not be null"); - /* - * Default compatibility placement prefers metadata/index/bloom blocks in L1 and data blocks in - * L2 when both tiers are available, but falls back to any available tier rather than rejecting - * placement. - */ - if (topologyView.getType() == CacheTopologyType.SINGLE) { + if (topologyView.getType() == CacheTopologyType.SINGLE_TIER) { return TierDecision.single(CacheTier.SINGLE); } + if (topologyView.getType() == CacheTopologyType.TIERED_INCLUSIVE) { + return TierDecision.multiple(Arrays.asList(CacheTier.L1, CacheTier.L2)); + } + if (isMetaOrIndexBlock(block)) { if (topologyView.getEngine(CacheTier.L1).isPresent()) { return TierDecision.single(CacheTier.L1); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java deleted file mode 100644 index c94deabdae5a..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleEngineTopology.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.cache; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; -import org.apache.hadoop.hbase.io.hfile.CacheStats; -import org.apache.hadoop.hbase.io.hfile.Cacheable; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Single-engine cache topology. - *

- * This topology wraps a single {@link CacheEngine}. It is primarily useful as a baseline topology - * and as a simple bridge for cache configurations that do not use L1/L2 tiering. - *

- */ -@InterfaceAudience.Private -public class SingleEngineTopology implements CacheTopology { - - private final String name; - private final CacheEngine engine; - private final CacheTopologyView view; - - public SingleEngineTopology(String name, CacheEngine engine) { - this.name = name; - this.engine = engine; - this.view = new CacheTopologyView(this); - } - - @Override - public String getName() { - return name; - } - - @Override - public CacheTopologyType getType() { - return CacheTopologyType.SINGLE; - } - - @Override - public List getEngines() { - return Collections.singletonList(engine); - } - - @Override - public Optional getEngine(CacheTier tier) { - return tier == CacheTier.SINGLE ? Optional.of(engine) : Optional.empty(); - } - - @Override - public CacheTopologyView getView() { - return view; - } - - @Override - public CacheStats getStats() { - return engine.getStats(); - } - - @Override - public boolean promote(BlockCacheKey cacheKey, Cacheable block, CacheEngine sourceEngine, - CacheEngine targetEngine) { - return false; - } - - @Override - public void shutdown() { - engine.shutdown(); - } - - @Override - public List getTiers() { - return Collections.singletonList(CacheTier.SINGLE); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java new file mode 100644 index 000000000000..9dfec5507678 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/SingleTierTopology.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Single-tier cache topology. + *

+ * A single-tier topology contains exactly one cache engine. It is used to represent legacy + * single-tier {@code BlockCache} implementations inside the topology-backed cache access framework. + * Unlike tiered topologies, this topology does not perform tier orchestration, promotion, or + * demotion. All cache operations are directed to the single L1 engine. + *

+ *

+ * This topology allows plain block caches to use the same {@link TopologyBackedCacheAccessService} + * path as combined caches while preserving the existing cache implementation underneath. + *

+ */ +@InterfaceAudience.Private +public class SingleTierTopology implements CacheTopology { + + private final String name; + private final CacheEngine engine; + private final CacheTopologyView view; + + /** + * Creates a single-tier cache topology. + * @param name topology name used for diagnostics + * @param engine cache engine backing the single tier + */ + public SingleTierTopology(String name, CacheEngine engine) { + this.name = name; + this.engine = engine; + this.view = new CacheTopologyView(this); + } + + /** + * Returns the diagnostic name of this topology. + * @return topology name + */ + @Override + public String getName() { + return name; + } + + /** + * Returns the topology type. + * @return {@link CacheTopologyType#SINGLE_TIER} + */ + @Override + public CacheTopologyType getType() { + return CacheTopologyType.SINGLE_TIER; + } + + /** + * Returns the cache engines that participate in this topology. + * @return singleton list containing the single cache engine + */ + @Override + public List getEngines() { + return Collections.singletonList(engine); + } + + /** + * Returns the tiers available in this topology. + *

+ * A single-tier topology exposes its only engine through {@link CacheTier#SINGLE}. This avoids + * assigning L1 or L2 semantics to a cache configuration that has only one active engine. + *

+ * @return singleton list containing {@link CacheTier#SINGLE} + */ + @Override + public List getTiers() { + return Collections.singletonList(CacheTier.SINGLE); + } + + /** + * Returns the cache engine associated with the requested tier. + *

+ * The only valid tier for this topology is {@link CacheTier#SINGLE}. The single tier does not + * imply L1 or L2 behavior; it only means that the topology has one active cache engine. + *

+ * @param tier cache tier to resolve + * @return the single cache engine for {@link CacheTier#SINGLE}; otherwise + * {@link Optional#empty()} + */ + @Override + public Optional getEngine(CacheTier tier) { + switch (tier) { + case SINGLE: + return Optional.of(engine); + default: + return Optional.empty(); + } + } + + /** + * Returns the topology view associated with this topology. + * @return topology view + */ + @Override + public CacheTopologyView getView() { + return view; + } + + /** + * Returns cache statistics for the single cache engine. + * @return cache statistics exposed by the backing engine + */ + @Override + public CacheStats getStats() { + return engine.getStats(); + } + + /** + * Attempts to promote a block within this topology. + *

+ * Single-tier topology has no higher or lower tier, so promotion is not supported. The method + * returns {@code false} without modifying the cache. + *

+ * @param cacheKey cache key identifying the block + * @param block cached block + * @param sourceEngine source cache engine + * @param targetEngine target cache engine + * @return {@code false} because single-tier topology does not support promotion + */ + @Override + public boolean promote(BlockCacheKey cacheKey, Cacheable block, CacheEngine sourceEngine, + CacheEngine targetEngine) { + return false; + } + + /** + * Attempts to demote a block within this topology. + *

+ * Single-tier topology has no lower tier, so demotion is not supported. The method returns + * {@code false} without modifying the cache. + *

+ * @param cacheKey cache key identifying the block + * @param block cached block + * @param sourceEngine source cache engine + * @param targetEngine target cache engine + * @return {@code false} because single-tier topology does not support demotion + */ + @Override + public boolean demote(BlockCacheKey cacheKey, Cacheable block, CacheEngine sourceEngine, + CacheEngine targetEngine) { + return false; + } + + /** + * Shuts down the single cache engine. + */ + @Override + public void shutdown() { + engine.shutdown(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java index 828368274e43..f98ddc6f262f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java @@ -18,12 +18,14 @@ package org.apache.hadoop.hbase.io.hfile.cache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; @@ -48,19 +50,13 @@ * operations. *

*

- * This class is the topology-backed counterpart to {@link BlockCacheBackedCacheAccessService}. The - * block-cache-backed implementation is useful for incremental migration with no behavior change. - * This implementation is useful once callers are ready to exercise the new topology and engine - * abstractions directly through {@link CacheAccessService}. - *

- *

* Representation selection is intentionally not invoked by this initial implementation. Until the * service can actually apply representation decisions safely, especially around HFileBlock * lifecycle and packed/unpacked storage, representation policy is left to a later integration step. *

*/ @InterfaceAudience.Private -public class TopologyBackedCacheAccessService implements CacheAccessService { +public class TopologyBackedCacheAccessService implements CacheAccessService, Iterable { private final CacheTopology topology; private final CachePlacementAdmissionPolicy policy; @@ -130,7 +126,6 @@ public String getName() { * @return cached block, or {@code null} if not present in any tier */ @Override - public Cacheable getBlock(BlockCacheKey cacheKey, CacheRequestContext context) { Objects.requireNonNull(cacheKey, "cacheKey must not be null"); Objects.requireNonNull(context, "context must not be null"); @@ -214,23 +209,79 @@ private void updateBlockMetrics(Cacheable block, BlockCacheKey key, CacheEngine } /** - * Adds a block to the cache using policy-selected target tiers. + * Caches a block using the topology-backed cache access service. *

- * This method first asks the configured policy whether the block should be admitted. If admitted, - * the policy selects the target tier or tiers. The block is then inserted into each selected - * engine using {@link CacheEngine#cacheBlock(BlockCacheKey, Cacheable, boolean, boolean)}. + * Single-tier topology preserves the legacy direct block-cache behavior by writing admitted + * blocks to the only backing engine. Tiered topologies use the placement policy to select one or + * more target tiers. *

+ * @param cacheKey cache key identifying the block + * @param block block to cache + * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} + */ + @Override + public void cacheBlock(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context) { + Objects.requireNonNull(cacheKey, "cacheKey must not be null"); + Objects.requireNonNull(block, "block must not be null"); + Objects.requireNonNull(context, "context must not be null"); + + if (topology.getType() == CacheTopologyType.SINGLE_TIER) { + cacheBlockToSingleTier(cacheKey, block, context); + return; + } + + cacheBlockToSelectedTiers(cacheKey, block, context); + } + + /** + * Caches a block into the only engine in a single-tier topology. *

- * The policy's representation decision is intentionally not applied in this initial - * implementation. The current block object is passed through unchanged. + * This method preserves the behavior of the legacy {@link BlockCacheBackedCacheAccessService} + * path. A single-tier topology has no placement decision to make: admitted blocks are written to + * the only available engine, which is exposed as {@link CacheTier#SINGLE}. *

- * @param cacheKey block cache key - * @param block block contents + * @param cacheKey cache key identifying the block + * @param block block to cache * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} */ + private void cacheBlockToSingleTier(BlockCacheKey cacheKey, Cacheable block, + CacheWriteContext context) { + Objects.requireNonNull(cacheKey, "cacheKey must not be null"); + Objects.requireNonNull(block, "block must not be null"); + Objects.requireNonNull(context, "context must not be null"); - @Override - public void cacheBlock(BlockCacheKey cacheKey, Cacheable block, CacheWriteContext context) { + AdmissionDecision admission = + policy.shouldAdmit(cacheKey, block, context, AdmissionPriority.NORMAL, topologyView); + if (!admission.isAdmitted()) { + return; + } + + Optional engine = topology.getEngine(CacheTier.SINGLE); + if (!engine.isPresent()) { + return; + } + + engine.get().cacheBlock(cacheKey, block, context.isInMemory(), context.isWaitWhenCache()); + } + + /** + * Caches a block into the tiers selected by the placement policy. + *

+ * This method is used for tiered topologies where the placement policy decides whether the block + * belongs in L1, L2, or multiple tiers. + *

+ * @param cacheKey cache key identifying the block + * @param block block to cache + * @param context cache write context + * @throws NullPointerException if {@code cacheKey}, {@code block}, or {@code context} is + * {@code null} + */ + private void cacheBlockToSelectedTiers(BlockCacheKey cacheKey, Cacheable block, + CacheWriteContext context) { Objects.requireNonNull(cacheKey, "cacheKey must not be null"); Objects.requireNonNull(block, "block must not be null"); Objects.requireNonNull(context, "context must not be null"); @@ -386,12 +437,23 @@ public long size() { } /** - * Returns aggregate occupied size of the block cache, in bytes. - * @return occupied space in cache, in bytes + * Returns the current total size of all cache engines in this topology. + *

+ * This method aggregates {@link CacheEngine#getCurrentSize()} rather than + * {@link CacheEngine#getCurrentDataSize()} so topology-backed diagnostics preserve legacy + * {@link BlockCache#getCurrentSize()} semantics. For cache implementations such as + * {@code LruBlockCache}, current size may include metadata and cache overhead in addition to + * cached data bytes. + *

+ * @return aggregate current total cache size in bytes */ @Override public long getCurrentSize() { - return getCurrentDataSize(); + long currentSize = 0L; + for (CacheEngine engine : topology.getEngines()) { + currentSize += engine.getCurrentSize(); + } + return currentSize; } /** @@ -626,6 +688,22 @@ public void notifyFileCachingCompleted(Path path, int blockCount, int dataBlockC } } + /** + * Returns an iterable view over cached blocks exposed by the cache engines in this topology. + *

+ * The returned iterable aggregates cached-block iterables from all engines that support this + * diagnostic capability. Engines that do not expose cached-block iteration are skipped. If no + * engine supports cached-block iteration, this method returns {@link Optional#empty()}. + *

+ *

+ * The aggregate iterable uses each underlying iterable's {@link Iterable#iterator()} method + * instead of {@link Iterable#spliterator()}. This is intentional because some legacy cache + * implementations and Mockito-based test doubles expose iteration through {@code iterator()} but + * may not provide a usable {@code spliterator()}. + *

+ * @return an aggregated cached-block iterable when at least one engine supports this capability; + * otherwise {@link Optional#empty()} + */ public Optional> asCachedBlockIterable() { List> iterables = new ArrayList<>(); @@ -637,9 +715,32 @@ public Optional> asCachedBlockIterable() { return Optional.empty(); } - Iterable cachedBlocks = () -> iterables.stream() - .flatMap(iterable -> StreamSupport.stream(iterable.spliterator(), false)).iterator(); + Iterable cachedBlocks = () -> new Iterator() { + private final Iterator> iterableIterator = iterables.iterator(); + private Iterator currentIterator = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + while (!currentIterator.hasNext() && iterableIterator.hasNext()) { + currentIterator = iterableIterator.next().iterator(); + } + return currentIterator.hasNext(); + } + + @Override + public CachedBlock next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return currentIterator.next(); + } + }; return Optional.of(cachedBlocks); } + + @Override + public Iterator iterator() { + return asCachedBlockIterable().orElse(Collections.emptyList()).iterator(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java index 3ce586c22a78..22810f526607 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessServices.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; import org.apache.hadoop.hbase.io.hfile.FirstLevelBlockCache; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; import org.apache.yetus.audience.InterfaceAudience; /** @@ -103,31 +104,189 @@ public static TopologyBackedCacheAccessService fromTieredExclusiveBlockCaches(St Objects.requireNonNull(l1, "l1 must not be null"); Objects.requireNonNull(l2, "l2 must not be null"); Objects.requireNonNull(policy, "policy must not be null"); - wireVictimCache(l1, l2); - CacheEngine l1Engine = CacheEngines.fromBlockCache(l1); + if (l1 instanceof FirstLevelBlockCache) { + ((FirstLevelBlockCache) l1).unsetVictimCache(); + } + CacheEngine l1Engine = CacheEngines.fromBlockCache(l1);// fromL1BlockCache(l1); CacheEngine l2Engine = CacheEngines.fromBlockCache(l2); CacheTopology topology = new TieredExclusiveTopology(name, l1Engine, l2Engine); return new TopologyBackedCacheAccessService(topology, policy); } /** - * Configures the legacy L1 to L2 victim-cache relationship used by CombinedBlockCache. + * Creates a topology-backed cache access service for an {@link InclusiveCombinedBlockCache}. + *

+ * The inclusive combined cache must expose exactly two non-null legacy block caches. The first + * cache is adapted as L1 using a non-victim-delegating engine, and the second cache is adapted as + * L2. This prevents L1 misses from internally consulting L2 through the legacy victim-cache + * mechanism and lets the topology-backed service control tier lookup and promotion policy. + *

+ * @param combinedBlockCache inclusive combined block cache to adapt + * @return topology-backed cache access service using a tiered inclusive topology + * @throws NullPointerException if {@code combinedBlockCache} is {@code null} + * @throws IllegalArgumentException if the combined cache does not expose exactly two non-null + * block caches + */ + public static TopologyBackedCacheAccessService + fromInclusiveCombinedBlockCache(InclusiveCombinedBlockCache combinedBlockCache) { + Objects.requireNonNull(combinedBlockCache, "combinedBlockCache must not be null"); + + BlockCache[] blockCaches = combinedBlockCache.getBlockCaches(); + if (blockCaches == null || blockCaches.length != 2) { + throw new IllegalArgumentException( + "InclusiveCombinedBlockCache must expose exactly two block caches"); + } + if (blockCaches[0] == null || blockCaches[1] == null) { + throw new IllegalArgumentException( + "InclusiveCombinedBlockCache must expose non-null L1 and L2 block caches"); + } + + return fromTieredInclusiveBlockCaches("inclusive-combined", blockCaches[0], blockCaches[1], + DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE); + } + + /** + * Creates a topology-backed cache access service from two legacy block caches using an inclusive + * tiered topology. *

- * The topology-backed service owns lookup and placement orchestration, but existing - * {@link FirstLevelBlockCache} implementations still use a direct victim-cache reference to move - * evicted blocks from L1 to L2. Keep this wiring while L1 and L2 are still legacy - * {@link BlockCache} implementations. + * The first supplied block cache is treated as the L1 tier and the second supplied block cache is + * treated as the L2 tier. Both legacy caches are adapted to {@link CacheEngine} instances using + * {@link CacheEngines#fromBlockCache(BlockCache)} and then assembled into a + * {@link TieredInclusiveTopology}. *

- * @param l1 first-level block cache - * @param l2 second-level block cache + *

+ * This helper is intended for compatibility with legacy inclusive combined-cache configurations + * while moving cache access and diagnostics to the {@link CacheAccessService} abstraction. + * Inclusive topology semantics differ from exclusive topology semantics: a block may exist in + * both tiers, and eviction from one tier does not necessarily imply eviction from the other tier. + *

+ * @param name topology name used for diagnostics + * @param l1 first-level block cache + * @param l2 second-level block cache + * @param policy cache placement and admission policy to use with the topology-backed service + * @return topology-backed cache access service backed by a tiered inclusive topology + * @throws NullPointerException if {@code name}, {@code l1}, {@code l2}, or {@code policy} is + * {@code null} */ - private static void wireVictimCache(BlockCache l1, BlockCache l2) { + public static TopologyBackedCacheAccessService fromTieredInclusiveBlockCaches(String name, + BlockCache l1, BlockCache l2, CachePlacementAdmissionPolicy policy) { + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(l1, "l1 must not be null"); + Objects.requireNonNull(l2, "l2 must not be null"); + Objects.requireNonNull(policy, "policy must not be null"); if (l1 instanceof FirstLevelBlockCache) { - try { - ((FirstLevelBlockCache) l1).setVictimCache(l2); - } catch (IllegalArgumentException e) { - // ignore if already wired - } + ((FirstLevelBlockCache) l1).unsetVictimCache(); + } + CacheEngine l1Engine = CacheEngines.fromBlockCache(l1); + CacheEngine l2Engine = CacheEngines.fromBlockCache(l2); + CacheTopology topology = new TieredInclusiveTopology(name, l1Engine, l2Engine); + + return new TopologyBackedCacheAccessService(topology, policy); + } + + /** + * Creates a topology-backed cache access service for a single legacy {@link BlockCache}. + *

+ * The supplied block cache is adapted to a {@link CacheEngine} and placed behind a + * {@link SingleTierTopology}. This makes single-tier caches use the same + * {@link TopologyBackedCacheAccessService} path as combined caches while preserving the existing + * block cache implementation underneath. + *

+ * @param name topology name used for diagnostics + * @param blockCache legacy block cache to adapt + * @param policy cache placement and admission policy + * @return topology-backed cache access service backed by a single-tier topology + * @throws NullPointerException if {@code name}, {@code blockCache}, or {@code policy} is + * {@code null} + */ + public static TopologyBackedCacheAccessService fromSingleBlockCache(String name, + BlockCache blockCache, CachePlacementAdmissionPolicy policy) { + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(blockCache, "blockCache must not be null"); + Objects.requireNonNull(policy, "policy must not be null"); + + CacheEngine engine = CacheEngines.fromBlockCache(blockCache); + CacheTopology topology = new SingleTierTopology(name, engine); + return new TopologyBackedCacheAccessService(topology, policy); + } + + /** + * Returns the legacy {@link BlockCache} wrapped by the cache engine for the requested tier. + *

+ * This helper is intended for tests that need to verify compatibility with legacy block cache + * implementations during the migration to topology-backed cache access. Production code should + * prefer {@link CacheAccessService} capability methods instead of unwrapping the underlying + * {@link BlockCache}. + *

+ *

+ * The supplied service must be a {@link TopologyBackedCacheAccessService}. The requested tier + * must resolve to a {@link BlockCacheBackedCacheEngine}. If either condition is not true, this + * method fails fast with an {@link IllegalArgumentException}. + *

+ * @param cacheAccessService cache access service to inspect + * @param tier cache tier to unwrap + * @return legacy block cache wrapped by the cache engine for the requested tier + * @throws NullPointerException if {@code cacheAccessService} or {@code tier} is {@code null} + * @throws IllegalArgumentException if the service is not topology-backed, if the requested tier + * is not present, or if the tier is not backed by a + * {@link BlockCacheBackedCacheEngine} + */ + public static BlockCache getBlockCache(CacheAccessService cacheAccessService, CacheTier tier) { + Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); + Objects.requireNonNull(tier, "tier must not be null"); + + if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) { + throw new IllegalArgumentException( + "cacheAccessService must be a TopologyBackedCacheAccessService"); + } + + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) cacheAccessService; + CacheTopology topology = topologyBackedService.getTopology(); + + CacheEngine engine = topology.getEngine(tier) + .orElseThrow(() -> new IllegalArgumentException("No cache engine found for tier " + tier)); + + if (!(engine instanceof BlockCacheBackedCacheEngine)) { + throw new IllegalArgumentException( + "Cache engine for tier " + tier + " must be a BlockCacheBackedCacheEngine"); + } + + return ((BlockCacheBackedCacheEngine) engine).getBlockCache(); + } + + /** + * Returns the legacy {@link BlockCache} wrapped by a single-tier topology-backed cache access + * service. + *

+ * Single-tier topology exposes its only engine through {@link CacheTier#SINGLE}. The only active + * engine is not assumed to be L1 or L2 because a single-tier configuration may be backed by + * different concrete cache implementations, including bucket cache. + *

+ * @param cacheAccessService cache access service to inspect + * @return legacy block cache wrapped by the single-tier cache engine + * @throws NullPointerException if {@code cacheAccessService} is {@code null} + * @throws IllegalArgumentException if the supplied service is not a topology-backed single-tier + * cache service or if the single tier is not backed by a + * {@link BlockCacheBackedCacheEngine} + */ + public static BlockCache getBlockCache(CacheAccessService cacheAccessService) { + Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); + + if (!(cacheAccessService instanceof TopologyBackedCacheAccessService)) { + throw new IllegalArgumentException( + "cacheAccessService must be a TopologyBackedCacheAccessService"); } + + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) cacheAccessService; + CacheTopology topology = topologyBackedService.getTopology(); + + if (topology.getType() != CacheTopologyType.SINGLE_TIER) { + throw new IllegalArgumentException( + "cacheAccessService must be backed by a single-tier topology"); + } + + return getBlockCache(cacheAccessService, CacheTier.SINGLE); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index 1d7286dbe6c6..2aa5a8d69912 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -43,10 +43,11 @@ import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory; import org.apache.hadoop.hbase.io.hfile.cache.NoOpCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessService; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessServices; import org.apache.hadoop.hbase.io.util.MemorySizeUtil; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.testclassification.IOTests; @@ -314,7 +315,6 @@ private void doBucketCacheConfigTest() { // TODO: Assert sizes allocated are right and proportions. LruBlockCache lbc = (LruBlockCache) CacheAccessServiceTestFactory.getFirstLevelBlockCache(service); - ; assertEquals(MemorySizeUtil.getOnHeapCacheSize(this.conf), lbc.getMaxSize()); BucketCache bc = (BucketCache) CacheAccessServiceTestFactory.getSecondLevelBlockCache(service); // getMaxSize comes back in bytes but we specified size in MB @@ -381,19 +381,23 @@ public void testBucketCacheConfigL1L2Setup() throws Exception { BlockCache bc = CacheAccessServiceTestFactory.getSecondLevelBlockCache(service); // getMaxSize comes back in bytes but we specified size in MB assertEquals(bcExpectedSize, ((BucketCache) bc).getMaxSize()); - // Test the L1+L2 deploy works as we'd expect with blocks evicted from L1 going to L2. + /* + * The topology-backed cache path intentionally clears legacy L1 victim-cache wiring when L1 and + * L2 are adapted as independent topology engines. Direct calls to the unwrapped L1 cache should + * therefore not be used to verify L1-to-L2 victim movement. Tier placement, promotion, and + * lookup are now owned by TopologyBackedCacheAccessService. + */ long initialL1BlockCount = lbc.getBlockCount(); long initialL2BlockCount = bc.getBlockCount(); Cacheable c = new DataCacheEntry(); BlockCacheKey bck = new BlockCacheKey("bck", 0); + lbc.cacheBlock(bck, c, false); + assertEquals(initialL1BlockCount + 1, lbc.getBlockCount()); assertEquals(initialL2BlockCount, bc.getBlockCount()); - assertNotNull(lbc.getBlock(bck, true, false, true)); assertNull(bc.getBlock(bck, true, false, true)); - waitForAnyBlockToMoveFromL1ToL2(lbc, bc, initialL2BlockCount); - assertTrue(bc.getBlockCount() > initialL2BlockCount); } /** @@ -491,8 +495,8 @@ void testCacheAccessServiceBackedByBlockCacheWhenBlockCacheIsConfigured() { CacheAccessService service = cacheConfig.getCacheAccessService(); - assertInstanceOf(BlockCacheBackedCacheAccessService.class, service); - assertSame(blockCache, ((BlockCacheBackedCacheAccessService) service).getBlockCache()); + assertInstanceOf(TopologyBackedCacheAccessService.class, service); + assertSame(blockCache, TopologyBackedCacheAccessServices.getBlockCache(service)); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index a016cbef03ab..75a03d425042 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -50,10 +50,10 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory; import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices; +import org.apache.hadoop.hbase.io.hfile.cache.TopologyBackedCacheAccessServices; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -202,7 +202,7 @@ public static Stream parameters() throws IOException { private void clearBlockCache(CacheAccessService cache) throws InterruptedException { // TODO: HBASE-30018 refactor later - BlockCache blockCache = ((BlockCacheBackedCacheAccessService) cache).getBlockCache(); + BlockCache blockCache = TopologyBackedCacheAccessServices.getBlockCache(cache); if (blockCache instanceof LruBlockCache) { ((LruBlockCache) blockCache).clearCache(); } else { @@ -240,7 +240,7 @@ public void setUp() throws IOException { cowType.shouldBeCached(BlockType.LEAF_INDEX)); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.BLOOM_CHUNK)); - cacheConf = new CacheConfig(conf, ((BlockCacheBackedCacheAccessService) cache).getBlockCache()); + cacheConf = new CacheConfig(conf, TopologyBackedCacheAccessServices.getBlockCache(cache)); fs = HFileSystem.get(conf); } @@ -442,7 +442,7 @@ private void testCachingDataBlocksDuringCompactionInternals(boolean useTags, .setCompressionType(compress).setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions) .setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()).build(); HRegion region = TEST_UTIL.createTestRegion(table, cfd, - ((BlockCacheBackedCacheAccessService) cache).getBlockCache()); + TopologyBackedCacheAccessServices.getBlockCache(cache)); int rowIdx = 0; long ts = EnvironmentEdgeManager.currentTime(); for (int iFile = 0; iFile < 5; ++iFile) { @@ -500,7 +500,7 @@ private void testCachingDataBlocksDuringCompactionInternals(boolean useTags, // of testing // BucketCache, we cannot verify block type as it is not stored in the cache. boolean cacheOnCompactAndNonBucketCache = cacheBlocksOnCompaction - && !(((BlockCacheBackedCacheAccessService) cache).getBlockCache() instanceof BucketCache); + && !(TopologyBackedCacheAccessServices.getBlockCache(cache) instanceof BucketCache); String assertErrorMessage = "\nTest description: " + testDescription + "\ncacheBlocksOnCompaction: " + cacheBlocksOnCompaction + "\n"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java index 37dbc0a92278..dc31fab4607f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java @@ -649,12 +649,6 @@ public static CacheAccessServiceTestInstance bucketInstance(String * direct access to the underlying legacy {@link BlockCache} for implementation-specific * assertions, cached-block iteration, metrics inspection, or other diagnostic checks. *

- *

- * Only {@link BlockCacheBackedCacheAccessService} is supported. Services backed by future - * topology/cache-engine implementations are not required to expose a legacy {@link BlockCache}. - * Tests that use this method should therefore be treated as compatibility tests, not as tests of - * the final pluggable-cache architecture. - *

* @param cacheAccessService cache access service * @return backing legacy block cache * @throws NullPointerException if {@code cacheAccessService} is {@code null} @@ -663,11 +657,7 @@ public static CacheAccessServiceTestInstance bucketInstance(String */ public static BlockCache blockCache(CacheAccessService cacheAccessService) { Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be null"); - if (cacheAccessService instanceof BlockCacheBackedCacheAccessService) { - return ((BlockCacheBackedCacheAccessService) cacheAccessService).getBlockCache(); - } - throw new IllegalArgumentException("CacheAccessService is not backed by a legacy BlockCache: " - + cacheAccessService.getClass().getName()); + return TopologyBackedCacheAccessServices.getBlockCache(cacheAccessService); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java deleted file mode 100644 index 8a7378c7b837..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestBlockCacheBackedCacheAccessService.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.cache; - -import static org.junit.Assert.assertThrows; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.Optional; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; -import org.apache.hadoop.hbase.io.hfile.BlockType; -import org.apache.hadoop.hbase.io.hfile.CacheStats; -import org.apache.hadoop.hbase.io.hfile.Cacheable; -import org.apache.hadoop.hbase.io.hfile.HFileBlock; -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -/** - * Tests for {@link BlockCacheBackedCacheAccessService} and related service helpers. - */ -@Tag(IOTests.TAG) -@Tag(SmallTests.TAG) -public class TestBlockCacheBackedCacheAccessService { - - private static final String HFILE_NAME = "file"; - - private static final long BLOCK_OFFSET = 1L; - - private static final long RANGE_START_OFFSET = 1L; - - private static final long RANGE_END_OFFSET = 10L; - - /** - * Verifies that context-based lookup delegates to the block-type aware legacy lookup method. - */ - @Test - void testGetBlockWithBlockTypeDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - when(blockCache.getBlock(key, true, true, false, BlockType.DATA)).thenReturn(block); - - CacheRequestContext context = CacheRequestContext.newBuilder().withCaching(true) - .withRepeat(true).withUpdateCacheMetrics(false).withBlockType(BlockType.DATA).build(); - - assertSame(block, service.getBlock(key, context)); - verify(blockCache).getBlock(key, true, true, false, BlockType.DATA); - } - - /** - * Verifies that context-based lookup delegates to the legacy lookup method without block type. - */ - @Test - void testGetBlockWithoutBlockTypeDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - when(blockCache.getBlock(key, true, false, true)).thenReturn(block); - - CacheRequestContext context = CacheRequestContext.newBuilder().withCaching(true) - .withRepeat(false).withUpdateCacheMetrics(true).build(); - - assertSame(block, service.getBlock(key, context)); - verify(blockCache).getBlock(key, true, false, true); - } - - /** - * Verifies that context-based insertion delegates in-memory and wait flags correctly. - */ - @Test - void testCacheBlockDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Cacheable block = mock(Cacheable.class); - - CacheWriteContext context = CacheWriteContext.newBuilder().withInMemory(true) - .withWaitWhenCache(true).withSource(CacheWriteSource.READ_MISS).build(); - - service.cacheBlock(key, block, context); - - verify(blockCache).cacheBlock(key, block, true, true); - } - - /** - * Verifies that invalidation methods delegate to the wrapped block cache. - */ - @Test - void testEvictionDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - - when(blockCache.evictBlock(key)).thenReturn(true); - when(blockCache.evictBlocksByHfileName(HFILE_NAME)).thenReturn(3); - when(blockCache.evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, RANGE_END_OFFSET)) - .thenReturn(2); - - assertTrue(service.evictBlock(key)); - assertEquals(3, service.evictBlocksByHfileName(HFILE_NAME)); - assertEquals(2, - service.evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, RANGE_END_OFFSET)); - - verify(blockCache).evictBlock(key); - verify(blockCache).evictBlocksByHfileName(HFILE_NAME); - verify(blockCache).evictBlocksRangeByHfileName(HFILE_NAME, RANGE_START_OFFSET, - RANGE_END_OFFSET); - } - - /** - * Verifies that stats, sizing, lifecycle, and optional helpers delegate to the wrapped cache. - */ - @Test - void testStatsSizingLifecycleAndHelpersDelegateToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - CacheStats stats = new CacheStats("test"); - HFileBlock hfileBlock = mock(HFileBlock.class); - BlockCacheKey key = new BlockCacheKey(HFILE_NAME, BLOCK_OFFSET); - Configuration conf = new Configuration(false); - - when(blockCache.getStats()).thenReturn(stats); - when(blockCache.getMaxSize()).thenReturn(100L); - when(blockCache.getFreeSize()).thenReturn(40L); - when(blockCache.size()).thenReturn(60L); - when(blockCache.getCurrentDataSize()).thenReturn(50L); - when(blockCache.getBlockCount()).thenReturn(10L); - when(blockCache.getDataBlockCount()).thenReturn(8L); - when(blockCache.blockFitsIntoTheCache(hfileBlock)).thenReturn(Optional.of(true)); - when(blockCache.isAlreadyCached(key)).thenReturn(Optional.of(false)); - when(blockCache.getBlockSize(key)).thenReturn(Optional.of(123)); - when(blockCache.isCacheEnabled()).thenReturn(true); - when(blockCache.waitForCacheInitialization(500L)).thenReturn(true); - - assertSame(stats, service.getStats()); - assertEquals(100L, service.getMaxSize()); - assertEquals(40L, service.getFreeSize()); - assertEquals(60L, service.size()); - assertEquals(50L, service.getCurrentDataSize()); - assertEquals(10L, service.getBlockCount()); - assertEquals(8L, service.getDataBlockCount()); - assertEquals(Optional.of(true), service.blockFitsIntoTheCache(hfileBlock)); - assertEquals(Optional.of(false), service.isAlreadyCached(key)); - assertEquals(Optional.of(123), service.getBlockSize(key)); - assertTrue(service.isCacheEnabled()); - assertTrue(service.waitForCacheInitialization(500L)); - - service.onConfigurationChange(conf); - service.shutdown(); - - verify(blockCache).onConfigurationChange(conf); - verify(blockCache).shutdown(); - } - - /** - * Verifies factory helper methods. - */ - @Test - void testCacheAccessServicesFactoryMethods() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); - - assertInstanceOf(BlockCacheBackedCacheAccessService.class, service); - assertSame(blockCache, ((BlockCacheBackedCacheAccessService) service).getBlockCache()); - assertInstanceOf(NoOpCacheAccessService.class, CacheAccessServices.disabled()); - } - - /** - * Verifies disabled-cache behavior. - */ - @Test - void testNoOpCacheAccessService() { - CacheAccessService service = new NoOpCacheAccessService(new CacheStats("noop")); - - assertEquals("NoOpCacheAccessService", service.getName()); - assertNull(service.getBlock(mock(BlockCacheKey.class), mock(CacheRequestContext.class))); - service.cacheBlock(mock(BlockCacheKey.class), mock(Cacheable.class), - mock(CacheWriteContext.class)); - assertFalse(service.evictBlock(mock(BlockCacheKey.class))); - assertEquals(0, service.evictBlocksByHfileName(HFILE_NAME)); - assertEquals(0L, service.getMaxSize()); - assertEquals(0L, service.getFreeSize()); - assertEquals(0L, service.size()); - assertEquals(0L, service.getCurrentDataSize()); - assertEquals(0L, service.getBlockCount()); - assertEquals(0L, service.getDataBlockCount()); - assertFalse(service.isCacheEnabled()); - assertFalse(service.waitForCacheInitialization(1L)); - service.onConfigurationChange(new Configuration(false)); - service.shutdown(); - } - - @Test - void testNotifyFileCachingCompletedDelegatesToBlockCache() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - Path fileName = new Path("/hbase/table/region/family/file"); - int totalBlockCount = 10; - int dataBlockCount = 8; - long size = 1024L; - - service.notifyFileCachingCompleted(fileName, totalBlockCount, dataBlockCount, size); - - verify(blockCache).notifyFileCachingCompleted(fileName, totalBlockCount, dataBlockCount, size); - } - - @Test - void testNotifyFileCachingCompletedRejectsNullPath() { - BlockCache blockCache = mock(BlockCache.class); - CacheAccessService service = new BlockCacheBackedCacheAccessService(blockCache); - - assertThrows(NullPointerException.class, - () -> service.notifyFileCachingCompleted(null, 10, 8, 1024L)); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java index 1c548b9f9304..65dff577bf3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCacheAccessServices.java @@ -39,7 +39,7 @@ public class TestCacheAccessServices { void testFromBlockCacheCreatesBlockCacheBackedServiceForRegularBlockCache() { BlockCache blockCache = mock(BlockCache.class); CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); - assertTrue(service instanceof BlockCacheBackedCacheAccessService); + assertTrue(service instanceof TopologyBackedCacheAccessService); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java index 209ceeae2028..c986fc7fa75a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.hadoop.hbase.io.hfile.FirstLevelBlockCache; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.jupiter.api.Tag; @@ -207,6 +208,50 @@ void testShutdownShutsDownBothTiers() { verify(l2).shutdown(); } + /** + * Verifies that topology-backed current-size reporting aggregates current size across tiers. + *

+ * Current size includes implementation-specific overhead and is distinct from current data size. + * The topology-backed service should therefore aggregate {@link CacheEngine#getCurrentSize()} + * from all engines rather than using data-size counters. + *

+ */ + @Test + void testCurrentSizeAggregatesAllTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + when(l1.getCurrentSize()).thenReturn(100L); + when(l2.getCurrentSize()).thenReturn(200L); + when(l1.getCurrentDataSize()).thenReturn(10L); + when(l2.getCurrentDataSize()).thenReturn(20L); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertEquals(300L, service.getCurrentSize()); + assertEquals(30L, service.getCurrentDataSize()); + } + + /** + * Verifies that tiered topology construction disables legacy L1 victim-cache delegation. + *

+ * Legacy combined-cache construction wires the first-level cache to the second-level cache + * through a victim cache. Once the caches are adapted as independent topology engines, the + * topology-backed service must control L2 lookup directly. The factory therefore removes the + * legacy victim-cache wiring before adapting L1. + *

+ */ + @Test + void testTieredExclusiveFactoryUnsetsL1VictimCache() { + FirstLevelBlockCache l1 = mock(FirstLevelBlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + TopologyBackedCacheAccessServices.fromTieredExclusiveBlockCaches("combined", l1, l2, + noPromotionPolicy()); + + verify(l1).unsetVictimCache(); + } + private static TopologyBackedCacheAccessService service(BlockCache l1, BlockCache l2, CachePlacementAdmissionPolicy policy) { return TopologyBackedCacheAccessServices.fromTieredExclusiveBlockCaches("combined", l1, l2, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java new file mode 100644 index 000000000000..a0a5185128a8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.hadoop.hbase.io.hfile.CachedBlock; +import org.apache.hadoop.hbase.io.hfile.FirstLevelBlockCache; +import org.apache.hadoop.hbase.io.hfile.InclusiveCombinedBlockCache; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) +public class TestInclusiveCombinedBlockCacheCompatibleTopologyBackedCacheAccessService { + + /** + * Verifies that {@link InclusiveCombinedBlockCache} is routed to a topology-backed cache access + * service using {@link CacheTopologyType#TIERED_INCLUSIVE}. + *

+ * This protects against accidentally routing {@link InclusiveCombinedBlockCache} through the + * exclusive combined-cache topology path. Inclusive and exclusive combined caches have different + * residency, promotion, and eviction semantics, so they must be represented by different topology + * types. + *

+ */ + @Test + void testInclusiveCombinedBlockCacheUsesTieredInclusiveTopology() { + InclusiveCombinedBlockCache combinedBlockCache = mock(InclusiveCombinedBlockCache.class); + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + when(combinedBlockCache.getBlockCaches()).thenReturn(new BlockCache[] { l1, l2 }); + + CacheAccessService service = CacheAccessServices.fromBlockCache(combinedBlockCache); + + assertTrue(service instanceof TopologyBackedCacheAccessService); + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) service; + assertEquals(CacheTopologyType.TIERED_INCLUSIVE, topologyBackedService.getTopology().getType()); + } + + /** + * Verifies that inclusive topology lookup checks L1 first and returns the L1 block when it is + * present. + *

+ * Inclusive caches prefer the first-level cache for reads. If a block is found in L1, the + * topology-backed service should return it without consulting L2. + *

+ */ + @Test + void testInclusiveL1HitReturnsBlockWithoutCheckingL2() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2, never()).getBlock(any(), anyBoolean(), anyBoolean(), anyBoolean()); + } + + /** + * Verifies that inclusive topology lookup checks L2 after an L1 miss. + *

+ * Unlike the exclusive combined-cache compatibility path, inclusive lookup does not need the L1 + * membership shortcut. A normal ordered tier scan is appropriate: L1 is checked first, and L2 is + * checked only if L1 does not contain the block. + *

+ */ + @Test + void testInclusiveL2HitReturnsBlockAfterL1Miss() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(null); + when(l2.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2).getBlock(key, true, false, true); + } + + /** + * Verifies that promotion in an inclusive topology copies a block to L1 without evicting it from + * L2. + *

+ * This is the key semantic difference from the exclusive topology. In an exclusive topology, + * promotion moves the block from L2 to L1 and removes the L2 copy. In an inclusive topology, the + * block may remain resident in both tiers. + *

+ */ + @Test + void testInclusivePromotionCopiesBlockToL1WithoutEvictingL2() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(null); + when(l2.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(l1, l2, promoteL2HitToL1Policy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).getBlock(key, true, false, true); + verify(l2).getBlock(key, true, false, true); + verify(l1).cacheBlock(key, block); + verify(l2, never()).evictBlock(key); + } + + /** + * Verifies that service-level eviction for an inclusive topology evicts from all tiers. + *

+ * An inclusive cache may contain the same block in both L1 and L2. Evicting only the first + * matching tier could leave another resident copy behind, so the topology-backed service must ask + * both tiers to evict the key. + *

+ */ + @Test + void testInclusiveEvictBlockEvictsFromBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(l1.evictBlock(key)).thenReturn(true); + when(l2.evictBlock(key)).thenReturn(true); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + assertTrue(service.evictBlock(key)); + + verify(l1).evictBlock(key); + verify(l2).evictBlock(key); + } + + /** + * Verifies that an inclusive topology can cache a block into both tiers when the placement policy + * selects both L1 and L2. + *

+ * This covers the inclusive cache residency model where the same block can be intentionally + * present in multiple tiers. + *

+ */ + @Test + void testInclusiveCacheBlockToBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = + service(l1, l2, admitToTiersPolicy(CacheTier.L1, CacheTier.L2)); + + service.cacheBlock(key, block, writeContext()); + + verify(l1).cacheBlock(key, block, false, false); + verify(l2).cacheBlock(key, block, false, false); + } + + /** + * Verifies that cached-block iteration is aggregated across both tiers for inclusive topology. + *

+ * Diagnostic code and compatibility tests use + * {@link CacheAccessServices#asCachedBlockIterable(CacheAccessService)} to enumerate cached + * blocks through the active cache access service. Since an inclusive topology has multiple + * backing engines, the service must expose cached blocks from both L1 and L2. + *

+ */ + @Test + void testInclusiveCachedBlockIterableAggregatesBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + CachedBlock l1Block = mock(CachedBlock.class); + CachedBlock l2Block = mock(CachedBlock.class); + + when(l1.iterator()).thenReturn(Arrays.asList(l1Block).iterator()); + when(l2.iterator()).thenReturn(Arrays.asList(l2Block).iterator()); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + Optional> iterable = CacheAccessServices.asCachedBlockIterable(service); + + assertTrue(iterable.isPresent()); + assertEquals(Arrays.asList(l1Block, l2Block), toList(iterable.get())); + } + + /** + * Verifies that shutting down an inclusive topology-backed service shuts down both cache tiers. + *

+ * The topology-backed service owns the topology-level lifecycle. For a two-tier inclusive + * topology, shutdown should be delegated to both L1 and L2 engines. + *

+ */ + @Test + void testInclusiveShutdownShutsDownBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + TopologyBackedCacheAccessService service = service(l1, l2, noPromotionPolicy()); + + service.shutdown(); + + verify(l1).shutdown(); + verify(l2).shutdown(); + } + + /** + * Verifies that a real {@link InclusiveCombinedBlockCache} is adapted so topology-backed lookup + * controls L1 and L2 access explicitly. + *

+ * The legacy inclusive combined-cache constructor wires the first-level cache to the second-level + * cache through the victim-cache path. When the cache is adapted to the topology-backed model, + * that legacy victim wiring must be removed so an L1 miss does not internally delegate to L2. + *

+ *

+ * After the wiring is removed, the topology-backed service should perform a normal inclusive + * lookup: query L1 first, observe the miss, then query L2 explicitly. + *

+ */ + @Test + void testRealInclusiveCombinedCacheDoesNotDelegateL1MissThroughVictimCache() { + FirstLevelBlockCache l1 = mock(FirstLevelBlockCache.class); + BlockCache l2 = mock(BlockCache.class); + InclusiveCombinedBlockCache combinedBlockCache = new InclusiveCombinedBlockCache(l1, l2); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(l1.getBlock(key, true, false, true)).thenReturn(null); + when(l2.getBlock(key, true, false, true)).thenReturn(block); + + CacheAccessService service = CacheAccessServices.fromBlockCache(combinedBlockCache); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(l1).unsetVictimCache(); + verify(l1).getBlock(key, true, false, true); + verify(l2).getBlock(key, true, false, true); + } + + /** + * Verifies that the default placement policy writes inclusive topology blocks to both tiers. + *

+ * Inclusive topology allows the same block to be resident in L1 and L2, so default placement + * should not split blocks by data or metadata type the way the exclusive combined-cache path + * does. + *

+ */ + @Test + void testDefaultPolicyWritesInclusiveBlocksToBothTiers() { + BlockCache l1 = mock(BlockCache.class); + BlockCache l2 = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = + service(l1, l2, DefaultHBaseCachePlacementAdmissionPolicy.INSTANCE); + + service.cacheBlock(key, block, writeContext()); + + verify(l1).cacheBlock(key, block, false, false); + verify(l2).cacheBlock(key, block, false, false); + } + + /** + * Verifies that inclusive topology construction disables legacy L1 victim-cache delegation. + *

+ * Inclusive combined-cache construction may wire L1 to L2 before the cache is adapted to the + * topology-backed model. The topology-backed inclusive service needs L1 and L2 as independent + * engines so L2 lookup and promotion policy are applied by the topology. + *

+ */ + @Test + void testTieredInclusiveFactoryUnsetsL1VictimCache() { + FirstLevelBlockCache l1 = mock(FirstLevelBlockCache.class); + BlockCache l2 = mock(BlockCache.class); + + TopologyBackedCacheAccessServices.fromTieredInclusiveBlockCaches("inclusive-combined", l1, l2, + noPromotionPolicy()); + + verify(l1).unsetVictimCache(); + } + + /** + * Creates a topology-backed cache access service using a tiered inclusive topology. + * @param l1 first-level block cache + * @param l2 second-level block cache + * @param policy cache placement and admission policy + * @return topology-backed cache access service using the inclusive topology path + */ + private static TopologyBackedCacheAccessService service(BlockCache l1, BlockCache l2, + CachePlacementAdmissionPolicy policy) { + return TopologyBackedCacheAccessServices.fromTieredInclusiveBlockCaches("inclusive-combined", + l1, l2, policy); + } + + /** + * Creates a cache request context used by read-path tests. + *

+ * The returned context enables caching, marks the request as non-repeat, and asks the cache to + * update cache metrics. These values match the read-path behavior covered by the topology-backed + * cache access service tests. + *

+ * @return cache request context for read-path tests + */ + private static CacheRequestContext requestContext() { + return CacheRequestContext.newBuilder().withCaching(true).withRepeat(false) + .withUpdateCacheMetrics(true).build(); + } + + /** + * Creates a cache write context used by cache population tests. + *

+ * The returned context uses the default non-in-memory and non-blocking write behavior expected by + * the existing compatibility tests. + *

+ * @return cache write context for cache population tests + */ + private static CacheWriteContext writeContext() { + return CacheWriteContext.newBuilder().withInMemory(false).withWaitWhenCache(false).build(); + } + + /** + * Creates a placement policy that never promotes a block after a cache hit. + *

+ * This policy is useful for lookup tests that need to verify only the lookup order and returned + * block without introducing promotion side effects. + *

+ * @return cache placement and admission policy that disables promotion + */ + private static CachePlacementAdmissionPolicy noPromotionPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), any(), any(), any())) + .thenReturn(PromotionDecision.none()); + return policy; + } + + /** + * Creates a placement policy that promotes an L2 hit to L1. + *

+ * The policy returns no promotion for L1 hits and requests promotion to L1 for L2 hits. In an + * inclusive topology, this should copy the block into L1 without evicting it from L2. + *

+ * @return cache placement and admission policy that promotes L2 hits to L1 + */ + private static CachePlacementAdmissionPolicy promoteL2HitToL1Policy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), eq(CacheTier.L1), any(), any())) + .thenReturn(PromotionDecision.none()); + when(policy.shouldPromote(any(), any(), eq(CacheTier.L2), any(), any())) + .thenReturn(PromotionDecision.promoteTo(CacheTier.L1, false)); + return policy; + } + + /** + * Creates a placement policy that admits writes to the supplied tiers. + *

+ * The returned policy admits every block and returns a multi-tier placement decision containing + * the tiers supplied by the caller. + *

+ * @param tiers cache tiers selected by the policy + * @return cache placement and admission policy that writes to the supplied tiers + */ + private static CachePlacementAdmissionPolicy admitToTiersPolicy(CacheTier... tiers) { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.admit()); + when(policy.selectTier(any(), any(), any(), any())) + .thenReturn(TierDecision.multiple(Arrays.asList(tiers))); + return policy; + } + + /** + * Copies an iterable of cached blocks into a list. + *

+ * The helper makes cached-block iterable assertions deterministic and easy to compare with the + * expected tier order. + *

+ * @param iterable cached-block iterable to copy + * @return list containing all cached blocks produced by the iterable + */ + private static List toList(Iterable iterable) { + List blocks = new ArrayList<>(); + for (CachedBlock block : iterable) { + blocks.add(block); + } + return blocks; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java new file mode 100644 index 000000000000..5a21b34b6799 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/TestSingleTierTopologyBackedCacheAccessService.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.io.hfile.Cacheable; +import org.apache.hadoop.hbase.io.hfile.CachedBlock; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) +public class TestSingleTierTopologyBackedCacheAccessService { + + /** + * Verifies that a plain single-tier {@link BlockCache} is adapted to a topology-backed cache + * access service. + *

+ * HBASE-30329 routes legacy block caches through {@link TopologyBackedCacheAccessService}. A + * plain non-combined block cache should be represented by {@link SingleTierTopology}, not by the + * old {@link BlockCacheBackedCacheAccessService} runtime path. + *

+ */ + @Test + void testSingleTierBlockCacheUsesTopologyBackedAccessService() { + BlockCache blockCache = mock(BlockCache.class); + + CacheAccessService service = CacheAccessServices.fromBlockCache(blockCache); + + assertTrue(service instanceof TopologyBackedCacheAccessService); + TopologyBackedCacheAccessService topologyBackedService = + (TopologyBackedCacheAccessService) service; + assertEquals(CacheTopologyType.SINGLE_TIER, topologyBackedService.getTopology().getType()); + } + + /** + * Verifies that a single-tier topology exposes the expected topology metadata. + *

+ * The topology should contain exactly one engine, expose only the SINGLE tier, and return the + * same engine for {@link CacheTier#SINGLE}. Other tiers should not resolve to an engine. + *

+ */ + @Test + void testSingleTierTopologyMetadata() { + CacheEngine engine = mock(CacheEngine.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertEquals("single", topology.getName()); + assertEquals(CacheTopologyType.SINGLE_TIER, topology.getType()); + assertEquals(Arrays.asList(engine), topology.getEngines()); + assertEquals(Arrays.asList(CacheTier.SINGLE), topology.getTiers()); + assertSame(engine, topology.getEngine(CacheTier.SINGLE).orElseThrow()); + assertFalse(topology.getEngine(CacheTier.L1).isPresent()); + assertFalse(topology.getEngine(CacheTier.L2).isPresent()); + } + + /** + * Verifies that a single-tier topology-backed service reads blocks from the single backing cache. + *

+ * Since there is only one tier, the access service should delegate the read to the L1 engine and + * return the block supplied by the wrapped block cache. + *

+ */ + @Test + void testGetBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + when(blockCache.getBlock(key, true, false, true)).thenReturn(block); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertSame(block, service.getBlock(key, requestContext())); + + verify(blockCache).getBlock(key, true, false, true); + } + + /** + * Verifies that a single-tier topology-backed service returns {@code null} when the backing cache + * misses. + *

+ * The service should not attempt any tier fallback because the topology contains only one cache + * engine. + *

+ */ + @Test + void testGetBlockReturnsNullOnMiss() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(blockCache.getBlock(key, true, false, true)).thenReturn(null); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertSame(null, service.getBlock(key, requestContext())); + + verify(blockCache).getBlock(key, true, false, true); + } + + /** + * Verifies that cache population for a single-tier topology writes to the single backing cache. + *

+ * The placement policy selects L1. Since single-tier topology exposes only L1, the service should + * delegate the cache write to the wrapped block cache. + *

+ */ + @Test + void testCacheBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = + service(blockCache, admitToTiersPolicy(CacheTier.SINGLE)); + + service.cacheBlock(key, block, writeContext()); + + verify(blockCache).cacheBlock(key, block, false, false); + } + + /** + * Verifies that a rejected block is not written to the single backing cache. + *

+ * When the placement and admission policy rejects the write, the topology-backed service should + * not call any {@code cacheBlock} overload on the wrapped block cache. + *

+ */ + @Test + void testRejectedBlockIsNotCached() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + + TopologyBackedCacheAccessService service = service(blockCache, rejectPolicy()); + + service.cacheBlock(key, block, writeContext()); + + verify(blockCache, never()).cacheBlock(any(), any()); + verify(blockCache, never()).cacheBlock(any(), any(), anyBoolean(), anyBoolean()); + } + + /** + * Verifies that service-level eviction for a single-tier topology delegates to the backing cache. + *

+ * A single-tier topology has only one possible resident tier, so eviction should be a direct + * delegation to that tier. + *

+ */ + @Test + void testEvictBlockDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + + when(blockCache.evictBlock(key)).thenReturn(true); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertTrue(service.evictBlock(key)); + + verify(blockCache).evictBlock(key); + } + + /** + * Verifies that single-tier cached-block iteration is exposed through the topology-backed access + * service. + *

+ * Diagnostic callers use {@link CacheAccessServices#asCachedBlockIterable(CacheAccessService)} + * rather than unwrapping the legacy block cache. The single-tier topology-backed path should + * expose the same cached blocks as the wrapped block cache. + *

+ */ + @Test + void testCachedBlockIterableDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + CachedBlock firstBlock = mock(CachedBlock.class); + CachedBlock secondBlock = mock(CachedBlock.class); + + when(blockCache.iterator()).thenReturn(Arrays.asList(firstBlock, secondBlock).iterator()); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + Optional> iterable = CacheAccessServices.asCachedBlockIterable(service); + + assertTrue(iterable.isPresent()); + assertEquals(Arrays.asList(firstBlock, secondBlock), toList(iterable.get())); + } + + /** + * Verifies that single-tier topology statistics are delegated to the backing engine. + *

+ * The topology itself does not aggregate multiple tiers, so its statistics should be exactly the + * statistics exposed by the single cache engine. + *

+ */ + @Test + void testTopologyStatsDelegatesToSingleEngine() { + CacheEngine engine = mock(CacheEngine.class); + CacheStats stats = mock(CacheStats.class); + + when(engine.getStats()).thenReturn(stats); + + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertSame(stats, topology.getStats()); + } + + /** + * Verifies that promotion is not supported by a single-tier topology. + *

+ * Promotion requires a source tier and a different target tier. Since this topology has only one + * tier, promotion is a no-op and should return {@code false}. + *

+ */ + @Test + void testSingleTierTopologyDoesNotPromote() { + CacheEngine engine = mock(CacheEngine.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertFalse(topology.promote(key, block, engine, engine)); + + verify(engine, never()).cacheBlock(any(), any()); + verify(engine, never()).evictBlock(any()); + } + + /** + * Verifies that demotion is not supported by a single-tier topology. + *

+ * Demotion requires a source tier and a different lower target tier. Since this topology has only + * one tier, demotion is a no-op and should return {@code false}. + *

+ */ + @Test + void testSingleTierTopologyDoesNotDemote() { + CacheEngine engine = mock(CacheEngine.class); + BlockCacheKey key = new BlockCacheKey("file", 1L); + Cacheable block = mock(Cacheable.class); + SingleTierTopology topology = new SingleTierTopology("single", engine); + + assertFalse(topology.demote(key, block, engine, engine)); + + verify(engine, never()).cacheBlock(any(), any()); + verify(engine, never()).evictBlock(any()); + } + + /** + * Verifies that shutting down a single-tier topology-backed service shuts down the backing cache. + *

+ * The topology-backed service owns the topology lifecycle. For a single-tier topology, shutdown + * should be delegated to the only backing engine. + *

+ */ + @Test + void testShutdownDelegatesToSingleTier() { + BlockCache blockCache = mock(BlockCache.class); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + service.shutdown(); + + verify(blockCache).shutdown(); + } + + /** + * Verifies that single-tier topology-backed cache access preserves legacy current-size reporting. + *

+ * The old {@link BlockCacheBackedCacheAccessService} path delegated + * {@link CacheAccessService#getCurrentSize()} to {@link BlockCache#getCurrentSize()}. After + * routing plain block caches through {@link TopologyBackedCacheAccessService}, the same value + * must still be reported through the cache access service. This is intentionally different from + * {@link CacheAccessService#getCurrentDataSize()} because some cache implementations include + * metadata or allocator overhead in their current size. + *

+ */ + @Test + void testSingleTierCurrentSizeDelegatesToBlockCacheCurrentSize() { + BlockCache blockCache = mock(BlockCache.class); + + when(blockCache.getCurrentSize()).thenReturn(1234L); + when(blockCache.getCurrentDataSize()).thenReturn(1000L); + + TopologyBackedCacheAccessService service = service(blockCache, noPromotionPolicy()); + + assertEquals(1234L, service.getCurrentSize()); + assertEquals(1000L, service.getCurrentDataSize()); + } + + /** + * Creates a topology-backed cache access service using a single-tier topology. + * @param blockCache legacy block cache backing the single tier + * @param policy cache placement and admission policy + * @return topology-backed cache access service using a single-tier topology + */ + private static TopologyBackedCacheAccessService service(BlockCache blockCache, + CachePlacementAdmissionPolicy policy) { + return TopologyBackedCacheAccessServices.fromSingleBlockCache("single", blockCache, policy); + } + + /** + * Creates a cache request context used by read-path tests. + *

+ * The returned context enables caching, marks the request as non-repeat, and asks the cache to + * update cache metrics. These values match the read-path behavior covered by the topology-backed + * cache access service tests. + *

+ * @return cache request context for read-path tests + */ + private static CacheRequestContext requestContext() { + return CacheRequestContext.newBuilder().withCaching(true).withRepeat(false) + .withUpdateCacheMetrics(true).build(); + } + + /** + * Creates a cache write context used by cache population tests. + *

+ * The returned context uses the default non-in-memory and non-blocking write behavior expected by + * the existing topology-backed cache access service tests. + *

+ * @return cache write context for cache population tests + */ + private static CacheWriteContext writeContext() { + return CacheWriteContext.newBuilder().withInMemory(false).withWaitWhenCache(false).build(); + } + + /** + * Creates a placement policy that never promotes a block after a cache hit. + *

+ * This policy is useful for lookup tests that need to verify only lookup delegation and returned + * block behavior without introducing promotion side effects. + *

+ * @return cache placement and admission policy that disables promotion + */ + private static CachePlacementAdmissionPolicy noPromotionPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldPromote(any(), any(), any(), any(), any())) + .thenReturn(PromotionDecision.none()); + return policy; + } + + /** + * Creates a placement policy that admits writes to the supplied tiers. + *

+ * The returned policy admits every block and returns a multi-tier placement decision containing + * the tiers supplied by the caller. + *

+ * @param tiers cache tiers selected by the policy + * @return cache placement and admission policy that writes to the supplied tiers + */ + private static CachePlacementAdmissionPolicy admitToTiersPolicy(CacheTier... tiers) { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.admit()); + when(policy.selectTier(any(), any(), any(), any())) + .thenReturn(TierDecision.multiple(Arrays.asList(tiers))); + return policy; + } + + /** + * Creates a placement policy that rejects every block. + *

+ * The returned policy is used to verify that rejected cache writes are not delegated to the + * backing cache. + *

+ * @return cache placement and admission policy that rejects every block + */ + private static CachePlacementAdmissionPolicy rejectPolicy() { + CachePlacementAdmissionPolicy policy = mock(CachePlacementAdmissionPolicy.class); + when(policy.shouldAdmit(any(), any(), any(), any(), any())) + .thenReturn(AdmissionDecision.reject("test rejection")); + return policy; + } + + /** + * Copies an iterable of cached blocks into a list. + *

+ * The helper makes cached-block iterable assertions deterministic and easy to compare with the + * expected order. + *

+ * @param iterable cached-block iterable to copy + * @return list containing all cached blocks produced by the iterable + */ + private static List toList(Iterable iterable) { + List blocks = new ArrayList<>(); + for (CachedBlock block : iterable) { + blocks.add(block); + } + return blocks; + } +}