diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/CacheSnippets.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/CacheSnippets.java new file mode 100644 index 00000000000..bcc00881d19 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/CacheSnippets.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.backend.Cache; + +public final class CacheSnippets { + /// What the loader below stands in for: a slow lookup in another service. + public static final class Price { + final String sku; + + Price(String sku) { + this.sku = sku; + } + } + + static Price fetchPrice(String sku) { + return new Price(sku); + } + + // tag::backend-cache[] + private static final Cache PRICES = Cache.builder("prices") + .maximumSize(10000) // or maximumWeight(bytes, weigher) + .expireAfterWriteMillis(60000) + .removalListener((sku, price, cause) -> { + if (cause == Cache.SIZE) { + System.out.println("evicted " + sku); + } + }) + .build(); + + public static Price price(String sku) { + // One load per SKU, however many requests ask at once. + return PRICES.get(sku, CacheSnippets::fetchPrice); + } + + public static void priceChanged(String sku) { + PRICES.invalidate(sku); + } + // end::backend-cache[] + + private CacheSnippets() { + } +} diff --git a/docs/developer-guide/Backend-Observability.asciidoc b/docs/developer-guide/Backend-Observability.asciidoc index 17746e80612..de3bceaf8b2 100644 --- a/docs/developer-guide/Backend-Observability.asciidoc +++ b/docs/developer-guide/Backend-Observability.asciidoc @@ -330,3 +330,88 @@ counter, so `http.server.request.duration` is scraped as The management endpoints are served before the application's routes, so a catch-all route can't answer them. + +=== Application caches + +`Cache` keeps values a server would otherwise fetch or compute again, with a bound +on how many it holds and how long each one lives. It's plain Java, so the same cache +runs on the JVM and in the packaged native server: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/CacheSnippets.java[tag=backend-cache,indent=0] +---- + +A builder sets the bounds, and nothing is bounded unless it's asked for: + +[cols="2,5"] +|=== +| Setting | Meaning + +| `maximumSize(n)` | At most `n` entries; the least recently used go first. +| `maximumWeight(w, weigher)` | At most `w` in total, as the weigher weighs each entry. An entry heavier than the whole bound isn't kept: it's reported as a `SIZE` removal, and the other entries stay. +| `expireAfterWriteMillis(ms)` | An entry lives this long after it's stored. +| `expireAfterAccessMillis(ms)` | An entry lives this long after it was last read or stored. +| `expiry(expiry)` | Each entry's lifetime, chosen when it's stored. It replaces the write lifetime for that entry; the access lifetime still applies. A lifetime of zero or less isn't stored and removes the key's old value, so a `get` of such a value calls its loader every time. +| `removalListener(listener)` | Called for every entry that leaves, with the cause: `EXPLICIT`, `REPLACED`, `EXPIRED` or `SIZE`. +| `clock(clock)` | Where the cache reads the time. Tests replace it to move time without waiting. +|=== + +==== Loading + +`get(key, loader)` returns the entry, or runs the loader on the calling thread when +there's none. Requests asking for a key while its load runs don't load it again: +they wait for that load and get its result. A waiting request on a virtual thread +yields, so a slow load holds back only the requests that want its key. + +A loader that throws stores nothing. Every request waiting for it gets the same +failure: an unchecked exception as it was thrown, a checked one wrapped in +`Cache.LoadException`. The next request loads again. A loader that returns null +stores nothing either. When the key is invalidated or `put` while its load runs, +the loaded value goes to the requests waiting for it and isn't stored over the newer +state. A loader can use the cache, but asking for its own key is refused rather than +waiting forever for itself. + +The loader runs as part of the request that started the load, with that request's +transaction, security context and trace. The others only receive its value, so a +loader shouldn't return anything specific to one user. + +==== Expiry and removal + +An expired entry is never returned. It's removed when a read finds it, by a +periodic sweep during writes, or by `cleanUp()`. `size()` cleans up first, so it +counts live entries only. + +The removal listener runs on the thread whose call removed the entry, after the +cache has released its lock, so it may use the cache. A listener that throws is +reported on standard error and changes nothing else. + +==== Metrics + +Every cache publishes its figures to `Metrics` under its name: + +[cols="2,1,4"] +|=== +| Name | Kind | Meaning + +| `cache..hits` | counter | Lookups answered from the cache. +| `cache..misses` | counter | Lookups that weren't. +| `cache..loads` | counter | Loads that finished, including those that returned null. +| `cache..load_failures` | counter | Loads that threw. +| `cache..evictions` | counter | Entries removed for the size or weight bound. +| `cache..load.duration` | histogram | Time spent in the loader, in milliseconds. +| `cache..size`, `.weight` | gauge | Entries held, and their total weight. +|=== + +The same figures are on the cache itself, from `hitCount()` to `evictionCount()`. +Give each cache its own name; two caches with one name share the counters. + +==== Scope and memory + +A cache belongs to one process. Two instances of a server each hold their own, and +invalidating a key in one doesn't reach the other. Values are held by reference, so +treat them as read only. + +Each entry costs about a hundred bytes beyond its key and value: a map entry with +its links in access order, and a node holding the value, its weight and its +timestamps. diff --git a/maven/backend/src/test/java/com/codename1/backend/CacheTest.java b/maven/backend/src/test/java/com/codename1/backend/CacheTest.java new file mode 100644 index 00000000000..51c4f1e476a --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/CacheTest.java @@ -0,0 +1,441 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import com.codename1.backend.metrics.Counter; +import com.codename1.backend.metrics.Metrics; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class CacheTest { + + /** Time moves only when the test says so. */ + static final class TestClock implements Cache.Clock { + long now = 1000; + + @Override + public long millis() { + return now; + } + } + + /** Records every removal as "key=value:cause". */ + static final class Recorder implements Cache.RemovalListener { + final List seen = new ArrayList(); + + @Override + public synchronized void removed(String key, String value, int cause) { + seen.add(key + "=" + value + ":" + cause); + } + } + + private static int names; + + private static String name() { + return "test" + (names++); + } + + private static Cache.Loader upper(final AtomicInteger calls) { + return new Cache.Loader() { + @Override + public String load(String key) { + calls.incrementAndGet(); + return key.toUpperCase(java.util.Locale.ROOT); + } + }; + } + + @Test + @DisplayName("a value is loaded once and then answered from the cache") + void loadsOnce() { + Cache c = Cache.builder(name()).build(); + AtomicInteger calls = new AtomicInteger(); + assertEquals("A", c.get("a", upper(calls))); + assertEquals("A", c.get("a", upper(calls))); + assertEquals(1, calls.get()); + assertEquals(1, c.hitCount()); + assertEquals(1, c.missCount()); + assertEquals(1, c.loadCount()); + } + + @Test + @DisplayName("concurrent callers for one key share a single load") + void concurrentLoadsAreDeduplicated() throws Exception { + final Cache c = Cache.builder(name()).build(); + final AtomicInteger calls = new AtomicInteger(); + final CountDownLatch entered = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + final Cache.Loader slow = new Cache.Loader() { + @Override + public String load(String key) throws Exception { + calls.incrementAndGet(); + entered.countDown(); + release.await(10, TimeUnit.SECONDS); + return "v"; + } + }; + final String[] results = new String[8]; + Thread[] threads = new Thread[results.length]; + for (int i = 0; i < threads.length; i++) { + final int at = i; + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + results[at] = c.get("k", slow); + } + }); + threads[i].start(); + } + assertTrue(entered.await(10, TimeUnit.SECONDS)); + // Every caller has had time to find the load in flight before it ends. + Thread.sleep(200); + release.countDown(); + for (Thread t : threads) { + t.join(10000); + } + assertEquals(1, calls.get()); + for (String r : results) { + assertEquals("v", r); + } + } + + @Test + @DisplayName("a failed load reaches every waiter, stores nothing, and the next call loads again") + void failuresAreSharedAndNotCached() throws Exception { + final Cache c = Cache.builder(name()).build(); + final CountDownLatch entered = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + final Cache.Loader failing = new Cache.Loader() { + @Override + public String load(String key) throws Exception { + entered.countDown(); + release.await(10, TimeUnit.SECONDS); + throw new IOException("down"); + } + }; + final Throwable[] waiterSaw = new Throwable[1]; + Thread first = new Thread(new Runnable() { + @Override + public void run() { + try { + c.get("k", failing); + } catch (Throwable t) { + // The owner's failure; asserted through the waiter below. + } + } + }); + first.start(); + assertTrue(entered.await(10, TimeUnit.SECONDS)); + Thread waiter = new Thread(new Runnable() { + @Override + public void run() { + try { + c.get("k", failing); + } catch (Throwable t) { + waiterSaw[0] = t; + } + } + }); + waiter.start(); + Thread.sleep(200); + release.countDown(); + first.join(10000); + waiter.join(10000); + assertTrue(waiterSaw[0] instanceof Cache.LoadException, String.valueOf(waiterSaw[0])); + assertTrue(waiterSaw[0].getCause() instanceof IOException); + assertEquals(1, c.loadFailureCount()); + assertEquals(0, c.size()); + AtomicInteger calls = new AtomicInteger(); + assertEquals("K", c.get("k", upper(calls))); + assertEquals(1, calls.get()); + } + + @Test + @DisplayName("an unchecked loader exception is rethrown as it is") + void runtimeExceptionsPassThrough() { + Cache c = Cache.builder(name()).build(); + assertThrows(IllegalStateException.class, () -> c.get("k", key -> { + throw new IllegalStateException("bad"); + })); + } + + @Test + @DisplayName("a null load is answered and not stored") + void nullIsNotStored() { + Cache c = Cache.builder(name()).build(); + assertNull(c.get("k", key -> null)); + assertEquals(0, c.size()); + assertEquals("x", c.get("k", key -> "x")); + } + + @Test + @DisplayName("a loader asking for its own key is refused instead of waiting for itself") + void recursiveLoadIsRefused() { + final Cache c = Cache.builder(name()).build(); + assertThrows(IllegalStateException.class, () -> c.get("k", key -> c.get("k", k -> "x"))); + // And the cache is usable afterwards: the refused load did not stay in flight. + assertEquals("y", c.get("k", key -> "y")); + } + + @Test + @DisplayName("expire-after-write removes the entry at its deadline and notifies EXPIRED") + void expireAfterWrite() { + TestClock clock = new TestClock(); + Recorder r = new Recorder(); + Cache c = Cache.builder(name()) + .expireAfterWriteMillis(100).clock(clock).removalListener(r).build(); + c.put("a", "1"); + clock.now += 99; + assertEquals("1", c.getIfPresent("a")); + clock.now += 1; + assertNull(c.getIfPresent("a")); + assertEquals("[a=1:" + Cache.EXPIRED + "]", r.seen.toString()); + } + + @Test + @DisplayName("expire-after-access is pushed back by every read") + void expireAfterAccess() { + TestClock clock = new TestClock(); + Cache c = Cache.builder(name()) + .expireAfterAccessMillis(100).clock(clock).build(); + c.put("a", "1"); + for (int i = 0; i < 5; i++) { + clock.now += 90; + assertEquals("1", c.getIfPresent("a")); + } + clock.now += 100; + assertNull(c.getIfPresent("a")); + } + + @Test + @DisplayName("a per-entry expiry overrides the default lifetime") + void perEntryExpiry() { + TestClock clock = new TestClock(); + Cache c = Cache.builder(name()) + .expireAfterWriteMillis(1000).clock(clock) + .expiry((key, value) -> key.startsWith("short") ? 10 : Long.MAX_VALUE).build(); + c.put("short", "1"); + c.put("long", "2"); + clock.now += 10; + assertNull(c.getIfPresent("short")); + clock.now += 100000; + assertEquals("2", c.getIfPresent("long")); + } + + @Test + @DisplayName("cleanUp and size remove every expired entry, not just the ones read") + void cleanUpSweeps() { + TestClock clock = new TestClock(); + Recorder r = new Recorder(); + Cache c = Cache.builder(name()) + .expireAfterWriteMillis(100).clock(clock).removalListener(r).build(); + c.put("a", "1"); + c.put("b", "2"); + clock.now += 100; + assertEquals(0, c.size()); + assertEquals(2, r.seen.size()); + } + + @Test + @DisplayName("above the size bound the least recently used entry goes, notified SIZE") + void sizeEvictionIsLru() { + Recorder r = new Recorder(); + Cache c = Cache.builder(name()) + .maximumSize(2).removalListener(r).build(); + c.put("a", "1"); + c.put("b", "2"); + c.getIfPresent("a"); + c.put("c", "3"); + assertNull(c.getIfPresent("b")); + assertEquals("1", c.getIfPresent("a")); + assertEquals("3", c.getIfPresent("c")); + assertEquals("[b=2:" + Cache.SIZE + "]", r.seen.toString()); + assertEquals(1, c.evictionCount()); + } + + @Test + @DisplayName("a weight bound evicts until the total fits, and an entry heavier than it is not kept") + void weightEviction() { + Cache c = Cache.builder(name()) + .maximumWeight(10, (key, value) -> value.length()).build(); + c.put("a", "12345"); + c.put("b", "12345"); + assertEquals(10, c.weight()); + c.put("c", "123"); + assertNull(c.getIfPresent("a")); + assertEquals(8, c.weight()); + long before = c.evictionCount(); + c.put("huge", "12345678901"); + assertEquals(before + 1, c.evictionCount()); + assertNull(c.getIfPresent("huge")); + assertEquals(8, c.weight()); + assertEquals(2, c.size()); + } + + @Test + @DisplayName("put over an entry notifies REPLACED; invalidate and invalidateAll notify EXPLICIT") + void replacementAndInvalidation() { + Recorder r = new Recorder(); + Cache c = Cache.builder(name()).removalListener(r).build(); + c.put("a", "1"); + c.put("a", "2"); + c.put("b", "3"); + c.invalidate("a"); + c.invalidate("missing"); + c.invalidateAll(); + assertEquals("[a=1:" + Cache.REPLACED + ", a=2:" + Cache.EXPLICIT + ", b=3:" + + Cache.EXPLICIT + "]", r.seen.toString()); + assertEquals(0, c.size()); + } + + @Test + @DisplayName("a key invalidated during its load keeps the loaded value out of the cache") + void invalidateDuringLoad() { + final Cache c = Cache.builder(name()).build(); + assertEquals("stale", c.get("k", key -> { + c.invalidate("k"); + return "stale"; + })); + assertNull(c.getIfPresent("k")); + // The load's own caller gets what it loaded; the cache keeps the put. + assertEquals("stale", c.get("k", key -> { + c.put("k", "fresh"); + return "stale"; + })); + assertEquals("fresh", c.getIfPresent("k")); + } + + @Test + @DisplayName("a get after invalidate does not join the load the invalidation superseded") + @Timeout(30) + void getAfterInvalidateLoadsAfresh() throws Exception { + final Cache c = Cache.builder(name()).build(); + final CountDownLatch entered = new CountDownLatch(1); + final CountDownLatch release = new CountDownLatch(1); + Thread first = new Thread(new Runnable() { + @Override + public void run() { + try { + c.get("k", key -> { + entered.countDown(); + release.await(10, TimeUnit.SECONDS); + return "old"; + }); + } catch (Throwable t) { + // Not under test. + } + } + }); + first.start(); + assertTrue(entered.await(10, TimeUnit.SECONDS)); + c.invalidate("k"); + assertEquals("new", c.get("k", key -> "new")); + release.countDown(); + first.join(10000); + assertEquals("new", c.getIfPresent("k")); + } + + @Test + @DisplayName("an entry whose per-entry expiry is zero is never stored and replaces the old one") + void zeroExpiryIsNotStored() { + Cache c = Cache.builder(name()).maximumSize(1) + .expiry((key, value) -> value.equals("dead") ? 0 : Long.MAX_VALUE).build(); + c.put("live", "live"); + c.put("other", "dead"); + assertEquals("live", c.getIfPresent("live")); + assertEquals(1, c.size()); + assertEquals(0, c.evictionCount()); + c.put("live", "dead"); + assertNull(c.getIfPresent("live")); + assertEquals(0, c.size()); + } + + @Test + @DisplayName("a listener that throws affects neither the cache nor the caller") + void throwingListener() { + Cache c = Cache.builder(name()).maximumSize(1) + .removalListener((key, value, cause) -> { + throw new RuntimeException("listener"); + }).build(); + c.put("a", "1"); + c.put("b", "2"); + c.invalidate("b"); + assertEquals(0, c.size()); + c.put("c", "3"); + assertEquals("3", c.getIfPresent("c")); + } + + @Test + @DisplayName("a weigher that throws during a load releases its waiters and stores nothing") + void throwingWeigher() { + Cache c = Cache.builder(name()) + .maximumWeight(10, (key, value) -> { + throw new IllegalArgumentException("weigher"); + }).build(); + assertThrows(IllegalArgumentException.class, () -> c.get("k", key -> "v")); + assertEquals(0, c.size()); + assertEquals(1, c.loadFailureCount()); + } + + @Test + @DisplayName("hits, misses, loads and evictions are published as metrics") + void metrics() { + String n = name(); + Cache c = Cache.builder(n).maximumSize(1).build(); + c.get("a", key -> "1"); + c.get("a", key -> "1"); + c.put("b", "2"); + Counter hits = Metrics.counter("cache." + n + ".hits", "", "{lookup}"); + assertSame(hits, Metrics.counter("cache." + n + ".hits", "", "{lookup}")); + assertEquals(1, hits.get()); + assertEquals(1, Metrics.counter("cache." + n + ".misses", "", "{lookup}").get()); + assertEquals(1, Metrics.counter("cache." + n + ".loads", "", "{load}").get()); + assertEquals(1, Metrics.counter("cache." + n + ".evictions", "", "{entry}").get()); + } + + @Test + @DisplayName("invalid settings are refused") + void invalidSettings() { + assertThrows(IllegalArgumentException.class, () -> Cache.builder("")); + assertThrows(IllegalStateException.class, () -> Cache.builder(name()) + .maximumSize(1).maximumWeight(1, (k, v) -> 1).build()); + assertThrows(IllegalArgumentException.class, () -> Cache.builder(name()) + .expireAfterWriteMillis(0)); + Cache c = Cache.builder(name()).build(); + assertThrows(NullPointerException.class, () -> c.put("a", null)); + } +} diff --git a/vm/backend/demo/selftest/com/demo/SelfTest.java b/vm/backend/demo/selftest/com/demo/SelfTest.java index 28d66bc0041..338e8485ee8 100644 --- a/vm/backend/demo/selftest/com/demo/SelfTest.java +++ b/vm/backend/demo/selftest/com/demo/SelfTest.java @@ -31,6 +31,7 @@ import com.codename1.backend.Base64Url; import com.codename1.backend.ByteSink; +import com.codename1.backend.Cache; import com.codename1.backend.Crypto; import com.codename1.backend.Database; import com.codename1.backend.Db; @@ -227,6 +228,7 @@ public static void main(String[] args) throws Exception { rotatedCaBundlesAreReRead(); futures(); jsonCodec(); + cache(); System.out.println("passed=" + passed + " failed=" + failures.size()); for(int iter = 0 ; iter < failures.size() ; iter++) { @@ -247,6 +249,83 @@ public static void main(String[] args) throws Exception { /// work on ParparVM. /// What the build's generated JSON codecs call, on the translated runtime: /// the typed reads, the refusal messages with their paths, and the dates. + /** + * The application cache on the translated runtime: the same contract + * CacheTest pins on the JVM -- one load per key under concurrency, expiry + * on a controlled clock, LRU eviction, and removal notifications. + */ + private static void cache() throws Exception { + final long[] now = new long[] {1000}; + final List removals = new ArrayList(); + Cache.Clock clock = new Cache.Clock() { + public long millis() { + return now[0]; + } + }; + Cache.RemovalListener listener = new Cache.RemovalListener() { + public void removed(Object key, Object value, int cause) { + synchronized (removals) { + removals.add(key + "=" + value + ":" + cause); + } + } + }; + Cache c = Cache.builder("selftest").maximumSize(2).expireAfterWriteMillis(100) + .clock(clock).removalListener(listener).build(); + c.put("a", "1"); + c.put("b", "2"); + c.getIfPresent("a"); + c.put("c", "3"); + check("cache evicts the least recently used", "null", String.valueOf(c.getIfPresent("b"))); + check("cache notifies a size eviction", "[b=2:" + Cache.SIZE + "]", removals.toString()); + now[0] += 100; + check("cache expires after write", "null", String.valueOf(c.getIfPresent("a"))); + check("cache size counts live entries", "0", String.valueOf(c.size())); + + final Cache shared = Cache.builder("selftest.shared").build(); + final int[] loads = new int[1]; + final Cache.Loader slow = new Cache.Loader() { + public Object load(Object key) throws Exception { + synchronized (loads) { + loads[0]++; + } + Thread.sleep(300); + return "v"; + } + }; + final String[] got = new String[6]; + Thread[] threads = new Thread[got.length]; + for (int i = 0; i < threads.length; i++) { + final int at = i; + threads[i] = new Thread(new Runnable() { + public void run() { + got[at] = String.valueOf(shared.get("k", slow)); + } + }); + threads[i].start(); + } + for (int i = 0; i < threads.length; i++) { + threads[i].join(10000); + } + check("concurrent cache callers share one load", "1", String.valueOf(loads[0])); + StringBuilder all = new StringBuilder(); + for (int i = 0; i < got.length; i++) { + all.append(got[i]); + } + check("every concurrent caller gets the loaded value", "vvvvvv", all.toString()); + + final Cache failing = Cache.builder("selftest.failing").build(); + checkThrows("a failed load is rethrown", new Body() { + public void run() throws Exception { + failing.get("k", new Cache.Loader() { + public Object load(Object key) throws Exception { + throw new IOException("down"); + } + }); + } + }); + check("a failed load stores nothing", "0", String.valueOf(failing.size())); + } + private static void jsonCodec() throws Exception { JsonCodec.Path root = JsonCodec.Path.ROOT; check("codec: a whole number", "42", String.valueOf(JsonCodec.readLong( diff --git a/vm/backend/src/com/codename1/backend/Cache.java b/vm/backend/src/com/codename1/backend/Cache.java new file mode 100644 index 00000000000..6cc06f3bd53 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Cache.java @@ -0,0 +1,778 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import com.codename1.backend.metrics.Counter; +import com.codename1.backend.metrics.Gauge; +import com.codename1.backend.metrics.Histogram; +import com.codename1.backend.metrics.Metrics; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// A bounded, process-local application cache: entries limited by count or +/// by weight, expiring after a write, after an access or at a time each entry +/// chooses, loaded at most once per key however many callers ask together. +/// +/// ```java +/// private static final Cache PRICES = Cache.builder("prices") +/// .maximumSize(10000) +/// .expireAfterWriteMillis(60000) +/// .build(); +/// ... +/// Price p = PRICES.get(sku, new Cache.Loader() { +/// public Price load(String key) throws Exception { +/// return priceService.lookup(key); +/// } +/// }); +/// ``` +/// +/// ## Loading +/// +/// [#get(Object, Loader)] answers a live entry, or runs the loader on the +/// calling thread with no lock held. A second caller asking for the same key +/// while that load runs does not load again: it waits for the first one and +/// gets its result -- the value, or the same failure. A loader that fails, or +/// answers null, leaves the cache as it was; nothing is stored, so the next +/// caller loads again. A key invalidated or [#put] while its load is running +/// keeps what that call did: the loaded value is handed to the callers that +/// waited for it and is not stored over the newer state. +/// +/// A loader may use the cache, but not for the key it is loading; that would +/// wait for itself, and is refused with `IllegalStateException`. +/// +/// A waiting virtual thread naps rather than blocking its host, so a slow +/// load holds back only the requests that want its key. +/// +/// ## Eviction and expiry +/// +/// Above the size or weight bound the least recently used entries go first. +/// An expired entry is never answered; it is removed, with an `EXPIRED` +/// notification, when a read finds it, when a write's periodic sweep does, +/// or on [#cleanUp]. [#size] cleans up first, so it counts only live +/// entries. +/// +/// ## Removal notifications +/// +/// The [RemovalListener] runs on the thread whose call removed the entry, +/// after the cache's lock is released, so it may use the cache. One that +/// throws is reported on standard error and does not affect the cache or the +/// call that triggered it. Notifications from one call are delivered in the +/// order the entries were removed. +/// +/// ## Metrics +/// +/// Every cache records `cache..hits`, `.misses`, `.loads`, +/// `.load_failures` and `.evictions` (counters), `.size` and `.weight` +/// (gauges) and `.load.duration` (a histogram, milliseconds) in +/// [Metrics]; the same figures are on the cache's own accessors. Two caches +/// built with one name share the counters, and the gauges read the newer. +/// +/// ## Scope and cost +/// +/// A cache lives in this process only: two instances of a server each hold +/// their own, and nothing is shared or invalidated between them. Values are +/// held by reference, so a value a caller changes changes for every reader. +/// Each entry costs about a hundred bytes beyond its key and value -- a hash +/// map entry with its access-order links, and a node holding the value, its +/// weight and three timestamps -- on the JVM and in the native binary alike. +/// +/// Server code: this class is not available in the app. +public final class Cache { + + /// Computes the value for a key that is absent. + public interface Loader { + /// The value, or null for "no value"; null is not stored. + V load(K key) throws Exception; + } + + /// The weight of an entry, for [Cache.Builder#maximumWeight]. + public interface Weigher { + /// Zero or more. + long weigh(K key, V value); + } + + /// A lifetime chosen per entry, from when it is written. + public interface Expiry { + /// Milliseconds the entry lives after being stored, or `Long.MAX_VALUE` + /// for no limit. Overrides [Cache.Builder#expireAfterWriteMillis] for this + /// entry; [Cache.Builder#expireAfterAccessMillis] still applies. Zero or + /// less means the value is not stored, and any old value for the key is + /// removed; a [Cache#get] of such a value calls its loader every time. + long millisToLive(K key, V value); + } + + /// Told about every entry that leaves the cache. + public interface RemovalListener { + /// `cause` is one of [#EXPLICIT], [#REPLACED], [#EXPIRED] or [#SIZE]. + void removed(K key, V value, int cause); + } + + /// Where the cache reads the time: monotonic milliseconds. Replace it in a + /// test to move time deterministically. + public interface Clock { + long millis(); + } + + /// Removed by [#invalidate] or [#invalidateAll]. + public static final int EXPLICIT = 1; + /// Overwritten by [#put] or by a load. + public static final int REPLACED = 2; + /// Its lifetime ran out. + public static final int EXPIRED = 3; + /// Evicted to stay within the size or weight bound. + public static final int SIZE = 4; + + /// Thrown by [#get(Object, Loader)] when the loader threw a checked + /// exception, which is its cause. An unchecked one is rethrown as it is. + public static final class LoadException extends RuntimeException { + public LoadException(String message, Throwable cause) { + super(message, cause); + } + } + + /// A cache's settings. Nothing is bounded or expires unless asked. + public static final class Builder { + private final String name; + private long maximumSize = -1; + private long maximumWeight = -1; + private Weigher weigher; + private long afterWrite = -1; + private long afterAccess = -1; + private Expiry expiry; + private RemovalListener listener; + private Clock clock; + + Builder(String name) { + if (name == null || name.length() == 0) { + throw new IllegalArgumentException("A cache needs a name, for its metrics"); + } + this.name = name; + } + + /// At most `entries` entries. Excludes [#maximumWeight]. + public Builder maximumSize(long entries) { + if (entries < 0) { + throw new IllegalArgumentException("maximumSize must not be negative"); + } + maximumSize = entries; + return this; + } + + /// At most `weight` in total, as `weigher` weighs entries. Excludes + /// [#maximumSize]. + public Builder maximumWeight(long weight, Weigher weigher) { + if (weight < 0 || weigher == null) { + throw new IllegalArgumentException("maximumWeight needs a weight and a weigher"); + } + maximumWeight = weight; + this.weigher = weigher; + return this; + } + + /// Entries expire this long after they are stored. + public Builder expireAfterWriteMillis(long millis) { + afterWrite = positive(millis); + return this; + } + + /// Entries expire this long after they were last read or stored. + public Builder expireAfterAccessMillis(long millis) { + afterAccess = positive(millis); + return this; + } + + /// Each entry's lifetime from its write, chosen by `expiry`. + public Builder expiry(Expiry expiry) { + this.expiry = expiry; + return this; + } + + /// Told about every removal; see the class description. + public Builder removalListener(RemovalListener listener) { + this.listener = listener; + return this; + } + + /// The time source; the default is the monotonic system clock. + public Builder clock(Clock clock) { + this.clock = clock; + return this; + } + + public Cache build() { + if (maximumSize >= 0 && maximumWeight >= 0) { + throw new IllegalStateException("A cache is bounded by size or by weight, not both"); + } + return new Cache(this); + } + + private static long positive(long millis) { + if (millis <= 0) { + throw new IllegalArgumentException("An expiry must be positive"); + } + return millis; + } + } + + private static final Clock SYSTEM_CLOCK = new Clock() { + @Override + public long millis() { + return System.nanoTime() / 1000000L; + } + }; + + /// A cache named `name`; the name prefixes its metrics. + public static Builder builder(String name) { + return new Builder(name); + } + + private static final class Node { + V value; + long weight; + long accessed; + /// When it expires by its write: Long.MAX_VALUE for never. + long writeExpires; + } + + /// One running load, shared by every caller of its key. + private static final class Load { + final Object owner; + boolean done; + /// Set when the key was invalidated or put during the load. + boolean superseded; + V value; + Throwable error; + + Load(Object owner) { + this.owner = owner; + } + } + + private static final class Removal { + final K key; + final V value; + final int cause; + + Removal(K key, V value, int cause) { + this.key = key; + this.value = value; + this.cause = cause; + } + } + + private final String name; + private final long maximumSize; + private final long maximumWeight; + private final Weigher weigher; + private final long afterWrite; + private final long afterAccess; + private final Expiry expiry; + private final RemovalListener listener; + private final Clock clock; + private final long sweepInterval; + + /// Access ordered, so iteration starts at the least recently used. Guarded + /// by `this`, as is everything below. + private final LinkedHashMap> entries = new LinkedHashMap>(16, 0.75f, true); + private final Map> loading = new HashMap>(); + private long totalWeight; + /// Starts at the minimum: nanoTime, and a test clock, may read negative. + private long nextSweep = Long.MIN_VALUE; + private long hits; + private long misses; + private long loads; + private long loadFailures; + private long evictions; + + private final Counter hitCounter; + private final Counter missCounter; + private final Counter loadCounter; + private final Counter failureCounter; + private final Counter evictionCounter; + private final Histogram loadDuration; + + private Cache(Builder b) { + name = b.name; + maximumSize = b.maximumSize; + maximumWeight = b.maximumWeight; + weigher = b.weigher; + afterWrite = b.afterWrite; + afterAccess = b.afterAccess; + expiry = b.expiry; + listener = b.listener; + clock = b.clock == null ? SYSTEM_CLOCK : b.clock; + long shortest = Long.MAX_VALUE; + if (afterWrite > 0) { + shortest = afterWrite; + } + if (afterAccess > 0) { + shortest = Math.min(shortest, afterAccess); + } + // A per-entry expiry can be anything; sweep once a second for it. The + // sweep is the only full walk, so it is kept to a fraction of the + // shortest lifetime rather than run on every write. + sweepInterval = shortest == Long.MAX_VALUE ? (expiry != null ? 1000 : Long.MAX_VALUE) + : Math.max(1, Math.min(expiry != null ? 1000 : shortest, shortest / 4)); + String prefix = "cache." + name + "."; + hitCounter = Metrics.counter(prefix + "hits", "Lookups answered from the cache", "{lookup}"); + missCounter = Metrics.counter(prefix + "misses", "Lookups the cache could not answer", + "{lookup}"); + loadCounter = Metrics.counter(prefix + "loads", "Loads that completed", "{load}"); + failureCounter = Metrics.counter(prefix + "load_failures", "Loads that threw", "{load}"); + evictionCounter = Metrics.counter(prefix + "evictions", + "Entries evicted for size or weight", "{entry}"); + loadDuration = Metrics.histogram(prefix + "load.duration", "Time spent loading", "ms"); + Metrics.gauge(prefix + "size", "Entries in the cache", "{entry}", new Gauge.Source() { + @Override + public double read() { + // Not size(): a metrics read must not run removal listeners on + // the exporter's thread. May include entries not yet swept. + synchronized (Cache.this) { + return entries.size(); + } + } + }); + Metrics.gauge(prefix + "weight", "Total weight of the entries", "1", new Gauge.Source() { + @Override + public double read() { + return weight(); + } + }); + } + + public String getName() { + return name; + } + + /// The live value for `key`, or null. Counts a hit or a miss. + public V getIfPresent(K key) { + List> removed = null; + V answer; + synchronized (this) { + long now = clock.millis(); + Node n = entries.get(key); + if (n != null && expired(n, now)) { + entries.remove(key); + totalWeight -= n.weight; + removed = add(null, key, n.value, EXPIRED); + n = null; + } + if (n == null) { + misses++; + answer = null; + } else { + n.accessed = now; + hits++; + answer = n.value; + } + } + (answer == null ? missCounter : hitCounter).increment(); + notifyAll(removed); + return answer; + } + + /// The live value for `key`, loading it with `loader` when absent. See the + /// class description for concurrent callers, failures and null. + public V get(K key, Loader loader) { + if (key == null || loader == null) { + throw new NullPointerException("A cache lookup needs a key and a loader"); + } + Object me = caller(); + Load load; + boolean mine; + List> removed = null; + synchronized (this) { + long now = clock.millis(); + Node n = entries.get(key); + if (n != null && expired(n, now)) { + entries.remove(key); + totalWeight -= n.weight; + removed = add(null, key, n.value, EXPIRED); + n = null; + } + if (n != null) { + n.accessed = now; + hits++; + V v = n.value; + hitCounter.increment(); + // Nothing was removed: an expired node would have left n null. + return v; + } + misses++; + load = loading.get(key); + mine = load == null; + if (mine) { + load = new Load(me); + loading.put(key, load); + } else if (load.owner.equals(me)) { + throw new IllegalStateException("The loader for " + key + + " asked the cache for that same key"); + } + } + missCounter.increment(); + notifyAll(removed); + if (mine) { + return runLoad(key, loader, load); + } + return await(load); + } + + private V runLoad(K key, Loader loader, Load load) { + long started = System.currentTimeMillis(); + V value = null; + Throwable error = null; + try { + value = loader.load(key); + } catch (Throwable t) { + error = t; + } + loadDuration.record(System.currentTimeMillis() - started); + List> removed = null; + synchronized (this) { + if (loading.get(key) == load) { //NOPMD CompareObjectsWithEquals - this load, by identity + loading.remove(key); + } + if (error == null && value != null && !load.superseded) { + try { + removed = store(key, value, null); + } catch (Throwable t) { + // A weigher or an expiry that threw: store() changes nothing + // before both have answered, so the cache is as it was, and + // the waiters must still be released below. + error = t; + } + } + if (error != null) { + loadFailures++; + } else { + loads++; + } + } + (error != null ? failureCounter : loadCounter).increment(); + countEvictions(removed); + synchronized (load) { + load.value = value; + load.error = error; + load.done = true; + load.notifyAll(); + } + notifyAll(removed); + return answer(load); + } + + /// Waits for another caller's load of the same key. A virtual thread naps, + /// as Database's gate does, so its host keeps running the others. + private V await(Load load) { + boolean virtual = VirtualThread.isVirtual(); + boolean interrupted = false; + long nap = 1; + while (true) { + synchronized (load) { + if (load.done) { + break; + } + if (!virtual) { + try { + load.wait(); + } catch (InterruptedException e) { + interrupted = true; + } + continue; + } + } + HttpServer.napUntil(System.currentTimeMillis() + nap); + nap = Math.min(20, nap * 2); + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return answer(load); + } + + @SuppressWarnings("unchecked") + private V answer(Load load) { + V value; + Throwable error; + synchronized (load) { + value = load.value; + error = load.error; + } + if (error == null) { + return value; + } + if (error instanceof RuntimeException) { + throw (RuntimeException) error; + } + if (error instanceof Error) { + throw (Error) error; + } + throw new LoadException("Loading " + name + " failed: " + error, error); + } + + /// Stores `value`, replacing what is there. + public void put(K key, V value) { + if (key == null || value == null) { + throw new NullPointerException("A cache stores neither null keys nor null values"); + } + List> removed; + synchronized (this) { + Load load = loading.get(key); + if (load != null) { + load.superseded = true; + } + removed = store(key, value, null); + } + countEvictions(removed); + notifyAll(removed); + } + + /// Removes `key`, and keeps a load of it already running from storing its + /// result. + public void invalidate(K key) { + List> removed = null; + synchronized (this) { + // Out of `loading` too, so a get() arriving after this starts a + // fresh load instead of joining one that may read the old value. + Load load = loading.remove(key); + if (load != null) { + load.superseded = true; + } + Node n = entries.remove(key); + if (n != null) { + totalWeight -= n.weight; + removed = add(null, key, n.value, EXPLICIT); + } + } + notifyAll(removed); + } + + /// Removes every entry, as [#invalidate] does each. + public void invalidateAll() { + List> removed = null; + synchronized (this) { + for (Load load : loading.values()) { + load.superseded = true; + } + loading.clear(); + for (Map.Entry> e : entries.entrySet()) { + removed = add(removed, e.getKey(), e.getValue().value, EXPLICIT); + } + entries.clear(); + totalWeight = 0; + } + notifyAll(removed); + } + + /// Removes every expired entry now. + public void cleanUp() { + List> removed; + synchronized (this) { + removed = sweep(clock.millis(), null); + } + notifyAll(removed); + } + + /// Live entries, after a [#cleanUp]. + public long size() { + cleanUp(); + synchronized (this) { + return entries.size(); + } + } + + /// The total weight of the entries: their count when no weigher is set. + public synchronized long weight() { + return weigher == null ? entries.size() : totalWeight; + } + + public synchronized long hitCount() { + return hits; + } + + public synchronized long missCount() { + return misses; + } + + /// Loads that completed, with a value or with null. + public synchronized long loadCount() { + return loads; + } + + public synchronized long loadFailureCount() { + return loadFailures; + } + + /// Entries evicted for size or weight; expiry and invalidation not counted. + public synchronized long evictionCount() { + return evictions; + } + + // ------------------------------------------------------------ internals + + /// Called holding the lock. + private List> store(K key, V value, List> removed) { + long now = clock.millis(); + long w = 1; + if (weigher != null) { + w = weigher.weigh(key, value); + if (w < 0) { + throw new IllegalArgumentException("A weigher answered a negative weight for " + key); + } + } + long life = expiry != null ? expiry.millisToLive(key, value) : afterWrite; + boolean dead = expiry != null && life <= 0; + boolean oversized = maximumWeight >= 0 && w > maximumWeight; + if (dead || oversized) { + // Not kept. Stored, a dead entry would hold size and weight until + // the next full sweep, and an oversized one would evict every live + // entry from the least recently used end before going itself. + Node old = entries.remove(key); + if (old != null) { + totalWeight -= old.weight; + removed = add(removed, key, old.value, expired(old, now) ? EXPIRED : REPLACED); + } + if (oversized && !dead) { + evictions++; + removed = add(removed, key, value, SIZE); + } + return removed; + } + Node n = new Node(); + n.value = value; + n.weight = w; + n.accessed = now; + if (expiry == null && life < 0 || life == Long.MAX_VALUE || now > Long.MAX_VALUE - life) { + n.writeExpires = Long.MAX_VALUE; + } else { + n.writeExpires = now + life; + } + Node old = entries.put(key, n); + if (old != null) { + totalWeight -= old.weight; + removed = add(removed, key, old.value, expired(old, now) ? EXPIRED : REPLACED); + } + totalWeight += w; + removed = sweep(now, removed); + return evict(removed); + } + + /// Removes expired entries; a full walk only every sweepInterval. + private List> sweep(long now, List> removed) { + boolean full = sweepInterval != Long.MAX_VALUE && now >= nextSweep; + Iterator>> it = entries.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> e = it.next(); + Node n = e.getValue(); + if (expired(n, now)) { + K key = e.getKey(); + it.remove(); + totalWeight -= n.weight; + removed = add(removed, key, n.value, EXPIRED); + } else if (!full) { + // The head is the least recently accessed, so access expiry + // never needs to look past the first live one. + break; + } + } + if (full) { + nextSweep = now + sweepInterval; + } + return removed; + } + + private List> evict(List> removed) { + Iterator>> it = entries.entrySet().iterator(); + while (it.hasNext() && over()) { + Map.Entry> e = it.next(); + // Read before removing: ParparVM's compact map has no entry objects, + // so an entry read after its removal answers null. + K key = e.getKey(); + Node n = e.getValue(); + it.remove(); + totalWeight -= n.weight; + evictions++; + removed = add(removed, key, n.value, SIZE); + } + return removed; + } + + private boolean over() { + if (maximumSize >= 0) { + return entries.size() > maximumSize; + } + return maximumWeight >= 0 && totalWeight > maximumWeight; + } + + private boolean expired(Node n, long now) { + return now >= n.writeExpires || afterAccess > 0 && now - n.accessed >= afterAccess; + } + + private void countEvictions(List> removed) { + if (removed == null) { + return; + } + long n = 0; + for (Removal r : removed) { + if (r.cause == SIZE) { + n++; + } + } + if (n > 0) { + evictionCounter.add(n); + } + } + + private static List> add(List> list, K key, V value, + int cause) { + if (list == null) { + list = new ArrayList>(); + } + list.add(new Removal(key, value, cause)); + return list; + } + + /// Delivers removals with no lock held; a listener that throws is reported + /// and skipped, so it can never leave the cache half updated. + private void notifyAll(List> removed) { + if (removed == null || listener == null) { + return; + } + for (Removal r : removed) { + try { + listener.removed(r.key, r.value, r.cause); + } catch (Throwable t) { + System.err.println("Cache " + name + ": removal listener threw " + t); + } + } + } + + private static Object caller() { + long virtual = VirtualThread.current(); + return virtual != 0 ? (Object) Long.valueOf(virtual) : Thread.currentThread(); + } +}