diff --git a/diskcache/core.py b/diskcache/core.py index 044ddc0..fe4875d 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -711,6 +711,28 @@ def remove(self, file_path): super().remove(file_path) +def _meta_result(value, expire_time, tag, store_time, meta=(None, None, None)): + """Return `value`, or a tuple of `value` and the requested metadata. + + The tuple holds, after the value, the expire time when `expire_time` is + true, the tag when `tag` is true and the store time when `store_time` is + true, in that order. `meta` supplies the three database values; the + default is used when the key is missing. + + """ + if not (expire_time or tag or store_time): + return value + db_expire_time, db_tag, db_store_time = meta + result = (value,) + if expire_time: + result += (db_expire_time,) + if tag: + result += (db_tag,) + if store_time: + result += (db_store_time,) + return result + + class Timeout(Exception): """Database timeout expired.""" @@ -1767,12 +1789,18 @@ def get( expire_time=False, tag=False, retry=False, + store_time=False, ): """Retrieve value from cache. If `key` is missing, return `default`. Raises :exc:`Timeout` error when database timeout occurs and `retry` is `False` (default). + When `expire_time`, `tag` or `store_time` is true, the result is a + tuple of the value followed by the requested metadata in that order: + the expire time and store time as seconds since the epoch, the tag as + stored. Metadata is `None` when the key is missing. + :param key: key for item :param default: value to return if key is missing (default None) :param bool read: if True, return file handle to value @@ -1781,6 +1809,8 @@ def get( (default False) :param bool tag: if True, return tag in tuple (default False) :param bool retry: retry if database timeout occurs (default False) + :param bool store_time: if True, return store_time in tuple + (default False) :return: value for item or default if key not found :raises Timeout: if database timeout occurs @@ -1788,15 +1818,12 @@ def get( db_key, raw = self._disk.put(key) update_column = EVICTION_POLICY[self.eviction_policy]['get'] select = ( - 'SELECT rowid, expire_time, tag, mode, filename, value' + 'SELECT rowid, expire_time, tag, mode, filename, value, store_time' ' FROM Cache WHERE key = ? AND raw = ?' ' AND (expire_time IS NULL OR expire_time > ?)' ) - if expire_time and tag: - default = (default, None, None) - elif expire_time or tag: - default = (default, None) + default = _meta_result(default, expire_time, tag, store_time) if not self.statistics and update_column is None: # Fast path, no transaction necessary. @@ -1806,7 +1833,17 @@ def get( if not rows: return default - ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows + ( + ( + rowid, + db_expire_time, + db_tag, + mode, + filename, + db_value, + db_store_time, + ), + ) = rows try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1831,8 +1868,16 @@ def get( return default ( - (rowid, db_expire_time, db_tag, mode, filename, db_value), - ) = rows # noqa: E127 + ( + rowid, + db_expire_time, + db_tag, + mode, + filename, + db_value, + db_store_time, + ), + ) = rows try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1851,14 +1896,8 @@ def get( if update_column is not None: sql(update % update_column.format(now=now), (rowid,)) - if expire_time and tag: - return (value, db_expire_time, db_tag) - elif expire_time: - return (value, db_expire_time) - elif tag: - return (value, db_tag) - else: - return value + meta = (db_expire_time, db_tag, db_store_time) + return _meta_result(value, expire_time, tag, store_time, meta) def __getitem__(self, key): """Return corresponding value for `key` from cache. @@ -2849,7 +2888,7 @@ def _select_delete( return count - def iterkeys(self, reverse=False): + def iterkeys(self, reverse=False, tag=ENOVAL): """Iterate Cache keys in database sort order. >>> cache = Cache() @@ -2860,36 +2899,39 @@ def iterkeys(self, reverse=False): >>> list(cache.iterkeys(reverse=True)) [4, 3, 2, 1, 0] + If `tag` is given, only keys with a matching tag are returned. Use + ``tag=None`` to iterate untagged keys. Omitting `tag` returns all keys. + Iteration includes expired items and does not read cached values. + :param bool reverse: reverse sort order (default False) + :param tag: tag to match (default ENOVAL, matching all tags) :return: iterator of Cache keys """ sql = self._sql limit = 100 _disk_get = self._disk.get + select = 'SELECT key, raw FROM Cache' + args = () + + if tag is None: + select += ' WHERE tag IS NULL' + elif tag is not ENOVAL: + select += ' WHERE tag = ?' + args = (tag,) + + iterate = select + (' WHERE (' if tag is ENOVAL else ' AND (') if reverse: - select = ( - 'SELECT key, raw FROM Cache' - ' ORDER BY key DESC, raw DESC LIMIT 1' - ) - iterate = ( - 'SELECT key, raw FROM Cache' - ' WHERE key = ? AND raw < ? OR key < ?' - ' ORDER BY key DESC, raw DESC LIMIT ?' - ) + iterate += 'key = ? AND raw < ? OR key < ?)' + order = ' ORDER BY key DESC, raw DESC' else: - select = ( - 'SELECT key, raw FROM Cache' - ' ORDER BY key ASC, raw ASC LIMIT 1' - ) - iterate = ( - 'SELECT key, raw FROM Cache' - ' WHERE key = ? AND raw > ? OR key > ?' - ' ORDER BY key ASC, raw ASC LIMIT ?' - ) + iterate += 'key = ? AND raw > ? OR key > ?)' + order = ' ORDER BY key ASC, raw ASC' - row = sql(select).fetchall() + select += order + ' LIMIT 1' + iterate += order + ' LIMIT ?' + row = sql(select, args).fetchall() if row: ((key, raw),) = row @@ -2899,7 +2941,7 @@ def iterkeys(self, reverse=False): yield _disk_get(key, raw) while True: - rows = sql(iterate, (key, raw, key, limit)).fetchall() + rows = sql(iterate, args + (key, raw, key, limit)).fetchall() if not rows: break diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 63013ba..857bfbe 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -117,10 +117,14 @@ def get( expire_time=False, tag=False, retry=False, + store_time=False, ): """Fetch a given key from the cache. If the key does not exist, return default, which itself defaults to None. + When `expire_time`, `tag` or `store_time` is true, the result is a + tuple of the value followed by the requested metadata in that order. + :param key: key for item :param default: return value if key is missing (default None) :param int version: key version number (default None, cache parameter) @@ -130,12 +134,16 @@ def get( (default False) :param tag: if True, return tag in tuple (default False) :param bool retry: retry if database timeout occurs (default False) + :param bool store_time: if True, return store_time in tuple + (default False) :return: value for item if key is found else default """ - # pylint: disable=arguments-differ + # pylint: disable=arguments-differ,too-many-arguments key = self.make_key(key, version=version) - return self._cache.get(key, default, read, expire_time, tag, retry) + return self._cache.get( + key, default, read, expire_time, tag, retry, store_time + ) def read(self, key, version=None): """Return file handle corresponding to `key` from Cache. diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 1b718bb..4a7a6ae 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -9,17 +9,13 @@ import tempfile import time -from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout +from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout, _meta_result from .persistent import Deque, Index -def _default_with_meta(default, expire_time, tag): - """Match Cache.get/pop: wrap default when expire_time or tag is requested.""" - if expire_time and tag: - return (default, None, None) - if expire_time or tag: - return (default, None) - return default +def _default_with_meta(default, expire_time, tag, store_time=False): + """Match Cache.get/pop: wrap default when metadata is requested.""" + return _meta_result(default, expire_time, tag, store_time) class FanoutCache: @@ -280,12 +276,18 @@ def get( expire_time=False, tag=False, retry=False, + store_time=False, ): """Retrieve value from cache. If `key` is missing, return `default`. If database timeout occurs then returns `default` unless `retry` is set to `True` (default `False`). + When `expire_time`, `tag` or `store_time` is true, the result is a + tuple of the value followed by the requested metadata in that order, + with `None` for metadata when the key is missing or the shard timed + out. + :param key: key for item :param default: return value if key is missing (default None) :param bool read: if True, return file handle to value @@ -294,15 +296,19 @@ def get( (default False) :param tag: if True, return tag in tuple (default False) :param bool retry: retry if database timeout occurs (default False) + :param bool store_time: if True, return store_time in tuple + (default False) :return: value for item if key is found else default """ index = self._hash(key) % self._count shard = self._shards[index] try: - return shard.get(key, default, read, expire_time, tag, retry) + return shard.get( + key, default, read, expire_time, tag, retry, store_time + ) except (Timeout, sqlite3.OperationalError): - return _default_with_meta(default, expire_time, tag) + return _default_with_meta(default, expire_time, tag, store_time) def __getitem__(self, key): """Return corresponding value for `key` from cache. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 0ec5bad..858a72f 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -145,7 +145,13 @@ file-like object, and tag metadata is stored with the key. Another method, The return value is a tuple containing the value, expire time (seconds from epoch), and tag. Because we passed ``read=True`` the value is returned as a -file-like object. +file-like object. A fourth keyword parameter, `store_time`, appends the time +the item was stored (seconds from epoch) as the last element of the tuple, so +the age of a cached item can be logged when it is used. + + >>> value, store_time = cache.get('key', store_time=True) + >>> type(store_time).__name__ + 'float' Use :meth:`touch <.Cache.touch>` to update the expiration time of an item in the cache. @@ -327,6 +333,20 @@ keys will be serialized which is likely to have a meaningless sorted order. >>> cache.peekitem(last=False) ('c', None) +To iterate keys with a matching tag, pass `tag` to :meth:`iterkeys +<.Cache.iterkeys>`. Filtering happens in the database without reading cached +values. Omitting `tag` includes all keys; ``tag=None`` selects untagged keys. +As with other iteration methods, expired items are included. + + >>> _ = cache.set('a', None, tag='letter') + >>> _ = cache.set('c', None, tag='letter') + >>> list(cache.iterkeys(tag='letter')) + ['a', 'c'] + >>> list(cache.iterkeys(reverse=True, tag='letter')) + ['c', 'a'] + >>> list(cache.iterkeys(tag=None)) + ['b'] + If only the first or last item in insertion order is desired then :meth:`peekitem <.Cache.peekitem>` is more efficient than using iteration. diff --git a/tests/test_core.py b/tests/test_core.py index 12ff1dc..a58eb19 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1038,6 +1038,84 @@ def test_iterkeys(cache): assert list(cache.iterkeys()) == [] +@pytest.mark.parametrize('reverse', [False, True]) +@pytest.mark.parametrize('tag', [None, '', b'', 0, False, 1.5, 'counter']) +def test_iterkeys_tag(cache, reverse, tag): + assert list(cache.iterkeys(reverse=reverse, tag=tag)) == [] + cache.set('a', 1, tag=tag) + cache.set('b', 2, tag='other') + cache.set('c', 3, tag=tag) + cache.set('d', 4) + + expected = ['a', 'c', 'd'] if tag is None else ['a', 'c'] + assert list(cache.iterkeys(reverse=reverse, tag=tag)) == ( + expected[::-1] if reverse else expected + ) + assert list(cache.iterkeys(reverse=reverse, tag='missing')) == [] + assert list(cache.iterkeys(reverse)) == ( + ['d', 'c', 'b', 'a'] if reverse else ['a', 'b', 'c', 'd'] + ) + + +@pytest.mark.parametrize('reverse', [False, True]) +@pytest.mark.parametrize('tag_index', [False, True]) +def test_iterkeys_tag_pagination(cache, reverse, tag_index): + if tag_index: + cache.create_tag_index() + + expected = [] + for convert in ( + int, + lambda num: 'key-%03d' % num, + lambda num: bytes([num]), + ): + for num in range(240): + key = convert(num) + tag = 'counter' if num % 2 else 'other' + cache.set(key, None, tag=tag) + if num % 2: + expected.append(key) + + assert list(cache.iterkeys(reverse=reverse, tag='counter')) == ( + expected[::-1] if reverse else expected + ) + + +@pytest.mark.parametrize('reverse', [False, True]) +def test_iterkeys_tag_raw_collision(cache, reverse): + pairs = [] + for num in range(105): + key = (num,) + raw_key = bytes(cache.disk.put(key)[0]) + cache.set(key, None, tag='counter') + cache.set(raw_key, None, tag='counter' if num % 2 else 'other') + pairs.append((raw_key, key, num % 2)) + + expected = [] + for raw_key, key, matching in sorted(pairs): + expected.append(key) + if matching: + expected.append(raw_key) + + assert list(cache.iterkeys(reverse=reverse, tag='counter')) == ( + expected[::-1] if reverse else expected + ) + + +def test_iterkeys_tag_does_not_fetch_values(cache): + cache.set('key', {'value': 1}, tag='counter') + with mock.patch.object(cache.disk, 'fetch') as fetch: + assert list(cache.iterkeys(tag='counter')) == ['key'] + fetch.assert_not_called() + + +def test_iterkeys_tag_includes_expired(cache): + cache.reset('cull_limit', 0) + cache.set('expired', 1, tag='counter', expire=-1) + cache.set('live', 2, tag='counter') + assert list(cache.iterkeys(tag='counter')) == ['expired', 'live'] + + def test_pickle(cache): for num, val in enumerate('abcde'): cache[val] = num @@ -2084,3 +2162,53 @@ def forker(): os.close(read_fd) assert cache.get('held') == 1 assert cache.get('child') == 2 + + +@pytest.mark.parametrize('statistics', [False, True]) +def test_get_store_time(cache, statistics): + if statistics: + cache.stats(enable=True) # Take the transactional path in get. + + before = time.time() + cache.set(0, 0, tag='number') + after = time.time() + + value, store_time = cache.get(0, store_time=True) + assert value == 0 + assert isinstance(store_time, float) + assert before <= store_time <= after + assert cache.get(0, store_time=True) == (0, store_time) + assert cache.get(0, expire_time=True, store_time=True) == ( + 0, + None, + store_time, + ) + assert cache.get(0, tag=True, store_time=True) == (0, 'number', store_time) + assert cache.get(0, expire_time=True, tag=True, store_time=True) == ( + 0, + None, + 'number', + store_time, + ) + assert cache.get(0, expire_time=True, tag=True) == (0, None, 'number') + assert cache.get(0) == 0 + + assert cache.get(1, store_time=True) == (None, None) + assert cache.get(1, default=7, tag=True, store_time=True) == ( + 7, + None, + None, + ) + assert cache.get(1, expire_time=True, tag=True, store_time=True) == ( + None, + None, + None, + None, + ) + + +def test_get_store_time_read(cache): + cache.set('key', io.BytesIO(b'value'), read=True) + reader, store_time = cache.get('key', read=True, store_time=True) + assert reader.read() == b'value' + assert isinstance(store_time, float) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 61162af..070c6d4 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -1081,6 +1081,20 @@ def test_pop(self): ) self.assertEqual(cache.pop(4, retry=False), 4) + def test_get_store_time(self): + cache.set(1, 1) + value, store_time = cache.get(1, store_time=True) + self.assertEqual(value, 1) + self.assertIsInstance(store_time, float) + value, expire_time, tag, same_store_time = cache.get( + 1, expire_time=True, tag=True, store_time=True + ) + self.assertEqual(value, 1) + self.assertIsInstance(expire_time, float) # Django's default timeout. + self.assertIsNone(tag) + self.assertEqual(same_store_time, store_time) + self.assertEqual(cache.get(2, store_time=True), (None, None)) + def test_cache(self): subcache = cache.cache('test') directory = os.path.join(cache.directory, 'cache', 'test') diff --git a/tests/test_fanout.py b/tests/test_fanout.py index 220c465..dbb78f1 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -341,6 +341,29 @@ def test_get_timeout_expire_time(cache): None, None, ) + assert cache.get(0, store_time=True) == (None, None) + assert cache.get(0, default=1, tag=True, store_time=True) == ( + 1, + None, + None, + ) + assert cache.get( + 0, default=1, expire_time=True, tag=True, store_time=True + ) == (1, None, None, None) + + +def test_get_store_time(cache): + cache.set(0, 0, tag='number') + value, store_time = cache.get(0, store_time=True) + assert value == 0 + assert isinstance(store_time, float) + assert cache.get(0, expire_time=True, tag=True, store_time=True) == ( + 0, + None, + 'number', + store_time, + ) + assert cache.get(1, store_time=True) == (None, None) def test_pop_timeout_expire_time(cache):