Add a store_time flag to get and tag filtering to iterkeys

diskcache/core.py

@@ -711,6 +711,28 @@ def remove(self, file_path):
         super().remove(file_path)
 
 
+def _meta_result(value, expire_time, tag, store_time, meta=(None, None, None)):
+    """Return `value`, or a tuple of `value` and the requested metadata.
+
+    The tuple holds, after the value, the expire time when `expire_time` is
+    true, the tag when `tag` is true and the store time when `store_time` is
+    true, in that order. `meta` supplies the three database values; the
+    default is used when the key is missing.
+
+    """
+    if not (expire_time or tag or store_time):
+        return value
+    db_expire_time, db_tag, db_store_time = meta
+    result = (value,)
+    if expire_time:
+        result += (db_expire_time,)
+    if tag:
+        result += (db_tag,)
+    if store_time:
+        result += (db_store_time,)
+    return result
+
+
 class Timeout(Exception):
     """Database timeout expired."""
 
@@ -1767,12 +1789,18 @@ def get(
         expire_time=False,
         tag=False,
         retry=False,
+        store_time=False,
     ):
         """Retrieve value from cache. If `key` is missing, return `default`.
 
         Raises :exc:`Timeout` error when database timeout occurs and `retry` is
         `False` (default).
 
+        When `expire_time`, `tag` or `store_time` is true, the result is a
+        tuple of the value followed by the requested metadata in that order:
+        the expire time and store time as seconds since the epoch, the tag as
+        stored. Metadata is `None` when the key is missing.
+
         :param key: key for item
         :param default: value to return if key is missing (default None)
         :param bool read: if True, return file handle to value
@@ -1781,22 +1809,21 @@ def get(
             (default False)
         :param bool tag: if True, return tag in tuple (default False)
         :param bool retry: retry if database timeout occurs (default False)
+        :param bool store_time: if True, return store_time in tuple
+            (default False)
         :return: value for item or default if key not found
         :raises Timeout: if database timeout occurs
 
         """
         db_key, raw = self._disk.put(key)
         update_column = EVICTION_POLICY[self.eviction_policy]['get']
         select = (
-            'SELECT rowid, expire_time, tag, mode, filename, value'
+            'SELECT rowid, expire_time, tag, mode, filename, value, store_time'
             ' FROM Cache WHERE key = ? AND raw = ?'
             ' AND (expire_time IS NULL OR expire_time > ?)'
         )
 
-        if expire_time and tag:
-            default = (default, None, None)
-        elif expire_time or tag:
-            default = (default, None)
+        default = _meta_result(default, expire_time, tag, store_time)
 
         if not self.statistics and update_column is None:
             # Fast path, no transaction necessary.
@@ -1806,7 +1833,17 @@ def get(
             if not rows:
                 return default
 
-            ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows
+            (
+                (
+                    rowid,
+                    db_expire_time,
+                    db_tag,
+                    mode,
+                    filename,
+                    db_value,
+                    db_store_time,
+                ),
+            ) = rows
 
             try:
                 value = self._disk.fetch(mode, filename, db_value, read)
@@ -1831,8 +1868,16 @@ def get(
                     return default
 
                 (
-                    (rowid, db_expire_time, db_tag, mode, filename, db_value),
-                ) = rows  # noqa: E127
+                    (
+                        rowid,
+                        db_expire_time,
+                        db_tag,
+                        mode,
+                        filename,
+                        db_value,
+                        db_store_time,
+                    ),
+                ) = rows
 
                 try:
                     value = self._disk.fetch(mode, filename, db_value, read)
@@ -1851,14 +1896,8 @@ def get(
                 if update_column is not None:
                     sql(update % update_column.format(now=now), (rowid,))
 
-        if expire_time and tag:
-            return (value, db_expire_time, db_tag)
-        elif expire_time:
-            return (value, db_expire_time)
-        elif tag:
-            return (value, db_tag)
-        else:
-            return value
+        meta = (db_expire_time, db_tag, db_store_time)
+        return _meta_result(value, expire_time, tag, store_time, meta)
 
     def __getitem__(self, key):
         """Return corresponding value for `key` from cache.
@@ -2849,7 +2888,7 @@ def _select_delete(
 
         return count
 
-    def iterkeys(self, reverse=False):
+    def iterkeys(self, reverse=False, tag=ENOVAL):
         """Iterate Cache keys in database sort order.
 
         >>> cache = Cache()
@@ -2860,36 +2899,39 @@ def iterkeys(self, reverse=False):
         >>> list(cache.iterkeys(reverse=True))
         [4, 3, 2, 1, 0]
 
+        If `tag` is given, only keys with a matching tag are returned. Use
+        ``tag=None`` to iterate untagged keys. Omitting `tag` returns all keys.
+        Iteration includes expired items and does not read cached values.
+
         :param bool reverse: reverse sort order (default False)
+        :param tag: tag to match (default ENOVAL, matching all tags)
         :return: iterator of Cache keys
 
         """
         sql = self._sql
         limit = 100
         _disk_get = self._disk.get
+        select = 'SELECT key, raw FROM Cache'
+        args = ()
+
+        if tag is None:
+            select += ' WHERE tag IS NULL'
+        elif tag is not ENOVAL:
+            select += ' WHERE tag = ?'
+            args = (tag,)
+
+        iterate = select + (' WHERE (' if tag is ENOVAL else ' AND (')
 
         if reverse:
-            select = (
-                'SELECT key, raw FROM Cache'
-                ' ORDER BY key DESC, raw DESC LIMIT 1'
-            )
-            iterate = (
-                'SELECT key, raw FROM Cache'
-                ' WHERE key = ? AND raw < ? OR key < ?'
-                ' ORDER BY key DESC, raw DESC LIMIT ?'
-            )
+            iterate += 'key = ? AND raw < ? OR key < ?)'
+            order = ' ORDER BY key DESC, raw DESC'
         else:
-            select = (
-                'SELECT key, raw FROM Cache'
-                ' ORDER BY key ASC, raw ASC LIMIT 1'
-            )
-            iterate = (
-                'SELECT key, raw FROM Cache'
-                ' WHERE key = ? AND raw > ? OR key > ?'
-                ' ORDER BY key ASC, raw ASC LIMIT ?'
-            )
+            iterate += 'key = ? AND raw > ? OR key > ?)'
+            order = ' ORDER BY key ASC, raw ASC'
 
-        row = sql(select).fetchall()
+        select += order + ' LIMIT 1'
+        iterate += order + ' LIMIT ?'
+        row = sql(select, args).fetchall()
 
         if row:
             ((key, raw),) = row
@@ -2899,7 +2941,7 @@ def iterkeys(self, reverse=False):
         yield _disk_get(key, raw)
 
         while True:
-            rows = sql(iterate, (key, raw, key, limit)).fetchall()
+            rows = sql(iterate, args + (key, raw, key, limit)).fetchall()
 
             if not rows:
                 break

diskcache/djangocache.py

@@ -117,10 +117,14 @@ def get(
         expire_time=False,
         tag=False,
         retry=False,
+        store_time=False,
     ):
         """Fetch a given key from the cache. If the key does not exist, return
         default, which itself defaults to None.
 
+        When `expire_time`, `tag` or `store_time` is true, the result is a
+        tuple of the value followed by the requested metadata in that order.
+
         :param key: key for item
         :param default: return value if key is missing (default None)
         :param int version: key version number (default None, cache parameter)
@@ -130,12 +134,16 @@ def get(
             (default False)
         :param tag: if True, return tag in tuple (default False)
         :param bool retry: retry if database timeout occurs (default False)
+        :param bool store_time: if True, return store_time in tuple
+            (default False)
         :return: value for item if key is found else default
 
         """
-        # pylint: disable=arguments-differ
+        # pylint: disable=arguments-differ,too-many-arguments
         key = self.make_key(key, version=version)
-        return self._cache.get(key, default, read, expire_time, tag, retry)
+        return self._cache.get(
+            key, default, read, expire_time, tag, retry, store_time
+        )
 
     def read(self, key, version=None):
         """Return file handle corresponding to `key` from Cache.

diskcache/fanout.py

@@ -9,17 +9,13 @@
 import tempfile
 import time
 
-from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout
+from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout, _meta_result
 from .persistent import Deque, Index
 
 
-def _default_with_meta(default, expire_time, tag):
-    """Match Cache.get/pop: wrap default when expire_time or tag is requested."""
-    if expire_time and tag:
-        return (default, None, None)
-    if expire_time or tag:
-        return (default, None)
-    return default
+def _default_with_meta(default, expire_time, tag, store_time=False):
+    """Match Cache.get/pop: wrap default when metadata is requested."""
+    return _meta_result(default, expire_time, tag, store_time)
 
 
 class FanoutCache:
@@ -280,12 +276,18 @@ def get(
         expire_time=False,
         tag=False,
         retry=False,
+        store_time=False,
     ):
         """Retrieve value from cache. If `key` is missing, return `default`.
 
         If database timeout occurs then returns `default` unless `retry` is set
         to `True` (default `False`).
 
+        When `expire_time`, `tag` or `store_time` is true, the result is a
+        tuple of the value followed by the requested metadata in that order,
+        with `None` for metadata when the key is missing or the shard timed
+        out.
+
         :param key: key for item
         :param default: return value if key is missing (default None)
         :param bool read: if True, return file handle to value
@@ -294,15 +296,19 @@ def get(
             (default False)
         :param tag: if True, return tag in tuple (default False)
         :param bool retry: retry if database timeout occurs (default False)
+        :param bool store_time: if True, return store_time in tuple
+            (default False)
         :return: value for item if key is found else default
 
         """
         index = self._hash(key) % self._count
         shard = self._shards[index]
         try:
-            return shard.get(key, default, read, expire_time, tag, retry)
+            return shard.get(
+                key, default, read, expire_time, tag, retry, store_time
+            )
         except (Timeout, sqlite3.OperationalError):
-            return _default_with_meta(default, expire_time, tag)
+            return _default_with_meta(default, expire_time, tag, store_time)
 
     def __getitem__(self, key):
         """Return corresponding value for `key` from cache.

docs/tutorial.rst

@@ -145,7 +145,13 @@ file-like object, and tag metadata is stored with the key. Another method,
 
 The return value is a tuple containing the value, expire time (seconds from
 epoch), and tag. Because we passed ``read=True`` the value is returned as a
-file-like object.
+file-like object. A fourth keyword parameter, `store_time`, appends the time
+the item was stored (seconds from epoch) as the last element of the tuple, so
+the age of a cached item can be logged when it is used.
+
+    >>> value, store_time = cache.get('key', store_time=True)
+    >>> type(store_time).__name__
+    'float'
 
 Use :meth:`touch <.Cache.touch>` to update the expiration time of an item in
 the cache.
@@ -327,6 +333,20 @@ keys will be serialized which is likely to have a meaningless sorted order.
     >>> cache.peekitem(last=False)
     ('c', None)
 
+To iterate keys with a matching tag, pass `tag` to :meth:`iterkeys
+<.Cache.iterkeys>`. Filtering happens in the database without reading cached
+values. Omitting `tag` includes all keys; ``tag=None`` selects untagged keys.
+As with other iteration methods, expired items are included.
+
+    >>> _ = cache.set('a', None, tag='letter')
+    >>> _ = cache.set('c', None, tag='letter')
+    >>> list(cache.iterkeys(tag='letter'))
+    ['a', 'c']
+    >>> list(cache.iterkeys(reverse=True, tag='letter'))
+    ['c', 'a']
+    >>> list(cache.iterkeys(tag=None))
+    ['b']
+
 If only the first or last item in insertion order is desired then
 :meth:`peekitem <.Cache.peekitem>` is more efficient than using iteration.