From a2846bdf2d63bb0a5d53c26525ad2460cfb94aaf Mon Sep 17 00:00:00 2001 From: August Vi Date: Sat, 10 Oct 2026 22:26:59 -0700 Subject: [PATCH] Open caches read-only when their database or directory is not writable or when asked, and stop rewriting unchanged settings, triggers and the tag index on open --- .pylintrc | 2 +- diskcache/__init__.py | 2 + diskcache/core.py | 249 +++++++++++++++++++++---- docs/api.rst | 7 + docs/tutorial.rst | 53 ++++++ tests/test_readonly.py | 400 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 674 insertions(+), 39 deletions(-) create mode 100644 tests/test_readonly.py diff --git a/.pylintrc b/.pylintrc index 6f61736..6054d77 100644 --- a/.pylintrc +++ b/.pylintrc @@ -303,7 +303,7 @@ max-locals=30 max-parents=7 # Maximum number of public methods for a class (see R0904). -max-public-methods=30 +max-public-methods=32 # Maximum number of return / yield for function / method body. max-returns=8 diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 5e5eb9b..3357f05 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -17,6 +17,7 @@ Disk, EmptyDirWarning, JSONDisk, + ReadOnlyError, SafeDisk, Timeout, UnknownFileWarning, @@ -48,6 +49,7 @@ 'JSONDisk', 'Lock', 'RLock', + 'ReadOnlyError', 'SafeDisk', 'Timeout', 'UNKNOWN', diff --git a/diskcache/core.py b/diskcache/core.py index fe4875d..df90353 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -14,6 +14,7 @@ import json import os import os.path as op +import pathlib import pickle import pickletools import sqlite3 @@ -737,6 +738,30 @@ class Timeout(Exception): """Database timeout expired.""" +class ReadOnlyError(sqlite3.OperationalError): # pylint: disable=R0903 + """Write attempted on a read-only cache. + + Raised by every method that would modify a cache opened read-only, either + explicitly with ``Cache(directory, read_only=True)`` or because the cache + database or its directory is not writable. A subclass of + :exc:`sqlite3.OperationalError`, the error SQLite itself raises for + writes to a read-only database, so existing handlers keep working. + """ + + +# Pragmas that are stored in the database file rather than set per +# connection; a read-only cache leaves them as the file has them. +_PERSISTENT_PRAGMAS = frozenset({'auto_vacuum', 'journal_mode'}) + +# SQL condition selecting rows that have not expired, evaluated by SQLite +# itself so it can be embedded in a statement without a parameter. Used by +# read-only caches, which cannot delete expired rows while reading. +_LIVE_CONDITION = ( + '(expire_time IS NULL' + " OR expire_time > (julianday('now') - 2440587.5) * 86400.0)" +) + + class UnknownFileWarning(UserWarning): """Warning used by Cache.check for unknown files.""" @@ -1022,12 +1047,21 @@ def _after_fork_in_child(): class Cache: """Disk and file backed cache.""" - def __init__(self, directory=None, timeout=60, disk=Disk, **settings): + def __init__( + self, + directory=None, + timeout=60, + disk=Disk, + *, + read_only=False, + **settings + ): """Initialize cache instance. :param str directory: cache directory :param float timeout: SQLite connection timeout :param disk: Disk type or subclass for serialization + :param bool read_only: open the cache read-only (default False) :param settings: any of DEFAULT_SETTINGS The cache directory, any missing parents and the subdirectories that @@ -1035,6 +1069,14 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): can read them. A directory that already exists keeps its mode; the cache never changes permissions. Windows ignores the mode bits. + Opening an existing cache writes nothing unless a setting differs + from the stored one. A cache whose database file or directory is not + writable opens read-only, as does a cache opened with `read_only`; + reads work and every method that would write raises + :exc:`ReadOnlyError`. The read-only state is not persisted and + `read_only` is not a setting. See the :ref:`tutorial + ` for details. + """ try: assert issubclass(disk, Disk) @@ -1056,6 +1098,8 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): finalizer = weakref.finalize(self, self._cons.close_all) finalizer.atexit = False + self._read_only = self._resolve_read_only(directory, read_only) + if not op.isdir(directory): try: _makedirs_private(op.abspath(directory)) @@ -1076,6 +1120,10 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): sql('SELECT key, value FROM Settings').fetchall() ) except sqlite3.OperationalError: + if self._read_only: + raise ReadOnlyError( + 'read-only cache in %r has no Settings table' % directory + ) from None current_settings = {} sets = DEFAULT_SETTINGS.copy() @@ -1091,11 +1139,12 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): if key.startswith('sqlite_'): self.reset(key, value, update=False) - sql( - 'CREATE TABLE IF NOT EXISTS Settings (' - ' key TEXT NOT NULL UNIQUE,' - ' value)' - ) + if not self._read_only: + sql( + 'CREATE TABLE IF NOT EXISTS Settings (' + ' key TEXT NOT NULL UNIQUE,' + ' value)' + ) # Setup Disk object (must happen after settings initialized). @@ -1106,21 +1155,67 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): } self._disk = disk(directory, **kwargs) - # Set cached attributes: updates settings and sets pragmas. + self._init_settings(sql, sets, current_settings) + + ((self._page_size,),) = sql('PRAGMA page_size').fetchall() + + if self._read_only: + # The schema, triggers and tag index are whatever the writer + # left; reads need none of them to change. + self.close() + self._timeout = timeout + self._sql # pylint: disable=pointless-statement + return + + self._init_schema(sql) + + # Close and re-open database connection with given timeout. + + self.close() + self._timeout = timeout + self._sql # pylint: disable=pointless-statement + + @staticmethod + def _resolve_read_only(directory, read_only): + # A database that exists but cannot be written, or whose directory + # cannot be written, is opened read-only. Opening read-only on + # purpose needs an existing database, since nothing is created. + db_path = op.join(directory, DBNAME) + db_exists = op.isfile(db_path) + + if read_only and not db_exists: + raise ReadOnlyError( + 'no cache database to open read-only in %r' % directory + ) + + writable = os.access(directory, os.W_OK) and os.access( + db_path, os.W_OK + ) + return bool(read_only) or (db_exists and not writable) + + def _init_settings(self, sql, sets, current_settings): + # Set cached attributes: updates settings and sets pragmas. Only a + # setting that is missing or differs from the stored value is + # written, so opening an existing cache leaves the database alone. for key, value in sets.items(): - query = 'INSERT OR REPLACE INTO Settings VALUES (?, ?)' - sql(query, (key, value)) - self.reset(key, value) + if ( + not self._read_only + and current_settings.get(key, ENOVAL) != value + ): + query = 'INSERT OR REPLACE INTO Settings VALUES (?, ?)' + sql(query, (key, value)) + self.reset(key, value, update=False) for key, value in METADATA.items(): - query = 'INSERT OR IGNORE INTO Settings VALUES (?, ?)' - sql(query, (key, value)) + if key not in current_settings and not self._read_only: + query = 'INSERT OR IGNORE INTO Settings VALUES (?, ?)' + sql(query, (key, value)) self.reset(key) - ((self._page_size,),) = sql('PRAGMA page_size').fetchall() - - # Setup Cache table. + def _init_schema(self, sql): + # Create the Cache table, its indexes and the triggers when they are + # missing. Every statement is a no-op on an existing database. sql( 'CREATE TABLE IF NOT EXISTS Cache (' @@ -1160,19 +1255,19 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): self._update_triggers(sql) - # Create tag index if requested. + # Create or drop the tag index only when the database disagrees + # with the setting, so a reopen does not write. - if self.tag_index: # pylint: disable=no-member + ((has_tag_index,),) = sql( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index'" + " AND name = 'Cache_tag_rowid'" + ).fetchall() + + if self.tag_index and not has_tag_index: # pylint: disable=no-member self.create_tag_index() - else: + elif has_tag_index and not self.tag_index: # pylint: disable=no-member self.drop_tag_index() - # Close and re-open database connection with given timeout. - - self.close() - self._timeout = timeout - self._sql # pylint: disable=pointless-statement - @property def directory(self): """Cache directory.""" @@ -1188,6 +1283,15 @@ def disk(self): """Disk used for serialization.""" return self._disk + @property + def read_only(self): + """True when the cache was opened read-only.""" + return self._read_only + + def _check_writable(self): + if self._read_only: + raise ReadOnlyError('cache in %r is read-only' % self._directory) + @property def _con(self): # Check the process ID to support forking. If it changed, the @@ -1211,11 +1315,23 @@ def _connect(self): # it, once that thread has exited. Each connection is still used by # one thread at a time, which is all SQLite's multi-thread mode asks. + path = op.join(self._directory, DBNAME) + + if self._read_only: + # mode=ro refuses writes at the SQLite level. A reader of a WAL + # database needs to create the -shm file next to it, so when the + # directory is not writable immutable=1 skips that file and all + # locking, which is safe only because nothing can write there. + path = pathlib.Path(op.abspath(path)).as_uri() + '?mode=ro' + if not os.access(self._directory, os.W_OK): + path += '&immutable=1' + con = sqlite3.connect( - op.join(self._directory, DBNAME), + path, timeout=self._timeout, isolation_level=None, check_same_thread=False, + uri=self._read_only, ) sentinel = _ThreadSentinel() stale = self._cons.add(threading.get_ident(), sentinel, con) @@ -1377,9 +1493,15 @@ def _transact(self, retry=False, filename=None): if tid == txn_id: begin = False else: + # A read-only cache cannot take the reserved lock that + # BEGIN IMMEDIATE asks for; its transactions only read. + begin_statement = ( + 'BEGIN' if self._read_only else 'BEGIN IMMEDIATE' + ) + while True: try: - sql('BEGIN IMMEDIATE') + sql(begin_statement) begin = True self._txn_id = tid break @@ -1392,11 +1514,19 @@ def _transact(self, retry=False, filename=None): try: yield sql, filenames.append - except BaseException: + except BaseException as error: if begin: assert self._txn_id == tid self._txn_id = None sql('ROLLBACK') + if ( + self._read_only + and isinstance(error, sqlite3.OperationalError) + and 'readonly' in str(error) + ): + raise ReadOnlyError( + 'cache in %r is read-only' % self._directory + ) from error raise else: if begin: @@ -1427,6 +1557,7 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() now = time.time() db_key, raw = self._disk.put(key) expire_time = None if expire is None else now + expire @@ -1599,6 +1730,7 @@ def touch(self, key, expire=None, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() now = time.time() db_key, raw = self._disk.put(key) expire_time = None if expire is None else now + expire @@ -1647,6 +1779,7 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() now = time.time() db_key, raw = self._disk.put(key) expire_time = None if expire is None else now + expire @@ -1701,6 +1834,7 @@ def incr(self, key, delta=1, default=0, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() now = time.time() db_key, raw = self._disk.put(key) select = ( @@ -1817,6 +1951,13 @@ def get( """ db_key, raw = self._disk.put(key) update_column = EVICTION_POLICY[self.eviction_policy]['get'] + statistics = self.statistics + + if self._read_only: + # Hits, misses and access times cannot be recorded, so take + # the path that does not write. + update_column = None + statistics = False select = ( 'SELECT rowid, expire_time, tag, mode, filename, value, store_time' ' FROM Cache WHERE key = ? AND raw = ?' @@ -1825,7 +1966,7 @@ def get( default = _meta_result(default, expire_time, tag, store_time) - if not self.statistics and update_column is None: + if not statistics and update_column is None: # Fast path, no transaction necessary. rows = self._sql(select, (db_key, raw, time.time())).fetchall() @@ -1863,7 +2004,7 @@ def get( rows = sql(select, (db_key, raw, time.time())).fetchall() if not rows: - if self.statistics: + if statistics: sql(cache_miss) return default @@ -1883,11 +2024,11 @@ def get( value = self._disk.fetch(mode, filename, db_value, read) except IOError: # Key was deleted before we could retrieve result. - if self.statistics: + if statistics: sql(cache_miss) return default - if self.statistics: + if statistics: sql(cache_hit) now = time.time() @@ -1971,6 +2112,7 @@ def pop( :raises Timeout: if database timeout occurs """ + self._check_writable() db_key, raw = self._disk.put(key) select = ( 'SELECT rowid, expire_time, tag, mode, filename, value' @@ -2023,6 +2165,7 @@ def __delitem__(self, key, retry=True): :raises Timeout: if database timeout occurs """ + self._check_writable() db_key, raw = self._disk.put(key) with self._transact(retry) as (sql, cleanup): @@ -2114,6 +2257,7 @@ def push( :raises Timeout: if database timeout occurs """ + self._check_writable() if prefix is None: min_key = 0 max_key = 999999999999999 @@ -2223,6 +2367,7 @@ def pull( :raises Timeout: if database timeout occurs """ + self._check_writable() # Caution: Nearly identical code exists in Cache.peek if prefix is None: min_key = 0 @@ -2347,11 +2492,12 @@ def peek( max_key = prefix + '-999999999999999' order = {'front': 'ASC', 'back': 'DESC'} + live = ' AND ' + _LIVE_CONDITION if self._read_only else '' select = ( 'SELECT rowid, key, expire_time, tag, mode, filename, value' - ' FROM Cache WHERE ? < key AND key < ? AND raw = 1' + ' FROM Cache WHERE ? < key AND key < ? AND raw = 1%s' ' ORDER BY key %s LIMIT 1' - ) % order[side] + ) % (live, order[side]) if expire_time and tag: default = default, None, None @@ -2370,7 +2516,9 @@ def peek( (rowid, key, db_expire, db_tag, mode, name, db_value), ) = rows - if db_expire is not None and db_expire < time.time(): + expired = db_expire is not None and db_expire < time.time() + + if expired and not self._read_only: sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) cleanup(name) else: @@ -2420,10 +2568,11 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): """ order = ('ASC', 'DESC') + live = ' WHERE ' + _LIVE_CONDITION if self._read_only else '' select = ( 'SELECT rowid, key, raw, expire_time, tag, mode, filename, value' - ' FROM Cache ORDER BY rowid %s LIMIT 1' - ) % order[last] + ' FROM Cache%s ORDER BY rowid %s LIMIT 1' + ) % (live, order[last]) while True: while True: @@ -2446,7 +2595,9 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): ), ) = rows - if db_expire is not None and db_expire < time.time(): + expired = db_expire is not None and db_expire < time.time() + + if expired and not self._read_only: sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) cleanup(name) else: @@ -2588,6 +2739,8 @@ def check(self, fix=False, retry=False): :raises Timeout: if database timeout occurs """ + if fix: + self._check_writable() # pylint: disable=access-member-before-definition,W0201 with warnings.catch_warnings(record=True) as warns: sql = self._sql @@ -2710,6 +2863,7 @@ def create_tag_index(self): :raises Timeout: if database timeout occurs """ + self._check_writable() sql = self._sql sql( 'CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid) ' @@ -2723,6 +2877,7 @@ def drop_tag_index(self): :raises Timeout: if database timeout occurs """ + self._check_writable() sql = self._sql sql('DROP INDEX IF EXISTS Cache_tag_rowid') self.reset('tag_index', 0) @@ -2746,6 +2901,7 @@ def evict(self, tag, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() select = ( 'SELECT rowid, filename FROM Cache' ' WHERE tag = ? AND rowid > ?' @@ -2773,6 +2929,7 @@ def expire(self, now=None, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() select = ( 'SELECT rowid, expire_time, filename FROM Cache' ' WHERE ? < expire_time AND expire_time < ?' @@ -2799,6 +2956,7 @@ def cull(self, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() now = time.time() # Remove expired items. @@ -2854,6 +3012,7 @@ def clear(self, retry=False): :raises Timeout: if database timeout occurs """ + self._check_writable() select = ( 'SELECT rowid, filename FROM Cache' ' WHERE rowid > ?' @@ -3005,6 +3164,10 @@ def stats(self, enable=True, reset=False): # pylint: disable=E0203,W0201 result = (self.reset('hits'), self.reset('misses')) + if self._read_only: + # Counters cannot be reset or enabled on a read-only cache. + return result + if reset: self.reset('hits', 0) self.reset('misses', 0) @@ -3103,17 +3266,27 @@ def reset(self, key, value=ENOVAL, update=True): if value is ENOVAL: select = 'SELECT value FROM Settings WHERE key = ?' - ((value,),) = sql_retry(select, (key,)).fetchall() + rows = sql_retry(select, (key,)).fetchall() + if not rows and self._read_only and key in METADATA: + value = METADATA[key] # A snapshot without counters. + else: + ((value,),) = rows setattr(self, key, value) return value if update: + self._check_writable() statement = 'UPDATE Settings SET value = ? WHERE key = ?' sql_retry(statement, (value, key)) if key.startswith('sqlite_'): pragma = key[7:] + if self._read_only and pragma in _PERSISTENT_PRAGMAS: + # Stored in the file; a read-only cache takes what is there. + setattr(self, key, value) + return value + # 2016-02-17 GrantJ - PRAGMA and isolation_level=None # don't always play nicely together. Retry setting the # PRAGMA. I think some PRAGMA statements expect to diff --git a/docs/api.rst b/docs/api.rst index 8142f8a..ad81aa2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -150,3 +150,10 @@ Timeout ------- .. autoexception:: diskcache.Timeout + +ReadOnlyError +------------- + +Read the :ref:`read-only tutorial ` for details. + +.. autoexception:: diskcache.ReadOnlyError diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 858a72f..1f7cedf 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -961,6 +961,59 @@ directory with the default :class:`Disk ` must be trusted, and a reader using :class:`SafeDisk ` raises :exc:`pickle.UnpicklingError` on those rows. +.. _tutorial-read-only: + +Read-Only Caches +---------------- + +Opening an existing cache writes nothing to it: settings are written only +when they differ from the stored ones, and the schema, triggers and tag index +are left alone when they already match. A cache directory kept as a test +fixture therefore stays byte-for-byte identical across runs, as long as the +eviction policy does not record access times and statistics are off. + +A cache whose database file or directory is not writable opens read-only +without any argument, so a cache baked into a read-only container layer or a +read-only mount works for reads. The same can be requested for a writable +directory: + + >>> cache = Cache('/tmp/mycachedir') + >>> cache['key'] = 'value' + >>> cache.close() + >>> snapshot = Cache('/tmp/mycachedir', read_only=True) + >>> snapshot.read_only + True + >>> snapshot['key'] + 'value' + +Reads, iteration, ``len``, :meth:`volume `, +:meth:`check `, memoized calls that hit and the +read-only methods of :class:`FanoutCache `, +:class:`Deque ` and :class:`Index ` all +work. Every method that would write raises :exc:`ReadOnlyError +`, a subclass of :exc:`sqlite3.OperationalError`, +before anything is written: + + >>> snapshot['other'] = 'value' + Traceback (most recent call last): + ... + diskcache.core.ReadOnlyError: cache in '/tmp/mycachedir' is read-only + >>> snapshot.close() + +A read-only cache cannot record hits, misses or access times, so +:meth:`stats ` returns the stored counters without +enabling or resetting them, the ``least-recently-used`` and +``least-frequently-used`` policies read without updating, and expired items +are skipped instead of removed. The ``read_only`` argument is not a setting +and is never persisted; passing it through ``FanoutCache(directory, +read_only=True)`` or the Django ``OPTIONS`` works the same way. + +Two details of the file system. When the directory is writable, SQLite may +create the ``cache.db-shm`` and ``cache.db-wal`` files next to a read-only +database while reading it. When the directory is not writable, the database +is opened in SQLite's immutable mode, which skips those files and all +locking; that is safe only because nothing can write to that directory. + Caveats ------- diff --git a/tests/test_readonly.py b/tests/test_readonly.py new file mode 100644 index 0000000..ce99219 --- /dev/null +++ b/tests/test_readonly.py @@ -0,0 +1,400 @@ +"""Test read-only caches and reopening without writes (upstream issue 113).""" + +import hashlib +import os +import os.path as op +import shutil +import sqlite3 +import stat +import tempfile + +import pytest + +import diskcache as dc + +POSIX = os.name != 'nt' +ROOT = getattr(os, 'geteuid', lambda: 1)() == 0 +needs_permissions = pytest.mark.skipif( + not POSIX or ROOT, reason='needs POSIX permissions that bind' +) +BIG = b'x' * 70000 # Above disk_min_file_size, so stored in a file. + + +def square(value): + """Memoized by the tests; the cache key is derived from this name.""" + return value * value + + +def sha256(path): + with open(path, 'rb') as reader: + return hashlib.sha256(reader.read()).hexdigest() + + +def listing(directory): + return sorted( + name + for name in os.listdir(directory) + if name.startswith(dc.core.DBNAME) + ) + + +def populate(cache): + """Store every value kind, tags, expired rows, a queue and a memoized call.""" + cache['text'] = 'text' + cache[b'bytes'] = b'bytes' + cache['float'] = 1.5 + cache['pickle'] = {'a': (1, 2)} + cache['big'] = BIG + cache.set('tagged', 1, tag='tag') + cache.set('expired', 1, expire=-1) + cache.push('q0', expire=-1) + cache.push('q1') + cache.push('q2', expire=-1) + cache.memoize()(square)(3) + return len(cache) + + +def check_reads(cache, count, peek=True): + assert cache['text'] == 'text' + assert cache[b'bytes'] == b'bytes' + assert cache['float'] == 1.5 + assert cache['pickle'] == {'a': (1, 2)} + assert cache['big'] == BIG + assert cache.get('tagged', tag=True) == (1, 'tag') + assert cache.get('expired') is None + assert 'text' in cache + assert 'missing' not in cache + assert len(cache) == count + assert cache.volume() > 0 + assert cache.check() == [] + assert len(list(cache.iterkeys())) == count + assert len(list(cache)) == count + + if peek: # Expired queue items are skipped rather than removed. + assert cache.peek()[1] == 'q1' + assert cache.peek(side='back')[1] == 'q1' + assert cache.peekitem(last=False)[0] == 'text' + + memoized = cache.memoize()(square) + assert cache.get(memoized.__cache_key__(3)) == 9 + assert memoized(3) == 9 + + +def check_writes_raise(cache): + memoized = cache.memoize()(square) + attempts = [ + lambda: cache.set('new', 1), + lambda: cache.__setitem__('new', 1), + lambda: cache.add('new', 1), + lambda: cache.incr('counter'), + lambda: cache.decr('counter'), + lambda: cache.touch('text'), + lambda: cache.pop('text'), + lambda: cache.__delitem__('text'), + lambda: cache.delete('text'), + lambda: cache.push('new'), + cache.pull, + lambda: cache.evict('tag'), + cache.expire, + cache.cull, + cache.clear, + cache.create_tag_index, + cache.drop_tag_index, + lambda: cache.check(fix=True), + lambda: cache.reset('size_limit', 5), + lambda: memoized(4), + ] + + for attempt in attempts: + with pytest.raises(dc.ReadOnlyError): + attempt() + + +@pytest.fixture +def directory(): + path = tempfile.mkdtemp() + yield path + for dirpath, dirnames, filenames in os.walk(path): + os.chmod(dirpath, 0o755) + for name in filenames: + os.chmod(op.join(dirpath, name), 0o644) + shutil.rmtree(path, ignore_errors=True) + + +def test_reopen_does_not_write(directory): + with dc.Cache(directory, cull_limit=0) as cache: + count = populate(cache) + + assert listing(directory) == [dc.core.DBNAME] + path = op.join(directory, dc.core.DBNAME) + digest, mtime = sha256(path), os.stat(path).st_mtime_ns + + for _ in range(2): + with dc.Cache(directory) as cache: + assert not cache.read_only + check_reads(cache, count, peek=False) # peek removes expired rows + + assert listing(directory) == [dc.core.DBNAME] + assert sha256(path) == digest + assert os.stat(path).st_mtime_ns == mtime + + +def test_reopen_writes_only_changed_settings(directory): + with dc.Cache(directory) as cache: + cache['a'] = 1 + + path = op.join(directory, dc.core.DBNAME) + digest = sha256(path) + + with dc.Cache(directory, size_limit=123) as cache: + assert cache.size_limit == 123 + + assert sha256(path) != digest + + with dc.Cache(directory) as cache: + assert cache.size_limit == 123 + + +def test_reopen_tag_index_only_when_changed(directory): + with dc.Cache(directory, tag_index=True) as cache: + cache.set('a', 1, tag='t') + + path = op.join(directory, dc.core.DBNAME) + digest = sha256(path) + + with dc.Cache(directory) as cache: + assert cache.tag_index == 1 + + assert sha256(path) == digest + + with dc.Cache(directory, tag_index=False) as cache: + assert cache.tag_index == 0 + + assert sha256(path) != digest + + +def test_statistics_write_on_read_but_not_when_read_only(directory): + with dc.Cache(directory, statistics=1) as cache: + cache['a'] = 1 + + path = op.join(directory, dc.core.DBNAME) + digest = sha256(path) + + with dc.Cache(directory) as cache: + assert cache['a'] == 1 + assert cache.get('missing') is None + + # Hits and misses are counters in the database, so reads write. + assert sha256(path) != digest + + with dc.Cache(directory) as cache: + assert cache.stats() == (1, 1) + + with dc.Cache(directory, read_only=True) as cache: + assert cache['a'] == 1 + assert cache.get('missing') is None + assert cache.stats(enable=False, reset=True) == (1, 1) + + with dc.Cache(directory) as cache: + assert cache.stats() == (1, 1) + + +def test_explicit_read_only(directory): + with dc.Cache(directory, cull_limit=0) as cache: + count = populate(cache) + + path = op.join(directory, dc.core.DBNAME) + digest = sha256(path) + + with dc.Cache(directory, read_only=True) as cache: + assert cache.read_only + check_reads(cache, count) + check_writes_raise(cache) + + assert sha256(path) == digest + + with dc.Cache(directory) as cache: + assert not cache.read_only + cache['new'] = 1 + del cache['new'] + + +def test_read_only_requires_database(directory): + with pytest.raises(dc.ReadOnlyError): + dc.Cache(directory, read_only=True) + + assert not op.exists(op.join(directory, dc.core.DBNAME)) + + +def test_read_only_error_is_operational_error(directory): + with dc.Cache(directory) as cache: + cache['a'] = 1 + + with dc.Cache(directory, read_only=True) as cache: + with pytest.raises(sqlite3.OperationalError): + cache['b'] = 2 + + +def test_read_only_eviction_policies(directory): + for policy in dc.EVICTION_POLICY: + subdir = op.join(directory, policy) + + with dc.Cache(subdir, eviction_policy=policy) as cache: + cache['a'] = 1 + + with dc.Cache(subdir, read_only=True) as cache: + assert cache['a'] == 1 + assert cache.get('a', expire_time=True) == (1, None) + + +def test_read_only_transaction_write_raises(directory): + with dc.Cache(directory) as cache: + cache['a'] = 1 + + with dc.Cache(directory, read_only=True) as cache: + with pytest.raises(dc.ReadOnlyError): + with cache.transact(): + cache._sql('DELETE FROM Cache') + + assert cache['a'] == 1 + + +def test_read_only_file_attribute(directory): + with dc.Cache(directory, cull_limit=0) as cache: + count = populate(cache) + + path = op.join(directory, dc.core.DBNAME) + os.chmod(path, stat.S_IREAD) + digest = sha256(path) + + try: + with dc.Cache(directory) as cache: + assert cache.read_only + check_reads(cache, count) + check_writes_raise(cache) + finally: + os.chmod(path, stat.S_IREAD | stat.S_IWRITE) + + assert sha256(path) == digest + + with dc.Cache(directory) as cache: + assert not cache.read_only + + +@needs_permissions +def test_read_only_directory(directory): + with dc.Cache(directory, cull_limit=0) as cache: + count = populate(cache) + + path = op.join(directory, dc.core.DBNAME) + digest = sha256(path) + files = sorted(os.listdir(directory)) + os.chmod(path, 0o444) + os.chmod(directory, 0o555) + + try: + with dc.Cache(directory) as cache: + assert cache.read_only + check_reads(cache, count) + check_writes_raise(cache) + finally: + os.chmod(directory, 0o755) + os.chmod(path, 0o644) + + assert sha256(path) == digest + assert sorted(os.listdir(directory)) == files + + +@needs_permissions +def test_read_only_directory_without_settings(directory): + path = op.join(directory, dc.core.DBNAME) + sqlite3.connect(path).close() + os.chmod(path, 0o444) + os.chmod(directory, 0o555) + + try: + with pytest.raises(dc.ReadOnlyError): + dc.Cache(directory) + finally: + os.chmod(directory, 0o755) + os.chmod(path, 0o644) + + +def test_read_only_fanout_deque_index(directory): + fanout_dir = op.join(directory, 'fanout') + deque_dir = op.join(directory, 'deque') + index_dir = op.join(directory, 'index') + + with dc.FanoutCache(fanout_dir, shards=2) as fanout: + fanout['a'] = 1 + fanout['big'] = BIG + + deque = dc.Deque('abc', deque_dir) + deque._cache.close() + index = dc.Index(index_dir, a=1, b=2) + index._cache.close() + + with dc.FanoutCache(fanout_dir, shards=2, read_only=True) as fanout: + assert all(shard.read_only for shard in fanout._shards) + assert fanout['a'] == 1 + assert fanout['big'] == BIG + assert len(fanout) == 2 + + with pytest.raises(dc.ReadOnlyError): + fanout['b'] = 2 + + deque = dc.Deque.fromcache(dc.Cache(deque_dir, read_only=True)) + assert list(deque) == ['a', 'b', 'c'] + assert deque[0] == 'a' + assert deque.peekleft() == 'a' + assert deque.peek() == 'c' + + with pytest.raises(dc.ReadOnlyError): + deque.append('d') + + deque._cache.close() + + index = dc.Index.fromcache(dc.Cache(index_dir, read_only=True)) + assert dict(index) == {'a': 1, 'b': 2} + assert index.peekitem() == ('b', 2) + + with pytest.raises(dc.ReadOnlyError): + index['c'] = 3 + + index._cache.close() + + +def test_read_only_django(directory): + pytest.importorskip('django') + from django.conf import settings + + if not settings.configured: + pytest.skip('Django settings are not configured') + + from diskcache import DjangoCache + + cache = DjangoCache(directory, {}) + cache.set('a', 1) + cache.close() + + cache = DjangoCache(directory, {'OPTIONS': {'read_only': True}}) + assert cache.get('a') == 1 + + with pytest.raises(dc.ReadOnlyError): + cache.set('b', 2) + + cache.close() + + +def test_read_only_without_metadata_rows(directory): + with dc.Cache(directory) as cache: + cache['a'] = 1 + cache._sql("DELETE FROM Settings WHERE key IN ('hits', 'misses')") + + with dc.Cache(directory, read_only=True) as cache: + assert cache['a'] == 1 + assert cache.stats() == (0, 0) + + with dc.Cache(directory) as cache: + assert cache.stats() == (0, 0) + assert cache.reset('hits') == 0