Add configure() and DISKCACHE_DISK so caches created without a disk use a process-wide default, with an explicit disk always winning

CHANGELOG.md

@@ -15,6 +15,10 @@ diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 20
 - The README is rewritten in Markdown around install, usage and the fork story; the comparison tables move to a new `comparisons` page in the documentation, which gets its own landing page, and the 2016 testimonials are dropped (PR #19).
 - The documentation is published on Read the Docs at https://diskcache-community.readthedocs.io/, built from `master` on every push; the README links point there instead of at the `.rst` sources (PR #18).
 
+### Features
+
+- `configure(disk)` sets the disk type of every `Cache`, `FanoutCache`, `Deque`, `Index` and `DjangoCache` created afterwards without a disk, and `default_disk()` returns the type in effect. Without `configure`, the environment decides: `DISKCACHE_DISK=safe` selects `SafeDisk`, with `DISKCACHE_SAFE_TYPES` and `DISKCACHE_SAFE_MODULES` passed to `SafeDisk.with_types`, and `DISKCACHE_DISK` also takes `default` or the import path of a disk class. A disk passed explicitly always wins, so libraries that choose their own disk keep it; caches that already exist keep theirs, and nothing is persisted. The `disk` arguments now default to `None`, meaning that type (PR #24).
+
 ### Packaging
 
 - **Breaking:** the package is imported as `diskcache_community` instead of `diskcache`, so it installs next to the upstream `diskcache` distribution without overwriting it, and a project can move to the fork while other dependencies still require `diskcache`. The public API, the `cache.db` schema, the file layout and the serialization are unchanged, so cache directories are shared with 5.6.3 in both directions. Django settings name `diskcache_community.DjangoCache` and `diskcache_community.SafeDisk`. The import-time warning about an upstream `diskcache` installed in the same environment is gone, and a CI job installs both distributions together, runs the golden tests in both directions and checks that uninstalling either leaves the other working (PR #22).

diskcache_community/__init__.py

@@ -18,6 +18,8 @@
     SafeDisk,
     Timeout,
     UnknownFileWarning,
+    configure,
+    default_disk,
 )
 from .fanout import FanoutCache
 from .persistent import Deque, Index
@@ -52,6 +54,8 @@
     'UNKNOWN',
     'UnknownFileWarning',
     'barrier',
+    'configure',
+    'default_disk',
     'memoize_stampede',
     'throttle',
 ]

diskcache_community/core.py

@@ -990,6 +990,138 @@ def remove(self, file_path):
         super().remove(file_path)
 
 
+ENV_DISK = 'DISKCACHE_DISK'
+ENV_SAFE_TYPES = 'DISKCACHE_SAFE_TYPES'
+ENV_SAFE_MODULES = 'DISKCACHE_SAFE_MODULES'
+
+_configured_disk = None  # pylint: disable=invalid-name
+
+
+def _check_disk_type(disk, source='disk'):
+    if not (isinstance(disk, type) and issubclass(disk, Disk)):
+        raise ValueError(
+            '%s must subclass diskcache_community.Disk, not %r'
+            % (source, disk)
+        )
+
+
+def configure(disk):
+    """Set the disk type of caches created without a `disk` argument.
+
+    The configured type applies to every :class:`Cache`, :class:`FanoutCache
+    <diskcache_community.FanoutCache>`, :class:`Deque
+    <diskcache_community.Deque>`, :class:`Index <diskcache_community.Index>`
+    and :class:`DjangoCache <diskcache_community.DjangoCache>` created
+    afterwards in this process that is not given a disk. A disk passed
+    explicitly always wins, and caches that already exist keep theirs.
+    ``configure(None)`` removes the setting, so the environment variables
+    apply again. The choice is never persisted in a cache directory.
+
+    >>> previous = configure(SafeDisk)
+    >>> type(Cache().disk) is SafeDisk
+    True
+    >>> configure(previous) is SafeDisk
+    True
+
+    :param disk: Disk type or subclass, or None
+    :return: the previously configured disk type, or None
+    :raises ValueError: if `disk` is not a Disk subclass
+
+    """
+    global _configured_disk  # pylint: disable=global-statement
+
+    if disk is not None:
+        _check_disk_type(disk)
+
+    previous, _configured_disk = _configured_disk, disk
+    return previous
+
+
+def default_disk():
+    """Return the disk type for a cache created without a `disk` argument.
+
+    That is the type set with :func:`configure`, else the type selected by
+    the environment variables, else :class:`Disk`.
+
+    ``DISKCACHE_DISK`` is empty or ``default`` for :class:`Disk`, ``safe``
+    for :class:`SafeDisk`, or the import path of a Disk subclass, such as
+    ``myapp.cache:AppDisk``. With ``safe``, ``DISKCACHE_SAFE_TYPES`` lists
+    classes to allow by import path and ``DISKCACHE_SAFE_MODULES`` lists
+    module names whose classes to allow, both comma-separated, as
+    :meth:`SafeDisk.with_types` takes them.
+
+    :return: Disk type or subclass
+    :raises ValueError: if an environment variable is invalid
+
+    """
+    if _configured_disk is not None:
+        return _configured_disk
+
+    environ = os.environ
+    return _disk_from_environment(
+        environ.get(ENV_DISK, ''),
+        environ.get(ENV_SAFE_TYPES, ''),
+        environ.get(ENV_SAFE_MODULES, ''),
+    )
+
+
+def _split_list(value):
+    return [entry.strip() for entry in value.split(',') if entry.strip()]
+
+
+def _import_object(path, source):
+    """Return the object at import path `path`, module:name or module.name."""
+    if ':' in path:
+        module_name, _, name = path.partition(':')
+    else:
+        module_name, _, name = path.rpartition('.')
+
+    try:
+        if not module_name or not name:
+            raise ImportError(path)
+        __import__(module_name)
+        obj = sys.modules[module_name]
+        for part in name.split('.'):
+            obj = getattr(obj, part)
+    except (ImportError, AttributeError) as error:
+        raise ValueError(
+            '%s: cannot import %r: %s' % (source, path, error)
+        ) from None
+
+    return obj
+
+
+@ft.lru_cache(maxsize=16)
+def _disk_from_environment(name, types, modules):
+    """Return the disk type that the environment variable values select."""
+    name = name.strip()
+    keyword = name.lower()
+    extras = types.strip() or modules.strip()
+
+    if keyword == 'safe':
+        classes = [
+            _import_object(entry, ENV_SAFE_TYPES)
+            for entry in _split_list(types)
+        ]
+        prefixes = _split_list(modules)
+        if not (classes or prefixes):
+            return SafeDisk
+        return SafeDisk.with_types(*classes, modules=prefixes)
+
+    if extras:
+        raise ValueError(
+            '%s and %s apply only with %s=safe'
+            % (ENV_SAFE_TYPES, ENV_SAFE_MODULES, ENV_DISK)
+        )
+
+    if keyword in ('', 'default'):
+        return Disk
+
+    disk = _import_object(name, ENV_DISK)
+    _check_disk_type(disk, ENV_DISK)
+    return disk
+
+
 def _meta_result(value, expire_time, tag, store_time, meta=(None, None, None)):
     """Return `value`, or a tuple of `value` and the requested metadata.
 
@@ -1329,7 +1461,7 @@ def __init__(
         self,
         directory=None,
         timeout=60,
-        disk=Disk,
+        disk=None,
         *,
         read_only=False,
         **settings
@@ -1338,7 +1470,8 @@ def __init__(
 
         :param str directory: cache directory
         :param float timeout: SQLite connection timeout
-        :param disk: Disk type or subclass for serialization
+        :param disk: Disk type or subclass for serialization (default None,
+            the type :func:`default_disk` returns)
         :param bool read_only: open the cache read-only (default False)
         :param settings: any of DEFAULT_SETTINGS
 
@@ -1356,6 +1489,9 @@ def __init__(
         <tutorial-read-only>` for details.
 
         """
+        if disk is None:
+            disk = default_disk()
+
         try:
             assert issubclass(disk, Disk)
         except (TypeError, AssertionError):

diskcache_community/djangocache.py

@@ -11,7 +11,7 @@
     # For older versions of Django simply use 300 seconds.
     DEFAULT_TIMEOUT = 300
 
-from .core import ENOVAL, Disk, args_to_key, full_name, resolve_ignore
+from .core import ENOVAL, args_to_key, full_name, resolve_ignore
 from .fanout import FanoutCache
 
 
@@ -23,10 +23,12 @@ def __init__(self, directory, params):
 
         The `params` mapping holds the keys of the Django ``CACHES`` setting.
         ``SHARDS`` and ``DATABASE_TIMEOUT`` configure the underlying
-        :class:`FanoutCache <diskcache_community.FanoutCache>`, ``DISK`` selects the
-        :class:`Disk <diskcache_community.Disk>` type or subclass used for serialization
-        as a class or a dotted import path such as ``'diskcache_community.SafeDisk'``,
-        and ``OPTIONS`` passes further settings through.
+        :class:`FanoutCache <diskcache_community.FanoutCache>`, ``DISK``
+        selects the :class:`Disk <diskcache_community.Disk>` type or subclass
+        used for serialization as a class or a dotted import path such as
+        ``'diskcache_community.SafeDisk'``, by default the type
+        :func:`default_disk <diskcache_community.default_disk>` returns, and
+        ``OPTIONS`` passes further settings through.
 
         :param str directory: cache directory
         :param dict params: cache parameters
@@ -36,7 +38,7 @@ def __init__(self, directory, params):
         shards = params.get('SHARDS', 8)
         timeout = params.get('DATABASE_TIMEOUT', 0.010)
         options = dict(params.get('OPTIONS', {}))
-        disk = params.get('DISK', options.pop('disk', Disk))
+        disk = params.get('DISK', options.pop('disk', None))
         if isinstance(disk, str):
             disk = import_string(disk)
         self._cache = FanoutCache(

diskcache_community/fanout.py

@@ -9,7 +9,14 @@
 import tempfile
 import time
 
-from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout, _meta_result
+from .core import (
+    DEFAULT_SETTINGS,
+    ENOVAL,
+    Cache,
+    Timeout,
+    _meta_result,
+    default_disk,
+)
 from .persistent import Deque, Index
 
 
@@ -22,21 +29,26 @@ class FanoutCache:
     """Cache that shards keys and values."""
 
     def __init__(
-        self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings
+        self, directory=None, shards=8, timeout=0.010, disk=None, **settings
     ):
         """Initialize cache instance.
 
         :param str directory: cache directory
         :param int shards: number of shards to distribute writes
         :param float timeout: SQLite connection timeout
-        :param disk: `Disk` instance for serialization
+        :param disk: `Disk` type or subclass for serialization (default None,
+            the type :func:`default_disk
+            <diskcache_community.default_disk>` returns)
         :param settings: any of `DEFAULT_SETTINGS`
 
         The cache directory, the shard directories and the directories of
         sub-caches, deques and indexes are created with mode ``0o700`` like
         :class:`Cache` directories; existing directories keep their mode.
 
         """
+        if disk is None:
+            disk = default_disk()
+
         if directory is None:
             directory = tempfile.mkdtemp(prefix='diskcache-')
         directory = str(directory)

diskcache_community/persistent.py

@@ -13,7 +13,7 @@
 from contextlib import contextmanager
 from shutil import rmtree
 
-from .core import ENOVAL, Cache, Disk
+from .core import ENOVAL, Cache
 
 
 def _make_compare(seq_op, doc):
@@ -75,7 +75,7 @@ class Deque(Sequence):
 
     """
 
-    def __init__(self, iterable=(), directory=None, maxlen=None, disk=Disk):
+    def __init__(self, iterable=(), directory=None, maxlen=None, disk=None):
         """Initialize deque instance.
 
         If directory is None then temporary directory created. The directory
@@ -84,8 +84,9 @@ def __init__(self, iterable=(), directory=None, maxlen=None, disk=Disk):
         :param iterable: iterable of items to append to deque
         :param directory: deque directory (default None)
         :param maxlen: max length (default None, no max)
-        :param disk: `Disk` type or subclass for serialization (default
-            :class:`Disk <diskcache_community.Disk>`)
+        :param disk: `Disk` type or subclass for serialization (default None,
+            the type :func:`default_disk <diskcache_community.default_disk>`
+            returns)
 
         """
         self._cache = Cache(directory, disk=disk, eviction_policy='none')
@@ -682,7 +683,7 @@ class Index(MutableMapping):
     where items are stored.
 
     Hashing protocol is not used. Keys are looked up by their serialized
-    format. See ``diskcache.Disk`` for details.
+    format. See ``diskcache_community.Disk`` for details.
 
     >>> index = Index()
     >>> index.update([('a', 1), ('b', 2), ('c', 3)])
@@ -698,16 +699,18 @@ class Index(MutableMapping):
 
     """
 
-    def __init__(self, *args, disk=Disk, **kwargs):
+    def __init__(self, *args, disk=None, **kwargs):
         """Initialize index in directory and update items.
 
         Optional first argument may be string specifying directory where items
         are stored. When None or not given, temporary directory is created.
 
-        The keyword argument `disk` selects the :class:`Disk <diskcache_community.Disk>`
-        type or subclass used for serialization. Every other keyword argument
-        is stored as an item, so an item named ``disk`` has to be set after
-        initialization.
+        The keyword argument `disk` selects the :class:`Disk
+        <diskcache_community.Disk>` type or subclass used for serialization,
+        by default the type :func:`default_disk
+        <diskcache_community.default_disk>` returns. Every other keyword
+        argument is stored as an item, so an item named ``disk`` has to be set
+        after initialization.
 
         >>> index = Index({'a': 1, 'b': 2, 'c': 3})
         >>> len(index)

docs/api.rst

@@ -157,6 +157,15 @@ Read the :ref:`SafeDisk tutorial <tutorial-safedisk>` for details.
    <diskcache_community.SafeDisk.with_types>` refuses to allow, because
    unpickling them is not memory-safe.
 
+Default disk
+------------
+
+Read the :ref:`default disk tutorial <tutorial-default-disk>` for details.
+
+.. autofunction:: diskcache_community.configure
+
+.. autofunction:: diskcache_community.default_disk
+
 Timeout
 -------