clone cms with menus, make menus standalone app

cms/cache/__init__.py

@@ -0,0 +1,87 @@
+import re
+
+from cms.utils.conf import get_cms_setting
+
+CMS_PAGE_CACHE_VERSION_KEY = get_cms_setting("CACHE_PREFIX") + '_PAGE_CACHE_VERSION'
+
+
+def _get_cache_version():
+    """
+    Returns the current page cache version, explicitly setting one if not
+    defined.
+    """
+    from django.core.cache import cache
+
+    version = cache.get(CMS_PAGE_CACHE_VERSION_KEY)
+
+    if version:
+        return version
+    else:
+        _set_cache_version(1)
+        return 1
+
+
+def _set_cache_version(version):
+    """
+    Set the cache version to the specified value.
+    """
+    from django.core.cache import cache
+
+    cache.set(
+        CMS_PAGE_CACHE_VERSION_KEY,
+        version,
+        get_cms_setting('CACHE_DURATIONS')['content']
+    )
+
+
+def invalidate_cms_page_cache():
+    """
+    Invalidates the CMS PAGE CACHE.
+    """
+
+    #
+    # NOTE: We're using a cache versioning strategy for invalidating the page
+    # cache when necessary. Instead of wiping all the old entries, we simply
+    # increment the version number rendering all previous entries
+    # inaccessible and left to expire naturally.
+    #
+    # ALSO NOTE: According to the Django documentation, a timeout value of
+    # `None' (in version 1.6+) is supposed to mean "cache forever", however,
+    # this is actually only implemented as only slightly less than 30 days in
+    # some backends (memcached, in particular). In older Djangos, `None' means
+    # "use default value".  To avoid issues arising from different Django
+    # versions and cache backend implementations, we will explicitly set the
+    # lifespan of the CMS_PAGE_CACHE_VERSION entry to whatever is set in
+    # settings.CACHE_DURATIONS['content']. This allows users to adjust as
+    # necessary for their backend.
+    #
+    # To prevent writing cache entries that will live longer than our version
+    # key, we will always re-write the current version number into the cache
+    # just after we write any new cache entries, thus ensuring that the
+    # version number will always outlive any entries written against that
+    # version. This is a cheap operation.
+    #
+    # If there are no new cache writes before the version key expires, its
+    # perfectly OK, since any previous entries cached against that version
+    # will have also expired, so, it'd be pointless to try to access them
+    # anyway.
+    #
+    version = _get_cache_version()
+    _set_cache_version(version + 1)
+
+
+CLEAN_KEY_PATTERN = re.compile(r'[^a-zA-Z0-9_-]')
+
+
+def _clean_key(key):
+    return CLEAN_KEY_PATTERN.sub('-', key)
+
+
+def _get_cache_key(name, page_lookup, lang, site_id):
+    from cms.models import Page
+    if isinstance(page_lookup, Page):
+        page_key = str(page_lookup.pk)
+    else:
+        page_key = str(page_lookup)
+    page_key = _clean_key(page_key)
+    return f'{get_cms_setting("CACHE_PREFIX")}{name}__page_lookup:{page_key}_site:{str(site_id)}_lang:{str(lang)}'

cms/cache/choices.py

@@ -0,0 +1,30 @@
+from django.conf import settings
+
+from cms.utils.conf import get_cms_setting
+
+
+def _site_cache_key(lang):
+    return "{}-{}".format(get_cms_setting('SITE_CHOICES_CACHE_KEY'), lang)
+
+
+def _page_cache_key(lang):
+    return "{}-{}".format(get_cms_setting('PAGE_CHOICES_CACHE_KEY'), lang)
+
+
+def _clean_many(prefix):
+    from django.core.cache import cache
+    keys = []
+    if settings.USE_I18N:
+        for lang in [language[0] for language in settings.LANGUAGES]:
+            keys.append(f"{prefix}-{lang}")
+    else:
+        keys = [f"{prefix}-{settings.LANGUAGE_CODE}"]
+    cache.delete_many(keys)
+
+
+def clean_site_choices_cache(sender, **kwargs):
+    _clean_many(get_cms_setting('SITE_CHOICES_CACHE_KEY'))
+
+
+def clean_page_choices_cache(sender, **kwargs):
+    _clean_many(get_cms_setting('PAGE_CHOICES_CACHE_KEY'))

cms/cache/page.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import datetime
+import hashlib
+from collections.abc import Mapping
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+from django.conf import settings
+from django.utils.cache import (
+    add_never_cache_headers,
+    patch_response_headers,
+    patch_vary_headers,
+)
+from django.utils.encoding import iri_to_uri
+from django.utils.timezone import now
+
+from cms.cache import _get_cache_key, _get_cache_version, _set_cache_version
+from cms.constants import EXPIRE_NOW, MAX_EXPIRATION_TTL
+from cms.toolbar.utils import get_toolbar_from_request
+from cms.utils import get_current_site
+from cms.utils.compat.response import get_response_headers
+from cms.utils.conf import get_cms_setting
+from cms.utils.helpers import get_timezone_name
+from cms.utils.i18n import get_default_language_for_site
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from django.http import HttpRequest, HttpResponse
+
+    from cms.models import Page
+
+
+def _page_cache_key(request: HttpRequest, vary_on: Iterable[str] | None = None) -> str:
+    """
+    Generate a cache key based on the request path and language.
+    The language is determined following django-cms's language resolution order.
+
+    ``vary_on`` is an optional iterable of header names declared by plugins via
+    ``get_vary_cache_on()``. The request's values for those headers are always
+    folded into the key so that responses varying on those headers are cached
+    separately per value (instead of the first variant being served to
+    everyone). A missing (``None``) or empty ``vary_on`` hashes to a constant.
+    """
+    site = get_current_site(request)
+    if hasattr(request, "LANGUAGE_CODE"):
+        language = request.LANGUAGE_CODE
+    else:
+        language = get_default_language_for_site(site.pk)
+
+    cache_key = "%s:%d:%s:%s" % (
+        get_cms_setting("CACHE_PREFIX"),
+        site.pk,
+        language,
+        hashlib.sha1(iri_to_uri(request.get_full_path()).encode("utf-8")).hexdigest(),
+    )
+    if settings.USE_TZ:
+        cache_key += ".%s" % get_timezone_name()
+    cache_key += ".%s" % _vary_on_hash(request, vary_on or [])
+    return cache_key
+
+
+def _vary_on_hash(request: HttpRequest, vary_on: Iterable[str]) -> str:
+    """Hash of the request's values for the given (plugin-declared) headers."""
+    ctx = hashlib.sha1()
+    for header in sorted(header.lower() for header in vary_on):
+        # Mirror Django's translation of header names to ``request.META`` keys.
+        meta_key = "HTTP_" + header.upper().replace("-", "_")
+        value = request.META.get(meta_key, "")
+        ctx.update(("%s=%s&" % (header, iri_to_uri(value))).encode("utf-8"))
+    return ctx.hexdigest()
+
+
+def _page_vary_headers_cache_key(request: HttpRequest) -> str:
+    """Key under which the list of plugin-declared vary headers is stored.
+
+    The list cannot be known on a cache read until the page is rendered, so it
+    is persisted on write and looked up first on read (mirroring Django's
+    ``learn_cache_key`` / ``get_cache_key`` two-step approach).
+    """
+    return _page_cache_key(request) + ".vary-on"
+
+
+def set_page_cache(response: HttpResponse) -> HttpResponse:
+    """Store a rendered page response in the CMS page cache.
+
+    The response is only cached for anonymous requests when the page cache is
+    enabled and the toolbar has not disabled caching. The cache timeout is the
+    smallest of the configured ``content`` duration and every cache-able
+    placeholder's TTL (as returned by ``get_cache_expiration()``); if any
+    placeholder requests ``EXPIRE_NOW`` the response is not cached at all.
+
+    When the response is cacheable, expiration and ``Vary`` headers are patched
+    onto it and two entries are written: the list of plugin-declared vary
+    headers (see :func:`_page_vary_headers_cache_key`) and the response payload
+    (content, headers and absolute expiry timestamp) under the header-aware
+    content key. Otherwise ``never cache`` headers are added.
+
+    Returns the (possibly header-patched) ``response``.
+    """
+    from django.core.cache import cache
+
+    request = response._request
+    toolbar = get_toolbar_from_request(request)
+    is_authenticated = request.user.is_authenticated
+
+    if is_authenticated or toolbar._cache_disabled or not get_cms_setting("PAGE_CACHE"):
+        add_never_cache_headers(response)
+        return response
+
+    # This *must* be TZ-aware
+    timestamp = now()
+
+    placeholders = toolbar.content_renderer.get_rendered_placeholders()
+    # Checks if there's a plugin using the legacy "cache = False"
+    placeholder_ttl_list = []
+    vary_cache_on_set = set()
+    for ph in placeholders:
+        # get_cache_expiration() always returns:
+        #     EXPIRE_NOW <= int <= MAX_EXPIRATION_IN_SECONDS
+        ttl = ph.get_cache_expiration(request, timestamp)
+        vary_cache_on = ph.get_vary_cache_on(request)
+
+        placeholder_ttl_list.append(ttl)
+        if ttl and vary_cache_on:
+            # We're only interested in vary headers if they come from
+            # a cache-able placeholder.
+            vary_cache_on_set |= set(vary_cache_on)
+
+    if EXPIRE_NOW not in placeholder_ttl_list:
+        if placeholder_ttl_list:
+            min_placeholder_ttl = min(x for x in placeholder_ttl_list)
+        else:
+            # Should only happen when there are no placeholders at all
+            min_placeholder_ttl = MAX_EXPIRATION_TTL
+        ttl = min(get_cms_setting("CACHE_DURATIONS")["content"], min_placeholder_ttl)
+
+        if ttl > 0:
+            # Adds expiration, etc. to headers
+            vary_on = sorted(vary_cache_on_set)
+            patch_response_headers(response, cache_timeout=ttl)
+            patch_vary_headers(response, vary_on)
+
+            version = _get_cache_version()
+            # We also store the absolute expiration timestamp to avoid
+            # recomputing it on cache-reads.
+            expires_datetime = timestamp + timedelta(seconds=ttl)
+            response_headers = get_response_headers(response)
+            # Persist whether this page opted out of Django's clickjacking
+            # middleware. Only "Allow" pages (no X-Frame-Options header) set
+            # xframe_options_exempt; "Inherit" pages leave it unset so the
+            # middleware adds the site default. The read path must replay this
+            # decision -- unconditionally exempting a cached response would drop
+            # X-Frame-Options from every inherit-default page (clickjacking).
+            xframe_options_exempt = getattr(response, "xframe_options_exempt", False)
+            # Persist the list of plugin-declared vary headers so the read path
+            # can rebuild the same (header-value aware) content key.
+            cache.set(
+                _page_vary_headers_cache_key(request),
+                vary_on,
+                ttl,
+                version=version,
+            )
+            cache.set(
+                _page_cache_key(request, vary_on),
+                (
+                    response.content,
+                    response_headers,
+                    expires_datetime,
+                    xframe_options_exempt,
+                ),
+                ttl,
+                version=version,
+            )
+            # See note in invalidate_cms_page_cache()
+            _set_cache_version(version)
+    return response
+
+
+def get_page_cache(request: HttpRequest) -> tuple[bytes, Mapping[str, str], datetime] | None:
+    """Return the cached page response for ``request`` or ``None``.
+
+    Mirrors Django's two-step ``get_cache_key`` lookup: first the list of
+    plugin-declared vary headers is read, then the request's values for those
+    headers are folded into the content key so the matching variant is
+    returned. The cached value is the ``(content, headers, expires_datetime)``
+    tuple stored by :func:`set_page_cache`, or ``None`` on a cache miss.
+    """
+    from django.core.cache import cache
+
+    version = _get_cache_version()
+    # First resolve which headers (if any) the cached page varies on, then
+    # build the content key from this request's values for those headers.
+    vary_on = cache.get(_page_vary_headers_cache_key(request), version=version)
+    return cache.get(_page_cache_key(request, vary_on), version=version)
+
+
+def get_xframe_cache(page: Page) -> int | None:
+    """Return the cached ``X-Frame-Options`` value for ``page`` or ``None``."""
+    from django.core.cache import cache
+
+    return cache.get("cms:xframe_options:%s" % page.pk)
+
+
+def set_xframe_cache(page: Page, xframe_options: int) -> None:
+    """Cache the resolved ``X-Frame-Options`` value for ``page``.
+
+    The cache version is re-written afterwards so the version key always
+    outlives the entries written against it (see
+    :func:`cms.cache.invalidate_cms_page_cache`).
+    """
+    from django.core.cache import cache
+
+    cache.set("cms:xframe_options:%s" % page.pk, xframe_options, version=_get_cache_version())
+    _set_cache_version(_get_cache_version())
+
+
+def _page_url_key(
+    page_lookup: Any,
+    lang: str,
+    site_id: int,
+    extra_key: str | None,
+) -> str:
+    """Build the cache key for a page's absolute URL.
+
+    ``page_lookup`` is the untyped page reference accepted by the ``page_url``
+    template tag (a :class:`~cms.models.Page`, pk, reverse id, ...). When
+    ``extra_key`` is a string it is folded into the key so callers can cache
+    distinct URLs (e.g. per query string) for the same page.
+    """
+    return "".join([
+        _get_cache_key("page_url", page_lookup, lang, site_id),
+        f"_key:{extra_key}" if isinstance(extra_key, str) else "",
+        "_type:absolute_url",
+    ])
+
+
+def set_page_url_cache(
+    page_lookup: Any,
+    lang: str,
+    site_id: int,
+    url: str,
+    extra_key: str | None = None,
+) -> None:
+    """Cache the absolute ``url`` of a page for the ``content`` duration.
+
+    The cache version is re-written afterwards so the version key always
+    outlives the entries written against it (see
+    :func:`cms.cache.invalidate_cms_page_cache`).
+    """
+    from django.core.cache import cache
+
+    cache.set(
+        _page_url_key(page_lookup, lang, site_id, extra_key),
+        url,
+        get_cms_setting("CACHE_DURATIONS")["content"],
+        version=_get_cache_version(),
+    )
+    _set_cache_version(_get_cache_version())
+
+
+def get_page_url_cache(
+    page_lookup: Any,
+    lang: str,
+    site_id: int,
+    extra_key: str | None = None,
+) -> str | None:
+    """Return the cached absolute URL for a page lookup, or ``None`` on a miss."""
+    from django.core.cache import cache
+
+    return cache.get(_page_url_key(page_lookup, lang, site_id, extra_key), version=_get_cache_version())

cms/cache/permissions.py

@@ -0,0 +1,76 @@
+from cms.utils.conf import get_cms_setting
+
+PERMISSION_KEYS = [
+    'add_page', 'change_page', 'change_page_advanced_settings',
+    'change_page_permissions', 'delete_page', 'move_page',
+    'publish_page', 'view_page',
+]
+
+
+def get_cache_key(user, site, key):
+    """Cache key for ``user``'s page permissions of ``key`` on ``site``.
+
+    The page permissions behind a key are computed per site, so the site has to
+    be part of the key: without it the value warmed for one site is served for
+    every other one, granting access to sites the user has no rights on.
+    """
+    return "%s:permission:%d:%d:%s" % (
+        get_cms_setting('CACHE_PREFIX'), user.pk or 0, site.pk, key)
+
+
+def get_cache_permission_version_key():
+    return "{}:permission:version".format(get_cms_setting('CACHE_PREFIX'))
+
+
+def get_cache_permission_version():
+    from django.core.cache import cache
+    try:
+        version = int(cache.get(get_cache_permission_version_key()))
+    except Exception:
+        version = 1
+    return int(version)
+
+
+def get_permission_cache(user, site, key):
+    """
+    Helper for reading values from cache
+    """
+    from django.core.cache import cache
+    return cache.get(get_cache_key(user, site, key), version=get_cache_permission_version())
+
+
+def set_permission_cache(user, site, key, value):
+    """
+    Helper method for storing values in cache. Stores used keys so
+    all of them can be cleaned when clean_permission_cache gets called.
+    """
+    from django.core.cache import cache
+
+    # store this key, so we can clean it when required
+    cache_key = get_cache_key(user, site, key)
+    cache.set(cache_key, value,
+              get_cms_setting('CACHE_DURATIONS')['permissions'],
+              version=get_cache_permission_version())
+
+
+def clear_user_permission_cache(user):
+    """
+    Cleans permission cache for given user, on every site.
+    """
+    from django.contrib.sites.models import Site
+    from django.core.cache import cache
+
+    cache.delete_many(
+        [get_cache_key(user, site, key) for site in Site.objects.all() for key in PERMISSION_KEYS],
+        version=get_cache_permission_version(),
+    )
+
+
+def clear_permission_cache():
+    from django.core.cache import cache
+    version = get_cache_permission_version()
+    if version > 1:
+        cache.incr(get_cache_permission_version_key())
+    else:
+        cache.set(get_cache_permission_version_key(), 2,
+                  get_cms_setting('CACHE_DURATIONS')['permissions'])

cms/cache/placeholder.py

@@ -0,0 +1,171 @@
+"""
+This module manages placeholder caching. We use a cache-versioning strategy
+in which each (placeholder x lang x site_id) manages its own version. The
+actual cache includes additional keys appropriate for the placeholders
+get_vary_cache_on().
+
+Invalidation of a placeholder's cache simply increments the version number for
+the (placeholder x lang) pair, which renders any cache entries for that
+placeholder under that version inaccessible. Those cache entries will simply
+expire and will be purged according to the policy of the cache backend in-use.
+
+The cache entries themselves may include additional sub-keys, according to the
+list of VARY header-names as returned by placeholder.get_vary_cache_on() and
+the current HTTPRequest object.
+
+The vary-on header-names are also stored with the version. This enables us to
+check for cache hits without re-computing placeholder.get_vary_cache_on().
+"""
+import hashlib
+import time
+
+from django.utils.timezone import now
+
+from cms.utils.conf import get_cms_setting
+from cms.utils.helpers import get_header_name, get_timezone_name
+
+
+def _get_placeholder_cache_version_key(placeholder, lang, site_id):
+    """
+    Returns the version key for the given «placeholder», «lang» and «site_id».
+
+    Invalidating this (via clear_placeholder_cache by replacing the stored
+    value with a new value) will effectively make all "sub-caches" relating to
+    this (placeholder x lang) inaccessible. Sub-caches include caches per TZ
+    and per VARY header.
+    """
+    prefix = get_cms_setting("CACHE_PREFIX")
+    key = f"{prefix}|placeholder_cache_version|id:{placeholder.pk}|lang:{str(lang)}|site:{site_id}"
+    # django itself adds "version" add the end of cache-keys, e.g. "<key>:1".
+    # -> If `cache.set()` is for example called with `version=""`, it still adds
+    #    `:` at the end. So if we check the length for `> 250`, a length of 249
+    #    or even 250 ends up in an InvalidCacheKey-exception.
+    # In order to avoid these errors, we hash the keys at a lower length to also
+    # have a little buffer.
+    if len(key) > 200:
+        key = "{prefix}|{hash}".format(
+            prefix=prefix,
+            hash=hashlib.sha1(key.encode("utf-8")).hexdigest(),
+        )
+    return key
+
+
+def _get_placeholder_cache_version(placeholder, lang, site_id):
+    """
+    Gets the (placeholder x lang)'s current version and vary-on header-names
+    list, if present, otherwise resets to («timestamp», []).
+    """
+    from django.core.cache import cache
+
+    key = _get_placeholder_cache_version_key(placeholder, lang, site_id)
+    cached = cache.get(key)
+    if cached:
+        version, vary_on_list = cached
+    else:
+        version = int(time.time() * 1000000)
+        vary_on_list = []
+        _set_placeholder_cache_version(placeholder, lang, site_id, version, vary_on_list)
+    return version, vary_on_list
+
+
+def _set_placeholder_cache_version(placeholder, lang, site_id, version, vary_on_list=None, duration=None):
+    """
+    Sets the (placeholder x lang)'s version and vary-on header-names list.
+    """
+    from django.core.cache import cache
+
+    key = _get_placeholder_cache_version_key(placeholder, lang, site_id)
+
+    if not version or version < 1:
+        version = int(time.time() * 1000000)
+
+    if vary_on_list is None:
+        vary_on_list = []
+
+    cache.set(key, (version, vary_on_list), duration)
+
+
+def _get_placeholder_cache_key(placeholder, lang, site_id, request, soft=False):
+    """
+    Returns the fully-addressed cache key for the given placeholder and
+    the request.
+
+    The kwarg «soft» should be set to True if getting the cache key to then
+    read from the cache. If instead the key retrieval is to support a cache
+    write, let «soft» be False.
+    """
+    prefix = get_cms_setting("CACHE_PREFIX")
+    version, vary_on_list = _get_placeholder_cache_version(placeholder, lang, site_id)
+    tz = get_timezone_name()
+    main_key = f"{prefix}|render_placeholder|id:{placeholder.pk}|lang:{lang}|site:{site_id}|tz:{tz}|v:{version}"
+
+    if not soft:
+        # We are about to write to the cache, so we want to get the latest
+        # vary_cache_on headers and the correct cache expiration, ignoring any
+        # we already have. If the placeholder has already been rendered, this
+        # will be very efficient (zero-additional queries) due to the caching
+        # of all its plugins during the rendering process anyway.
+        vary_on_list = placeholder.get_vary_cache_on(request)
+        duration = placeholder.get_cache_expiration(request, now())
+        # Update the main placeholder cache version
+        _set_placeholder_cache_version(placeholder, lang, site_id, version, vary_on_list, duration)
+
+    sub_key_list = []
+    for key in vary_on_list:
+        value = request.META.get(get_header_name(key)) or "_"
+        sub_key_list.append(key + ":" + value)
+
+    cache_key = main_key
+    if sub_key_list:
+        cache_key += "|" + "|".join(sub_key_list)
+
+    # django itself adds "version" add the end of cache-keys, e.g. "<key>:1".
+    # -> If `cache.set()` is for example called with `version=""`, it still adds
+    #    `:` at the end. So if we check the length for `> 250`, a length of 249
+    #    or even 250 ends up in an InvalidCacheKey-exception.
+    # In order to avoid these errors, we hash the keys at a lower length to also
+    # have a little buffer.
+    if len(cache_key) > 200:
+        cache_key = "{prefix}|{hash}".format(
+            prefix=prefix,
+            hash=hashlib.sha1(cache_key.encode("utf-8")).hexdigest(),
+        )
+
+    return cache_key
+
+
+def set_placeholder_cache(placeholder, lang, site_id, content, request):
+    """
+    Sets the (correct) placeholder cache with the rendered placeholder.
+    """
+    from django.core.cache import cache
+
+    key = _get_placeholder_cache_key(placeholder, lang, site_id, request)
+
+    duration = min(get_cms_setting("CACHE_DURATIONS")["content"], placeholder.get_cache_expiration(request, now()))
+    cache.set(key, content, duration)
+    # "touch" the cache-version, so that it stays as fresh as this content.
+    version, vary_on_list = _get_placeholder_cache_version(placeholder, lang, site_id)
+    _set_placeholder_cache_version(placeholder, lang, site_id, version, vary_on_list, duration=duration)
+
+
+def get_placeholder_cache(placeholder, lang, site_id, request):
+    """
+    Returns the placeholder from cache respecting the placeholder's
+    VARY headers.
+    """
+    from django.core.cache import cache
+
+    key = _get_placeholder_cache_key(placeholder, lang, site_id, request, soft=True)
+    content = cache.get(key)
+    return content
+
+
+def clear_placeholder_cache(placeholder, lang, site_id):
+    """
+    Invalidates all existing cache entries for (placeholder x lang x site_id).
+    We don't need to re-store the vary_on_list, because the cache is now
+    effectively empty.
+    """
+    version = int(time.time() * 1000000)
+    _set_placeholder_cache_version(placeholder, lang, site_id, version, [])