Added ManifestStaticFilesStorage to staticfiles contrib app.

import os

import posixpath

import re

import json

from django.conf import settings

from django.core.cache import (caches, InvalidCacheBackendError,

        return super(StaticFilesStorage, self).path(name)

class CachedFilesMixin(object):

class HashedFilesMixin(object):

    default_template = """url("%s")"""

    patterns = (

        ("*.css", (

    )

    def __init__(self, *args, **kwargs):

        super(CachedFilesMixin, self).__init__(*args, **kwargs)

        try:

            self.cache = caches['staticfiles']

        except InvalidCacheBackendError:

            # Use the default backend

            self.cache = default_cache

        super(HashedFilesMixin, self).__init__(*args, **kwargs)

        self._patterns = OrderedDict()

        self.hashed_files = {}

        for extension, patterns in self.patterns:

            for pattern in patterns:

                if isinstance(pattern, (tuple, list)):

            unparsed_name[2] += '?'

        return urlunsplit(unparsed_name)

    def cache_key(self, name):

        return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()

    def url(self, name, force=False):

        """

        Returns the real URL in DEBUG mode.

            if urlsplit(clean_name).path.endswith('/'):  # don't hash paths

                hashed_name = name

            else:

                cache_key = self.cache_key(name)

                hashed_name = self.cache.get(cache_key)

                if hashed_name is None:

                    hashed_name = self.hashed_name(clean_name).replace('\\', '/')

                    # set the cache if there was a miss

                    # (e.g. if cache server goes down)

                    self.cache.set(cache_key, hashed_name)

                hashed_name = self.stored_name(clean_name)

        final_url = super(CachedFilesMixin, self).url(hashed_name)

        final_url = super(HashedFilesMixin, self).url(hashed_name)

        # Special casing for a @font-face hack, like url(myfont.eot?#iefix")

        # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax

            return

        # where to store the new paths

        hashed_paths = {}

        hashed_files = OrderedDict()

        # build a list of adjustable files

        matches = lambda path: matches_patterns(path, self._patterns.keys())

                    # then save the processed result

                    content_file = ContentFile(force_bytes(content))

                    saved_name = self._save(hashed_name, content_file)

                    hashed_name = force_text(saved_name.replace('\\', '/'))

                    hashed_name = force_text(self.clean_name(saved_name))

                    processed = True

                else:

                    # or handle the case in which neither processing nor

                    if not hashed_file_exists:

                        processed = True

                        saved_name = self._save(hashed_name, original_file)

                        hashed_name = force_text(saved_name.replace('\\', '/'))

                        hashed_name = force_text(self.clean_name(saved_name))

                # and then set the cache accordingly

                hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name

                hashed_files[self.hash_key(name)] = hashed_name

                yield name, hashed_name, processed

        # Finally set the cache

        self.cache.set_many(hashed_paths)

        # Finally store the processed paths

        self.hashed_files.update(hashed_files)

    def clean_name(self, name):

        return name.replace('\\', '/')

    def hash_key(self, name):

        return name

    def stored_name(self, name):

        hash_key = self.hash_key(name)

        cache_name = self.hashed_files.get(hash_key)

        if cache_name is None:

            cache_name = self.clean_name(self.hashed_name(name))

            # store the hashed name if there was a miss, e.g.

            # when the files are still processed

            self.hashed_files[hash_key] = cache_name

        return cache_name

class ManifestFilesMixin(HashedFilesMixin):

    manifest_version = '1.0'  # the manifest format standard

    manifest_name = 'staticfiles.json'

    def __init__(self, *args, **kwargs):

        super(ManifestFilesMixin, self).__init__(*args, **kwargs)

        self.hashed_files = self.load_manifest()

    def read_manifest(self):

        try:

            with self.open(self.manifest_name) as manifest:

                return manifest.read()

        except IOError:

            return None

    def load_manifest(self):

        content = self.read_manifest()

        if content is None:

            return OrderedDict()

        try:

            stored = json.loads(content, object_pairs_hook=OrderedDict)

        except ValueError:

            pass

        else:

            version = stored.get('version', None)

            if version == '1.0':

                return stored.get('paths', OrderedDict())

        raise ValueError("Couldn't load manifest '%s' (version %s)" %

                         (self.manifest_name, self.manifest_version))

    def post_process(self, *args, **kwargs):

        all_post_processed = super(ManifestFilesMixin,

                                   self).post_process(*args, **kwargs)

        for post_processed in all_post_processed:

            yield post_processed

        payload = {'paths': self.hashed_files, 'version': self.manifest_version}

        if self.exists(self.manifest_name):

            self.delete(self.manifest_name)

        self._save(self.manifest_name, ContentFile(json.dumps(payload)))

class _MappingCache(object):

    """

    A small dict-like wrapper for a given cache backend instance.

    """

    def __init__(self, cache):

        self.cache = cache

    def __setitem__(self, key, value):

        self.cache.set(key, value)

    def __getitem__(self, key):

        value = self.cache.get(key, None)

        if value is None:

            raise KeyError("Couldn't find a file name '%s'" % key)

        return value

    def clear(self):

        self.cache.clear()

    def update(self, data):

        self.cache.set_many(data)

    def get(self, key, default=None):

        try:

            return self[key]

        except KeyError:

            return default

class CachedFilesMixin(HashedFilesMixin):

    def __init__(self, *args, **kwargs):

        super(CachedFilesMixin, self).__init__(*args, **kwargs)

        try:

            self.hashed_files = _MappingCache(caches['staticfiles'])

        except InvalidCacheBackendError:

            # Use the default backend

            self.hashed_files = _MappingCache(default_cache)

    def hash_key(self, name):

        key = hashlib.md5(force_bytes(self.clean_name(name))).hexdigest()

        return 'staticfiles:%s' % key

class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):

    pass

class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):

    """

    A static file system storage backend which also saves

    hashed copies of the files it saves.

    """

    pass

class AppStaticStorage(FileSystemStorage):

    """

    A file system storage backend that takes an app module and works

uses this behind the scenes to replace the paths with their hashed

counterparts and update the cache appropriately.

CachedStaticFilesStorage

------------------------

ManifestStaticFilesStorage

--------------------------

.. class:: storage.CachedStaticFilesStorage

.. versionadded:: 1.7

.. class:: storage.ManifestStaticFilesStorage

A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`

    storage backend which caches the files it saves by appending the MD5 hash

    of the file's content to the filename. For example, the file

storage backend which stores the file names it handles by appending the MD5

hash of the file's content to the filename. For example, the file

``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.

The purpose of this storage is to keep serving the old files in case some

files matching other saved files with the path of the cached copy (using

the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`

method). The regular expressions used to find those paths

    (``django.contrib.staticfiles.storage.CachedStaticFilesStorage.cached_patterns``)

    by default cover the `@import`_ rule and `url()`_ statement of `Cascading

(``django.contrib.staticfiles.storage.HashedFilesMixin.patterns``)

by default covers the `@import`_ rule and `url()`_ statement of `Cascading

Style Sheets`_. For example, the ``'css/styles.css'`` file with the

content

    @import url("../admin/css/base.css");

    would be replaced by calling the

    :meth:`~django.core.files.storage.Storage.url`

    method of the ``CachedStaticFilesStorage`` storage backend, ultimately

would be replaced by calling the :meth:`~django.core.files.storage.Storage.url`

method of the ``ManifestStaticFilesStorage`` storage backend, ultimately

saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following

content:

    @import url("../admin/css/base.27e20196a850.css");

    To enable the ``CachedStaticFilesStorage`` you have to make sure the

To enable the ``ManifestStaticFilesStorage`` you have to make sure the

following requirements are met:

* the :setting:`STATICFILES_STORAGE` setting is set to

      ``'django.contrib.staticfiles.storage.CachedStaticFilesStorage'``

  ``'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'``

* the :setting:`DEBUG` setting is set to ``False``

* you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template

  tag to refer to your static files in your templates

  :djadmin:`collectstatic` management command

Since creating the MD5 hash can be a performance burden to your website

    during runtime, ``staticfiles`` will automatically try to cache the

    hashed name for each file path using Django's :doc:`caching

    framework</topics/cache>`. If you want to override certain options of the

    cache backend the storage uses, simply specify a custom entry in the

    :setting:`CACHES` setting named ``'staticfiles'``. It falls back to using

    the ``'default'`` cache backend.

during runtime, ``staticfiles`` will automatically store the mapping with

hashed names for all processed files in a file called ``staticfiles.json``.

This happens once when you run the :djadmin:`collectstatic` management

command.

.. method:: file_hash(name, content=None)

The method that is used when creating the hashed name of a file.

Needs to return a hash for the given file name and content.

By default it calculates a MD5 hash from the content's chunks as

    mentioned above.

mentioned above. Feel free to override this method to use your own

hashing algorithm.

.. _`far future Expires headers`: http://developer.yahoo.com/performance/rules.html#expires

.. _`@import`: http://www.w3.org/TR/CSS2/cascade.html#at-import

.. _`url()`: http://www.w3.org/TR/CSS2/syndata.html#uri

.. _`Cascading Style Sheets`: http://www.w3.org/Style/CSS/

CachedStaticFilesStorage

------------------------

.. class:: storage.CachedStaticFilesStorage

``CachedStaticFilesStorage`` is a similar class like the

:class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage` class

but uses Django's :doc:`caching framework</topics/cache>` for storing the

hashed names of processed files instead of a static manifest file called

``staticfiles.json``. This is mostly useful for situations in which you don't

have accesss to the file system.

If you want to override certain options of the cache backend the storage uses,

simply specify a custom entry in the :setting:`CACHES` setting named

``'staticfiles'``. It falls back to using the ``'default'`` cache backend.

.. currentmodule:: django.contrib.staticfiles.templatetags.staticfiles

Template tags

  and :attr:`~django.core.files.storage.FileSystemStorage.directory_permissions_mode`

  parameters. See :djadmin:`collectstatic` for example usage.

* The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`

  backend gets a sibling class called

  :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`

  that doesn't use the cache system at all but instead a JSON file called

  ``staticfiles.json`` for storing the mapping between the original file name

  (e.g. ``css/styles.css``) and the hashed file name (e.g.

  ``css/styles.55e7cbb9ba48.css``. The ``staticfiles.json`` file is created

  when running the :djadmin:`collectstatic` management command and should

  be a less expensive alternative for remote storages such as Amazon S3.

  See the :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`

  docs for more information.

:mod:`django.contrib.syndication`

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

    pass

# we set DEBUG to False here since the template tag wouldn't work otherwise

@override_settings(**dict(

    TEST_SETTINGS,

    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage',

    DEBUG=False,

))

class TestCollectionCachedStorage(BaseCollectionTestCase,

        BaseStaticFilesTestCase, TestCase):

    """

    Tests for the Cache busting storage

    """

    def cached_file_path(self, path):

        fullpath = self.render_template(self.static_template_snippet(path))

def hashed_file_path(test, path):

    fullpath = test.render_template(test.static_template_snippet(path))

    return fullpath.replace(settings.STATIC_URL, '')

class TestHashedFiles(object):

    hashed_file_path = hashed_file_path

    def test_template_tag_return(self):

        """

        Test the CachedStaticFilesStorage backend.

                                 "/static/path/?query")

    def test_template_tag_simple_content(self):

        relpath = self.cached_file_path("cached/styles.css")

        relpath = self.hashed_file_path("cached/styles.css")

        self.assertEqual(relpath, "cached/styles.93b1147e8552.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_ignored_completely(self):

        relpath = self.cached_file_path("cached/css/ignored.css")

        relpath = self.hashed_file_path("cached/css/ignored.css")

        self.assertEqual(relpath, "cached/css/ignored.6c77f2643390.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b'//foobar', content)

    def test_path_with_querystring(self):

        relpath = self.cached_file_path("cached/styles.css?spam=eggs")

        relpath = self.hashed_file_path("cached/styles.css?spam=eggs")

        self.assertEqual(relpath,

                         "cached/styles.93b1147e8552.css?spam=eggs")

        with storage.staticfiles_storage.open(

            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_with_fragment(self):

        relpath = self.cached_file_path("cached/styles.css#eggs")

        relpath = self.hashed_file_path("cached/styles.css#eggs")

        self.assertEqual(relpath, "cached/styles.93b1147e8552.css#eggs")

        with storage.staticfiles_storage.open(

                "cached/styles.93b1147e8552.css") as relfile:

            self.assertIn(b"other.d41d8cd98f00.css", content)

    def test_path_with_querystring_and_fragment(self):

        relpath = self.cached_file_path("cached/css/fragments.css")

        relpath = self.hashed_file_path("cached/css/fragments.css")

        self.assertEqual(relpath, "cached/css/fragments.75433540b096.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b'#default#VML', content)

    def test_template_tag_absolute(self):

        relpath = self.cached_file_path("cached/absolute.css")

        relpath = self.hashed_file_path("cached/absolute.css")

        self.assertEqual(relpath, "cached/absolute.23f087ad823a.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b'/static/cached/img/relative.acae32e4532b.png', content)

    def test_template_tag_denorm(self):

        relpath = self.cached_file_path("cached/denorm.css")

        relpath = self.hashed_file_path("cached/denorm.css")

        self.assertEqual(relpath, "cached/denorm.c5bd139ad821.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b'url("img/relative.acae32e4532b.png', content)

    def test_template_tag_relative(self):

        relpath = self.cached_file_path("cached/relative.css")

        relpath = self.hashed_file_path("cached/relative.css")

        self.assertEqual(relpath, "cached/relative.2217ea7273c2.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

    def test_import_replacement(self):

        "See #18050"

        relpath = self.cached_file_path("cached/import.css")

        relpath = self.hashed_file_path("cached/import.css")

        self.assertEqual(relpath, "cached/import.2b1d40b0bbd4.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            self.assertIn(b"""import url("styles.93b1147e8552.css")""", relfile.read())

    def test_template_tag_deep_relative(self):

        relpath = self.cached_file_path("cached/css/window.css")

        relpath = self.hashed_file_path("cached/css/window.css")

        self.assertEqual(relpath, "cached/css/window.9db38d5169f3.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

            self.assertIn(b'url("img/window.acae32e4532b.png")', content)

    def test_template_tag_url(self):

        relpath = self.cached_file_path("cached/url.css")

        relpath = self.hashed_file_path("cached/url.css")

        self.assertEqual(relpath, "cached/url.615e21601e4b.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            self.assertIn(b"https://", relfile.read())

    def test_cache_invalidation(self):

        name = "cached/styles.css"

        hashed_name = "cached/styles.93b1147e8552.css"

        # check if the cache is filled correctly as expected

        cache_key = storage.staticfiles_storage.cache_key(name)

        cached_name = storage.staticfiles_storage.cache.get(cache_key)

        self.assertEqual(self.cached_file_path(name), cached_name)

        # clearing the cache to make sure we re-set it correctly in the url method

        storage.staticfiles_storage.cache.clear()

        cached_name = storage.staticfiles_storage.cache.get(cache_key)

        self.assertEqual(cached_name, None)

        self.assertEqual(self.cached_file_path(name), hashed_name)

        cached_name = storage.staticfiles_storage.cache.get(cache_key)

        self.assertEqual(cached_name, hashed_name)

    def test_post_processing(self):

        """Test that post_processing behaves correctly.

        self.assertIn(os.path.join('cached', 'css', 'img', 'window.png'), stats['unmodified'])

        self.assertIn(os.path.join('test', 'nonascii.css'), stats['post_processed'])

    def test_cache_key_memcache_validation(self):

        """

        Handle cache key creation correctly, see #17861.

        """

        name = "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + "\x16" + "\xb4"

        cache_key = storage.staticfiles_storage.cache_key(name)

        cache_validator = BaseCache({})

        cache_validator.validate_key(cache_key)

        self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7')

    def test_css_import_case_insensitive(self):

        relpath = self.cached_file_path("cached/styles_insensitive.css")

        relpath = self.hashed_file_path("cached/styles_insensitive.css")

        self.assertEqual(relpath, "cached/styles_insensitive.2f0151cca872.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()

        self.assertEqual("Post-processing 'faulty.css' failed!\n\n", err.getvalue())

# we set DEBUG to False here since the template tag wouldn't work otherwise

@override_settings(**dict(

    TEST_SETTINGS,

    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage',

    DEBUG=False,

))

class TestCollectionCachedStorage(TestHashedFiles, BaseCollectionTestCase,

        BaseStaticFilesTestCase, TestCase):

    """

    Tests for the Cache busting storage

    """

    def test_cache_invalidation(self):

        name = "cached/styles.css"

        hashed_name = "cached/styles.93b1147e8552.css"

        # check if the cache is filled correctly as expected

        cache_key = storage.staticfiles_storage.hash_key(name)

        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)

        self.assertEqual(self.hashed_file_path(name), cached_name)

        # clearing the cache to make sure we re-set it correctly in the url method

        storage.staticfiles_storage.hashed_files.clear()

        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)

        self.assertEqual(cached_name, None)

        self.assertEqual(self.hashed_file_path(name), hashed_name)

        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)

        self.assertEqual(cached_name, hashed_name)

    def test_cache_key_memcache_validation(self):

        """

        Handle cache key creation correctly, see #17861.

        """

        name = "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + "\x16" + "\xb4"

        cache_key = storage.staticfiles_storage.hash_key(name)

        cache_validator = BaseCache({})

        cache_validator.validate_key(cache_key)

        self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7')

# we set DEBUG to False here since the template tag wouldn't work otherwise

@override_settings(**dict(

    TEST_SETTINGS,

    STATICFILES_STORAGE='django.contrib.staticfiles.storage.ManifestStaticFilesStorage',

    DEBUG=False,

))

class TestCollectionManifestStorage(TestHashedFiles, BaseCollectionTestCase,

        BaseStaticFilesTestCase, TestCase):

    """

    Tests for the Cache busting storage

    """

    def test_manifest_exists(self):

        filename = storage.staticfiles_storage.manifest_name

        path = storage.staticfiles_storage.path(filename)

        self.assertTrue(os.path.exists(path))

    def test_loaded_cache(self):

        self.assertNotEqual(storage.staticfiles_storage.hashed_files, {})

        manifest_content = storage.staticfiles_storage.read_manifest()

        self.assertIn('"version": "%s"' %

                      storage.staticfiles_storage.manifest_version,

                      force_text(manifest_content))

# we set DEBUG to False here since the template tag wouldn't work otherwise

@override_settings(**dict(

    TEST_SETTINGS,

    """

    Tests for the Cache busting storage

    """

    def cached_file_path(self, path):

        fullpath = self.render_template(self.static_template_snippet(path))

        return fullpath.replace(settings.STATIC_URL, '')

    hashed_file_path = hashed_file_path

    def test_template_tag_return(self):

        """

                                 "/static/path/?query")

    def test_template_tag_simple_content(self):

        relpath = self.cached_file_path("cached/styles.css")

        relpath = self.hashed_file_path("cached/styles.css")

        self.assertEqual(relpath, "cached/styles.deploy12345.css")

        with storage.staticfiles_storage.open(relpath) as relfile:

            content = relfile.read()