commit 5efd6261c8a5ac4c98a4f5d76477a7326e5138f2 Author: Erik Stein Date: Thu Mar 24 11:36:44 2016 +0100 Initial import. diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..4c7c564 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Erik Stein diff --git a/TODO b/TODO new file mode 100644 index 0000000..7b495bd --- /dev/null +++ b/TODO @@ -0,0 +1,31 @@ +NEXT + + +REVIEW + +[] python-iiif https://github.com/zimeon/iiif +[] https://github.com/fish2000/instakit +[] http://django-daguerre.readthedocs.org/en/latest/guides/areas.html +[] https://github.com/codingjoe/django-stdimage +[] https://github.com/brmc/django-media-helper#general-info + + +POSTPONED + +[] multipage-PDF + +[] snapshots + +[] manual upload + filesystem check schon, muss aber per sftp angelegt werden + + +MAYBE + +[] Integrate django-image-crop +[] https://github.com/francescortiz/image/tree/master/image + + +ISSUES + +[] on save look for manual overrides of variants and ask to delete them diff --git a/assetkit/README.rst b/assetkit/README.rst new file mode 100644 index 0000000..e69de29 diff --git a/assetkit/__init__.py b/assetkit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/assetkit/cachefiles/__init__.py b/assetkit/cachefiles/__init__.py new file mode 100644 index 0000000..91e2c00 --- /dev/null +++ b/assetkit/cachefiles/__init__.py @@ -0,0 +1,161 @@ +from copy import copy +from django.conf import settings +from django.core.files import File +from django.utils.functional import SimpleLazyObject +from imagekit.files import BaseIKFile +from imagekit.registry import generator_registry +from imagekit.signals import content_required, existence_required +from imagekit.utils import get_logger, get_singleton, generate, get_by_qname + + +class MediaAssetCacheFile(BaseIKFile, File): + """ + A file that represents the result of a generator. Creating an instance of + this class is not enough to trigger the generation of the file. In fact, + one of the main points of this class is to allow the creation of the file + to be deferred until the time that the cache file strategy requires it. + + """ + + def __init__(self, generator, name=None, storage=None, cachefile_backend=None, cachefile_strategy=None): + """ + :param generator: The object responsible for generating a new image. + :param name: The filename + :param storage: A Django storage object that will be used to save the + file. + :param cachefile_backend: The object responsible for managing the + state of the file. + :param cachefile_strategy: The object responsible for handling events + for this file. + + """ + self.generator = generator + + if not name: + try: + name = generator.cachefile_name + except AttributeError: + fn = get_by_qname(settings.IMAGEKIT_CACHEFILE_NAMER, 'namer') + name = fn(generator) + self.name = name + + storage = storage or getattr(generator, 'cachefile_storage', + None) or get_singleton(settings.IMAGEKIT_DEFAULT_FILE_STORAGE, + 'file storage backend') + self.cachefile_backend = ( + cachefile_backend + or getattr(generator, 'cachefile_backend', None) + or get_singleton(settings.IMAGEKIT_DEFAULT_CACHEFILE_BACKEND, + 'cache file backend')) + self.cachefile_strategy = ( + cachefile_strategy + or getattr(generator, 'cachefile_strategy', None) + or get_singleton(settings.IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY, + 'cache file strategy') + ) + + super(MediaAssetCacheFile, self).__init__(storage=storage) + + def _require_file(self): + if getattr(self, '_file', None) is None: + content_required.send(sender=self, file=self) + self._file = self.storage.open(self.name, 'rb') + + # The ``path`` and ``url`` properties are overridden so as to not call + # ``_require_file``, which is only meant to be called when the file object + # will be directly interacted with (e.g. when using ``read()``). These only + # require the file to exist; they do not need its contents to work. This + # distinction gives the user the flexibility to create a cache file + # strategy that assumes the existence of a file, but can still make the file + # available when its contents are required. + + def _storage_attr(self, attr): + if getattr(self, '_file', None) is None: + existence_required.send(sender=self, file=self) + fn = getattr(self.storage, attr) + return fn(self.name) + + @property + def path(self): + return self._storage_attr('path') + + @property + def url(self): + return self._storage_attr('url') + + def generate(self, force=False): + """ + Generate the file. If ``force`` is ``True``, the file will be generated + whether the file already exists or not. + + """ + if force or getattr(self, '_file', None) is None: + self.cachefile_backend.generate(self, force) + + def _generate(self): + # Generate the file + content = generate(self.generator) + + actual_name = self.storage.save(self.name, content) + + # We're going to reuse the generated file, so we need to reset the pointer. + content.seek(0) + + # Store the generated file. If we don't do this, the next time the + # "file" attribute is accessed, it will result in a call to the storage + # backend (in ``BaseIKFile._get_file``). Since we already have the + # contents of the file, what would the point of that be? + self.file = File(content) + + if actual_name != self.name: + get_logger().warning( + 'The storage backend %s did not save the file with the' + ' requested name ("%s") and instead used "%s". This may be' + ' because a file already existed with the requested name. If' + ' so, you may have meant to call generate() instead of' + ' generate(force=True), or there may be a race condition in the' + ' file backend %s. The saved file will not be used.' % ( + self.storage, + self.name, actual_name, + self.cachefile_backend + ) + ) + + def __bool__(self): + if not self.name: + return False + + # Dispatch the existence_required signal before checking to see if the + # file exists. This gives the strategy a chance to create the file. + existence_required.send(sender=self, file=self) + + try: + check = self.cachefile_strategy.should_verify_existence(self) + except AttributeError: + # All synchronous backends should have created the file as part of + # `existence_required` if they wanted to. + check = getattr(self.cachefile_backend, 'is_async', False) + return self.cachefile_backend.exists(self) if check else True + + def __getstate__(self): + state = copy(self.__dict__) + + # file is hidden link to "file" attribute + state.pop('_file', None) + + return state + + def __nonzero__(self): + # Python 2 compatibility + return self.__bool__() + + +class LazyMediaAssetCacheFile(SimpleLazyObject): + def __init__(self, generator_id, *args, **kwargs): + def setup(): + generator = generator_registry.get(generator_id, *args, **kwargs) + return MediaAssetCacheFile(generator) + super(LazyMediaAssetCacheFile, self).__init__(setup) + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, str(self) or 'None') diff --git a/assetkit/cachefiles/backends.py b/assetkit/cachefiles/backends.py new file mode 100644 index 0000000..89a9b58 --- /dev/null +++ b/assetkit/cachefiles/backends.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + +from ..utils import get_singleton, get_cache, sanitize_cache_key +import warnings +from copy import copy +from django.core.exceptions import ImproperlyConfigured + + +from imagekit.cachefiles.backends import CacheFileState, CachedFileBackend + + +class MediaAssetCacheFileState(CacheFileState): + """ + Manual means that the user has manually added + """ + + MANUAL = 'manual' + + +class MediaAssetCachedFileBackend(CachedFileBackend): + # As we will defer a lot to a celery queue, set the timeout higher + existence_check_timeout = 20 + """ + The number of seconds to wait before rechecking to see if the file exists. + If the image is found to exist, that information will be cached using the + timeout specified in your CACHES setting (which should be very high). + However, when the file does not exist, you probably want to check again + in a relatively short amount of time. This attribute allows you to do that. + + """ + + def get_state(self, file, check_if_unknown=True): + key = self.get_key(file) + state = self.cache.get(key) + if state is None and check_if_unknown: + exists = self._exists(file) + state = CacheFileState.EXISTS if exists else CacheFileState.DOES_NOT_EXIST + self.set_state(file, state) + return state + + def set_state(self, file, state): + key = self.get_key(file) + if state == CacheFileState.DOES_NOT_EXIST: + self.cache.set(key, state, self.existence_check_timeout) + else: + self.cache.set(key, state) + + def __getstate__(self): + state = copy(self.__dict__) + # Don't include the cache when pickling. It'll be reconstituted based + # on the settings. + state.pop('_cache', None) + return state + + def exists(self, file): + return self.get_state(file) == CacheFileState.EXISTS + + def generate(self, file, force=False): + raise NotImplementedError + + def generate_now(self, file, force=False): + if force or self.get_state(file) not in (CacheFileState.GENERATING, CacheFileState.EXISTS): + self.set_state(file, CacheFileState.GENERATING) + file._generate() + self.set_state(file, CacheFileState.EXISTS) + file.close() + + +class Simple(MediaAssetCachedFileBackend): + """ + The most basic file backend. The storage is consulted to see if the file + exists. Files are generated synchronously. + + """ + + def generate(self, file, force=False): + # FIXME Don't try this for large files like movies or recordings + self.generate_now(file, force=force) + + def _exists(self, file): + """ + The variant be either the automatically generated file or a + user uploaded file. Check both and set the state to OVERRIDEN + appropriately. + """ + return bool(getattr(file, '_file', None) + or file.storage.exists(file.name)) + + +def _generate_file(backend, file, force=False): + backend.generate_now(file, force=force) + + +class BaseAsync(Simple): + """ + Base class for cache file backends that generate files asynchronously. + """ + is_async = True + + def generate(self, file, force=False): + # Schedule the file for generation, unless we know for sure we don't + # need to. If an already-generated file sneaks through, that's okay; + # ``generate_now`` will catch it. We just want to make sure we don't + # schedule anything we know is unnecessary--but we also don't want to + # force a costly existence check. + state = self.get_state(file, check_if_unknown=False) + if state not in (CacheFileState.GENERATING, CacheFileState.EXISTS, CacheFileState.MANUAL): + self.schedule_generation(file, force=force) + + def schedule_generation(self, file, force=False): + # overwrite this to have the file generated in the background, + # e. g. in a worker queue. + raise NotImplementedError + + +try: + from celery import task +except ImportError: + pass +else: + _celery_task = task(ignore_result=True, serializer='pickle')(_generate_file) + + +class Celery(BaseAsync): + """ + A backend that uses Celery to generate the images. + """ + def __init__(self, *args, **kwargs): + try: + import celery # noqa + except ImportError: + raise ImproperlyConfigured('You must install celery to use' + ' imagekit.cachefiles.backends.Celery.') + super(Celery, self).__init__(*args, **kwargs) + + def schedule_generation(self, file, force=False): + _celery_task.delay(self, file, force=force) + + + +class Auto(BaseAsync): + """ + A backend that decides between immediately generating a variant + or queuing with celery, depending on media type, file size and + encoding. + """ \ No newline at end of file diff --git a/assetkit/cachefiles/namers.py b/assetkit/cachefiles/namers.py new file mode 100644 index 0000000..111265a --- /dev/null +++ b/assetkit/cachefiles/namers.py @@ -0,0 +1,46 @@ +""" +Functions responsible for returning filenames for the given image generator. +Users are free to define their own functions; these are just some some sensible +choices. + +""" + +from django.conf import settings +import os +from ..utils import suggest_extension + + +def spec_slug_as_path(generator): + """ + A namer that, given the following source file name:: + + /original/bulldog.jpg + + will generate a relative name like this:: + + /5ff3233527c5ac3e4b596343b440ff67.jpg + + which is meant to be added to both + + /cache/ + /manual/ + . + + """ + source_filename = getattr(generator.source, 'name', None) + spec_slug = getattr(generator, 'slug', None) + + ext = suggest_extension(source_filename or '', generator.format) + + + if source_filename is None or os.path.isabs(source_filename): + # Generally, we put the file right in the cache file directory. + dir = settings.IMAGEKIT_CACHEFILE_DIR + else: + # For source files with relative names (like Django media files), + # use the source's name to create the new filename. + dir = os.path.join(settings.IMAGEKIT_CACHEFILE_DIR, + os.path.splitext(source_filename)[0]) + + return os.path.normpath(os.path.join(dir, + '%s%s' % (generator.get_hash(), ext))) diff --git a/assetkit/cachefiles/strategies.py b/assetkit/cachefiles/strategies.py new file mode 100644 index 0000000..fca1ef1 --- /dev/null +++ b/assetkit/cachefiles/strategies.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + + +class Asynchronous(object): + """ + A strategy that queue's the generation right when the file is saved. + + """ + + def on_source_saved(self, file): + file.generate() + + def should_verify_existence(self, file): + return True + + # def on_existence_required(self, file): + # file.generate() + + # def on_content_required(self, file): + # file.generate() diff --git a/assetkit/exceptions.py b/assetkit/exceptions.py new file mode 100644 index 0000000..82ad085 --- /dev/null +++ b/assetkit/exceptions.py @@ -0,0 +1,4 @@ + + +class ConversionNotApplicable(Exception): + pass diff --git a/assetkit/files/__init__.py b/assetkit/files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/assetkit/files/storage.py b/assetkit/files/storage.py new file mode 100644 index 0000000..3c1090b --- /dev/null +++ b/assetkit/files/storage.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + +import os +import warnings + +from django.core.files.storage import FileSystemStorage +from django.conf import settings +from django.utils._os import safe_join +from django.utils.deconstruct import deconstructible +from django.utils.encoding import filepath_to_uri +from django.utils.six.moves.urllib.parse import urljoin + + +__all__ = ('MediaAssetStorage',) + + +ORIGINAL_FILE_PREFIX = 'original' +CACHED_VARIANT_PREFIX = 'cache' +MANUAL_VARIANT_PREFIX = 'manual' + + +""" +Storage setup: + + //original/filename.ext + /cache//. + /manual//. + +TODO Add snapshot (versioning) to path after asset_uuid. + +""" + + +class BaseVariantSubStorage(FileSystemStorage): + def delete(self, parent_storage, name): + """ + Deletes the whole storage path + """ + raise NotImplemented + + +@deconstructible +class MediaAssetStorage(FileSystemStorage): + """ + Standard media assets filesystem storage + """ + + def __init__(self, location=None, base_url=None, file_permissions_mode=None, + directory_permissions_mode=None): + if location is None: + location = getattr(settings, 'MEDIA_ASSETS_ROOT', settings.MEDIA_ROOT) + if base_url is None: + base_url = getattr(settings, 'MEDIA_ASSETS_URL', settings.MEDIA_URL) + if not base_url.endswith('/'): + base_url += '/' + super(MediaAssetStorage, self).__init__(self) + + def delete(self, name): + super(MediaAssetStorage, self).delete(name) + # FIXME Delete all cached files, too + warnings.warn("Cached files for asset \"%s\" are not deleted." % name) + + def path(self, name): + """ + `name` must already contain the whole asset filesystem path. + """ + return safe_join(self.location, ORIGINAL_FILE_PREFIX, name) + + def size(self, name): + return os.path.getsize(self.path(name)) + + def url(self, name): + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + return urljoin(self.base_url, ORIGINAL_FILE_PREFIX, filepath_to_uri(name)) diff --git a/assetkit/models/__init__.py b/assetkit/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/assetkit/models/assets.py b/assetkit/models/assets.py new file mode 100644 index 0000000..4c6560e --- /dev/null +++ b/assetkit/models/assets.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + +from django.db import models +from django.utils.translation import ugexttext_lazy as _ + +from imagekit.models import ImageSpecField +from imagekit.processors import Adjust, Thumbnail, ResizeToFit + +from ..exceptions import ConversionNotApplicable +from ..processors import Noop, Placeholder + +from .fields import MediaAssetFileField + + +MEDIA_TYPE_CHOICES = ( + (UNKNOWN, _("unknown")), + (IMAGE, _("image")), + (MOVIE, _("movie")), + (AUDIO, _("audio")), + (PDF, _("pdf")), + (SVG, _("svg")), + (VECTOR, _("eps")), + (DOCUMENT, _("document")), + ) + + +class ImageMediaHandler(object): + # thumbnail = ImageSpecField(ThumbnailSpec) + + thumbnail = ImageSpecField(source='file', + processors=[Adjust(contrast=1.2, sharpness=1.1), + Thumbnail(100, 50)], + format='JPEG', options={'quality': 90}) + + +# class PDFFile(object) +# def get_page(self, num): +# def get_page_count(self): + + +class PDFMediaHandler(object): + # thumbnail = ImageSpec + # small_downloadable = PDFSpecField + + def page_count(self): + # self.original_file = PDFFile() + return count(self.original_file.pages) + + +class MovieMediaHandler(object): + # thumbnail = ImageSpec + # desktop_webm = MovieSpecField + # desktop_mp4 = MovieSpecField + pass + + +class MediaAsset(models.Model): + """ + TODO Django should have a internal=True parameter on FileFields, preventing URL based access to the files + + Depending on the type the Asset might have different + """ + # TODO Added uuid field + original_file = MediaAssetFileField(_("original file"), metadata_field='metadata') + original_mimetype = models.CharField(_("mimetype"), blank=True, max_length=50) + metadata = models.JSONField() + media_type = models.ChoiceField(_("media type"), blank=True, choices=MEDIA_TYPE_CHOICES, default=MEDIA_TYPE_CHOICES[0][0]) + description = models.TextField(_("internal description"), null=True, blank=True) + slug = SlugField(help_text=_("should contain only the most specific part of the description, e.g.\"poster\"; the whole filename will be automatically build from the object the asset is attached and ")) + # copyright = models.TextField(_("copyright"), null=True, blank=True) + # author = models.TextField(_("author"), null=True, blank=True) + + def save(self, *args, **kwargs): + # TODO Fill media_type field + # TODO Fill mimetype field + super(MediaAsset, self).save(*args, **kwargs) + + @property + def media_handler(self): + """ + Returns a media_handler instance depending on the type + of the original file. + + >>> m = MediaAsset.objects.first() + >>> m.original_file + 'media/0/reference.png' + >>> m.media_handler.thumbnail + """ + diff --git a/assetkit/models/fields.py b/assetkit/models/fields.py new file mode 100644 index 0000000..cb3e942 --- /dev/null +++ b/assetkit/models/fields.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + + +from imagekit.models.fields.utils import ImageSpecFileDescriptor +from imagekit.models.fields import ImageSpecField +from ..files.storage import MediaAssetFileSystemStorage +from ...cachefiles import MediaAssetCacheFile + + +class MediaAssetCacheFileDescriptor(ImageSpecFileDescriptor): + """ + - Manages manually overriden cached files + - knows to handle ConversionNotApplicable exceptions and + provides svg-icons for this case + + """ + + def __init__(self, field, attname, source_field_name): + self.attname = attname + self.field = field + self.source_field_name = source_field_name + + def __get__(self, instance, owner): + if instance is None: + return self.field + else: + source = getattr(instance, self.source_field_name) + spec = self.field.get_spec(source=source) + file = MediaAssetCacheFile(spec) + instance.__dict__[self.attname] = file + return file + + def __set__(self, instance, value): + instance.__dict__[self.attname] = value + + +class MediaAssetFileField(ImageSpecField): + """ + - default storage + """ + + def __init__(self, storage=MediaAssetFileSystemStorage): + pass + + diff --git a/assetkit/models/media_types.py b/assetkit/models/media_types.py new file mode 100644 index 0000000..1695dc8 --- /dev/null +++ b/assetkit/models/media_types.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + + +UNKNOWN = 'unknown' +IMAGE = 'image' +MOVIE = 'movie' +AUDIO = 'audio' +PDF = 'pdf' +DOCUMENT = 'document' +SVG = 'svg' +VECTOR = 'vector' + +# TODO Implemen __all__ dynamically, made from MediaType classes carrying more information about the types \ No newline at end of file diff --git a/assetkit/pgkmeta.py b/assetkit/pgkmeta.py new file mode 100644 index 0000000..0549487 --- /dev/null +++ b/assetkit/pgkmeta.py @@ -0,0 +1,5 @@ +__title__ = 'django-assetkit' +__author__ = 'Erik Stein' +__version__ = '0.1' +__license__ = 'BSD' +__all__ = ['__title__', '__author__', '__version__', '__license__'] diff --git a/assetkit/processors/__init__.py b/assetkit/processors/__init__.py new file mode 100644 index 0000000..55a1051 --- /dev/null +++ b/assetkit/processors/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + + +class Noop(object): + def process(self, source): + result = source + return result + + +class Placeholder(object): + def __init__(self, label_text="placeholder"): + # The label text is intentionally not translated + self.label_text = label_text + + def process(self, source): + # Ignore source + raise NotImplemented + diff --git a/assetkit/specs/__init__.py b/assetkit/specs/__init__.py new file mode 100644 index 0000000..8d79f1e --- /dev/null +++ b/assetkit/specs/__init__.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2016 + + +class UniversalThumbnail(ImageSpec): + """ + Produces thumbnail images from any type of media, using + per-type initial conversion to an image. + """ + slug = 'thumbnail' + not_applicable_types = (UNKNOWN, AUDIO, DOCUMENT) + base_conversion_map = { + IMAGE: [], + MOVIE: [], + } + _processors = [ResizeToFill(100, 50)] + format = 'JPEG' + options = {'quality': 60} + + @property + def processors(self): + instance, field_name = get_field_info(self.source) + if instance.media_type in self.not_applicable_types: + raise ConversionNotApplicable + + if instance.media_type == + + processors = self.base_conversion_map[instance.media_type] + + + """ + Dynamic spec, depending on the source's media type + e.g. first convert a movie to an image, depending on a time code + for the movie poster + + see http://django-imagekit.readthedocs.org/en/develop/advanced_usage.html#specs-that-change + """ diff --git a/assetkit/tests.py b/assetkit/tests.py new file mode 100644 index 0000000..3863ec3 --- /dev/null +++ b/assetkit/tests.py @@ -0,0 +1,3 @@ + + +class \ No newline at end of file diff --git a/docs/strategy.rst b/docs/strategy.rst new file mode 100644 index 0000000..e74deea --- /dev/null +++ b/docs/strategy.rst @@ -0,0 +1,95 @@ + + +Hauptsächlich ist es ein mediakit; +d.h. imagekit erweitert auf alle arten von dateien (letztendlich inkl. document), +das eine grundsätzliche struktur bietet + +1. für automatische konvertierung in jeweils andere formate + denkbar eben auch bild -> pdf, film -> bild etc., pdf -> epub + conversion-quere mit celery + klarer state "noch nicht konvertiert" + +2. klare dateisystemverwaltung der originaldateien + automatische / manuelle konvertierungen + manuelle konvertierung heißt: die automatische konvertierung kann jederzeit überschrieben werden + +3. zugriffskontrolle: + grundsätzlich ist keine datei öffentlich zugänglich, + specs können mit zugriffskontrolle (anonym etc.) zugänglich gemacht werden, + dazu wird x-sendfile u.ä. verwendet + +4. benennung: + öffentlich zugänglich sollten alle dateien über generierte + namen sein, also slug + konvertierungstyp, so dass heruntergeladene dateien sinnvoll benannt sind + +5. umgang mit mehrseitigen dokumenten, also + mehrseitige PDF-dokument in bilder zu verwandeln + über spezial-descriptor, der page_count und page[index] zurückgeben kann? + dann auch jeweils alle seiten im cache ablegen? + +6. jeder asset bekommt eine eindeutige globale uuid, für interoperabilität mit z.b. iiif + +7. chunk upload + +8. batch upload + + +imagekit + processors + pilkit-processors + neue processors: + movie + pdf + assure conversion processors: + assure_image + schaut, wie die ursprungsdatei (z.b. film, pdf) in ein bild konvertiert werden muss + + +brauche async-processors + die also zwangsläufig die konvertierung an einen anderen prozess auslagern (ffmpeg), + und darüber aber bescheid wissen; + vermutlich ist es richtig, ein lock-file zu schreiben mit z.b. eine pid; oder evtl. einem celery-task-id + + +cachefiles + Cachefile ist gleich konvertierte Datei; + Cachefile sollte auch immer das manual-pedant verwalten/checken; + s.a. CacheFileState: + MANUAL_OVERRIDE + cachefile backend schreiben, dass unsere dateisystemlayout implementiert + + +zugriffskontrolle + möglichst über extra modul realisieren; + permissions als zusätzl. config, nicht integriert in specs + zugriffspfad über urls.py/views + d.h. mediakit selbst liefert alles direkt aus dem filesystem aus + außer: originaldateien; diese sollten immer abseits gespeichert werden, damit alles öffentlich mindestens durch specs kontrollierbar ist + + +brauche eigene ImageKitConf + +SourceGroup? + +MovieSpec + +PDFSpec + +TextDocumentSpec + better: MarkupSpec? + TextFormatSpec? + html + doc + docx + odt + epub + markdown + rst + + + +Admin/Upload per Rest-Interface, +aufwendigem Widget + d.h. in Django-Struktur ist nur das original_file-Feld + + + diff --git a/example_site/main/__init__.py b/example_site/main/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/example_site/main/medialibrary/__init__.py b/example_site/main/medialibrary/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/example_site/main/medialibrary/admin.py b/example_site/main/medialibrary/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/example_site/main/medialibrary/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/example_site/main/medialibrary/apps.py b/example_site/main/medialibrary/apps.py new file mode 100644 index 0000000..f79a626 --- /dev/null +++ b/example_site/main/medialibrary/apps.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + +from django.apps import AppConfig + + +class MedialibraryConfig(AppConfig): + name = 'medialibrary' diff --git a/example_site/main/medialibrary/migrations/__init__.py b/example_site/main/medialibrary/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/example_site/main/medialibrary/models.py b/example_site/main/medialibrary/models.py new file mode 100644 index 0000000..85b3ea8 --- /dev/null +++ b/example_site/main/medialibrary/models.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import uuid +from django.db import models +from django.utils.encoding import python_2_unicode_compatible +from django.utils.text import slugify +from django.utils.translation import ugettext_lazy as _ + +from ..storages import ProtectedMediaAssetStorage + +# TODO Define the central storage somewhere more central +originals_storage = ProtectedMediaAssetStorage() + + +# TODO Move this to special FileField class +def get_upload_path(instance, filename): + """ + Returns //original/ + where + - uuid is taken from instance, + - filename is slugified and shortened to a max length of 255 characters including the extension. + """ + MAX_LENGTH = 255 + name, ext = os.path.splitext(filename) + name = slugify(name) + name = name[:(MAX_LENGTH - len(ext))] + filename = "%s%s" % (name, ext) + return os.path.join(instance.get_uuid(), instance.storage.ORIGINAL_FILE_PREFIX, filename) + + +@python_2_unicode_compatible +class MediaAsset(models.Model): + uuid_hex = models.CharField(max_length=32, null=False, editable=False) + original_file = models.FileField(upload_to=get_upload_path, + storage=originals_storage + ) + name = models.CharField(_('name'), max_length=50) + + def __str__(self): + return self.name + + # TODO Make auto-initialized UUID-field + def get_uuid(self): + if not self.uuid_hex: + self.uuid_hex = uuid.uuid4().hex + return uuid.UUID(self.uuid_hex) + diff --git a/example_site/main/medialibrary/tests.py b/example_site/main/medialibrary/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/example_site/main/medialibrary/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/example_site/main/medialibrary/views.py b/example_site/main/medialibrary/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/example_site/main/medialibrary/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/example_site/main/settings.py b/example_site/main/settings.py new file mode 100644 index 0000000..5182a9f --- /dev/null +++ b/example_site/main/settings.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2015 + +import os + + +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) +BASE_DIR = os.path.dirname(PROJECT_DIR) +ENV_DIR = os.path.dirname(BASE_DIR) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'a+!qvbt4np4m$&@5s6=coc*yt4(mnrludzo(f=paaf&)-6zeuw' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + + 'imagekit', + 'main.medialibrary', + 'main', +] + +MIDDLEWARE_CLASSES = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'main.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + os.path.join(PROJECT_DIR, 'templates'), + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'main.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.9/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(ENV_DIR, 'var', 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.9/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.9/howto/static-files/ + +# MEDIA_ASSETS_URL = '/assets/' +# MEDIA_ASSETS_ROOT = os.path.join(BASE_DIR, 'media') + + +STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage' + +STATIC_URL = '/static/' +MEDIA_URL = '/media/' + +STATIC_ROOT = os.path.join(ENV_DIR, 'var', 'static') +# All media uploads should be protected: MEDIA_ROOT = os.path.join(ENV_DIR, 'var', 'media') +PROTECTED_MEDIA_ASSETS_ROOT = os.path.join(ENV_DIR, 'var', 'protected') + + +# IMAGEKIT_DEFAULT_CACHEFILE_BACKEND = 'assetkit.cachefiles.backend.MediaAssetCacheBackend' +# IMAGEKIT_DEFAULT_FILE_STORAGE = 'private_media.storages.PrivateMediaStorage' +# ORIGINALFILE_ROOT = os.path.join(ENV_DIR, 'var', 'protected') +# IMAGEKIT_CACHEFILE_DIR = '' # Directly in media root + +# PRIVATE_MEDIA_URL = '/media-archive/' +# PRIVATE_MEDIA_INTERNAL_URL = '/media-archive-x-accel-redirect/' +# PRIVATE_MEDIA_ROOT = os.path.join(ROOT_DIR, 'var', 'media-private') + +# PRIVATE_MEDIA_SERVER = 'private_media.servers.NginxXAccelRedirectServer' +# PRIVATE_MEDIA_PERMISSIONS = 'm1web.multimedia.permissions.AssetPermissions' + diff --git a/example_site/main/storages.py b/example_site/main/storages.py new file mode 100644 index 0000000..d6693a4 --- /dev/null +++ b/example_site/main/storages.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import warnings +from django.conf import settings +from django.core.files.storage import FileSystemStorage +from django.utils._os import safe_join +from django.utils.six.moves.urllib.parse import urljoin + + +ORIGINAL_FILE_PREFIX = 'original' +CACHED_VARIANT_PREFIX = 'cache' +MANUAL_VARIANT_PREFIX = 'manual' + + +# TODO ? @deconstructible +class ProtectedMediaAssetStorage(FileSystemStorage): + """ + Alllows uploads to /original/-subdirectories, but never provides URLs + to those files, instead raising an error. + """ + + ORIGINAL_FILE_PREFIX = ORIGINAL_FILE_PREFIX + + def __init__(self, location=None, file_permissions_mode=None, + directory_permissions_mode=None): + if location is None: + location = getattr(settings, 'PROTECTED_MEDIA_ASSETS_ROOT', settings.PROTECTED_MEDIA_ASSETS_ROOT) + super(ProtectedMediaAssetStorage, self).__init__(self, location=location, file_permissions_mode=None, + directory_permissions_mode=None) + + def delete(self, name): + super(ProtectedMediaAssetStorage, self).delete(name) + # FIXME Delete all cached files, too + warnings.warn("Cached files for asset \"%s\" are not deleted." % name) + + def path(self, name): + """ + `name` must already contain the whole asset filesystem path. + """ + return safe_join(self.location, ORIGINAL_FILE_PREFIX, name) + + def size(self, name): + return os.path.getsize(self.path(name)) + + def url(self, name): + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + + +# TODO ? @deconstructible +class + return urljoin(self.base_url, ORIGINAL_FILE_PREFIX, filepath_to_uri(name)) diff --git a/example_site/main/urls.py b/example_site/main/urls.py new file mode 100644 index 0000000..9c687fe --- /dev/null +++ b/example_site/main/urls.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +# Erik Stein , 2015 + +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin +from django.conf.urls.static import static +from django.utils.translation import ugettext_lazy as _ + + +urlpatterns = [ + url(r'^admin/', include(admin.site.urls)), + + # url(r'^', include('private_media.urls')), +] + +# Never used: urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + diff --git a/example_site/main/wsgi.py b/example_site/main/wsgi.py new file mode 100644 index 0000000..303baac --- /dev/null +++ b/example_site/main/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for example_site project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + +application = get_wsgi_application() diff --git a/example_site/manage.py b/example_site/manage.py new file mode 100755 index 0000000..a09fede --- /dev/null +++ b/example_site/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3d23f31 --- /dev/null +++ b/setup.py @@ -0,0 +1,71 @@ +#/usr/bin/env python +import codecs +import os +from setuptools import setup, find_packages +import sys + + +# Workaround for multiprocessing/nose issue. See http://bugs.python.org/msg170215 +try: + import multiprocessing +except ImportError: + pass + + +if 'publish' in sys.argv: + os.system('python setup.py sdist upload') + sys.exit() + + +read = lambda filepath: codecs.open(filepath, 'r', 'utf-8').read() + +def exec_file(filepath, globalz=None, localz=None): + exec(read(filepath), globalz, localz) + +# Load package meta from the pkgmeta module without loading assetkit. +pkgmeta = {} +exec_file(os.path.join(os.path.dirname(__file__), + 'assetkit', 'pkgmeta.py'), pkgmeta) + + +setup( + name='django-assetkit', + version=pkgmeta['__version__'], + description='Automated media asset processing for Django models.', + long_description=read(os.path.join(os.path.dirname(__file__), 'README.rst')), + author='Erik Stein', + author_email='erik@classlibrary.net', + license='BSD', + url='http://github.com/sha-red/django-assetkit/', + packages=find_packages(exclude=['*.tests', '*.tests.*', 'tests.*', 'tests']), + zip_safe=False, + include_package_data=True, + tests_require=[ + 'Pillow', + ], + test_suite='testrunner.run_tests', + install_requires=[ + 'django-appconf>=0.5', + 'django-imagekit>=3.0', + 'pilkit>=0.2.0', + 'six', + ], + extras_require={ + 'async': ['django-celery>=3.0'], + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Topic :: Utilities' + ], +) diff --git a/testrunner.py b/testrunner.py new file mode 100644 index 0000000..e4d27c7 --- /dev/null +++ b/testrunner.py @@ -0,0 +1,19 @@ +# A wrapper for Django's test runner. +# See http://ericholscher.com/blog/2009/jun/29/enable-setuppy-test-your-django-apps/ +# and http://gremu.net/blog/2010/enable-setuppy-test-your-django-apps/ +import os +import sys + +os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings' +test_dir = os.path.dirname(__file__) +sys.path.insert(0, test_dir) + +from django.test.utils import get_runner +from django.conf import settings + + +def run_tests(): + cls = get_runner(settings) + runner = cls() + failures = runner.run_tests(['tests']) + sys.exit(failures)