# HG changeset patch # User Brian Neal # Date 1378949483 18000 # Node ID e888d627928ff81a96f08c46b6cb4bec6da044bb # Parent d33bedc3be74deff7689688c12ae9ea8eda46d0b Refactored the processing of image uploads. I suspect I will use this general algorithm in other places (like POTD), so I made it reusable. diff -r d33bedc3be74 -r e888d627928f core/functions.py --- a/core/functions.py Mon Sep 09 20:53:08 2013 -0500 +++ b/core/functions.py Wed Sep 11 20:31:23 2013 -0500 @@ -1,7 +1,9 @@ -"""This file houses various core utility functions for GPP""" +"""This file houses various core utility functions""" +from contextlib import contextmanager import datetime +import logging +import os import re -import logging from django.contrib.sites.models import Site from django.conf import settings @@ -10,6 +12,17 @@ import core.tasks +@contextmanager +def temp_open(path, mode): + """A context manager for closing and removing temporary files.""" + fp = open(path, mode) + try: + yield fp + finally: + fp.close() + os.remove(path) + + def send_mail(subject, message, from_email, recipient_list, defer=True, **kwargs): """ The main send email function. Use this function to send email from the @@ -52,7 +65,7 @@ site = Site.objects.get_current() subject = '[%s] %s' % (site.name, subject) send_mail(subject, - msg, + message, '%s@%s' % (settings.GPP_NO_REPLY_EMAIL, site.domain), [mail_tuple[1] for mail_tuple in settings.MANAGERS]) diff -r d33bedc3be74 -r e888d627928f core/image_uploader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/image_uploader.py Wed Sep 11 20:31:23 2013 -0500 @@ -0,0 +1,97 @@ +"""This module contains a function to upload an image file to a S3Bucket. + +The image can be resized and a thumbnail can be generated and uploaded as well. + +""" +import logging +from io import BytesIO +import os.path +import tempfile +import uuid + +from PIL import Image + +from core.functions import temp_open + + +logger = logging.getLogger(__name__) + + +def upload(fp, bucket, metadata=None, new_size=None, thumb_size=None): + """Upload an image file to a given S3Bucket. + + The image can optionally be resized and a thumbnail can be generated and + uploaded as well. + + Parameters: + fp - The image file to process. This is expected to be an instance of + Django's UploadedFile class. The file must have a name attribute and + we expect the name to have an extension. This is extension is + used for the uploaded image and thumbnail names. + bucket - A core.s3.S3Bucket instance to upload to. + metadata - If not None, must be a dictionary of metadata to apply to the + uploaded file and thumbnail. + new_size - If not None, the image will be resized to the dimensions + specified by new_size, which must be a (width, height) tuple. + thumb_size - If not None, a thumbnail image will be created with the + dimensions specified by thumb_size, which must be a (width, + height) tuple. The thumbnail will use the same metadata, if + present, as the image. The thumbnail filename will be the + same basename as the image with a 't' appended. The + extension will be the same as the original image. + + A tuple is returned: (image_url, thumb_url) where thumb_url will be None if + a thumbnail was not requested. + """ + + logger.info('Processing image file: {}'.format(fp.name)) + + # Trying to use PIL (or Pillow) on a Django UploadedFile is often + # problematic because the file is often an in-memory file if it is under + # a certain size. This complicates matters and many of the operations we try + # to perform on it fail if this is the case. To get around these issues, + # we make a copy of the file on the file system and operate on the copy. + # First generate a unique name and temporary file path. + unique_key = uuid.uuid4().hex + ext = os.path.splitext(fp.name)[1] + temp_name = os.path.join(tempfile.gettempdir(), unique_key + ext) + + # Write the UploadedFile to a temporary file on disk + with temp_open(temp_name, 'wb') as temp_file: + for chunk in fp.chunks(): + temp_file.write(chunk) + temp_file.close() + + # Resize image if necessary + if new_size: + image = Image.open(temp_name) + if image.size > new_size: + logger.debug('Resizing from {} to {}'.format(image.size, new_size)) + image.thumbnail(new_size, Image.ANTIALIAS) + image.save(temp_name) + + # Create thumbnail if necessary + thumb = None + if thumb_size: + logger.debug('Creating thumbnail {}'.format(thumb_size)) + image = Image.open(temp_name) + image.thumbnail(thumb_size, Image.ANTIALIAS) + thumb = BytesIO() + image.save(thumb, format=image.format) + + # Upload images to S3 + file_key = unique_key + ext + logging.debug('Uploading image') + image_url = bucket.upload_from_filename(file_key, temp_name, metadata) + + thumb_url = None + if thumb: + logging.debug('Uploading thumbnail') + thumb_key = '{}t{}'.format(unique_key, ext) + thumb_url = bucket.upload_from_string(thumb_key, + thumb.getvalue(), + metadata) + + logger.info('Completed processing image file: {}'.format(fp.name)) + + return (image_url, thumb_url) diff -r d33bedc3be74 -r e888d627928f core/s3.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/s3.py Wed Sep 11 20:31:23 2013 -0500 @@ -0,0 +1,84 @@ +"""s3.py + +This module provides Amazon S3 convenience wrappers. + +""" +from boto.s3.connection import S3Connection +from boto.s3.key import Key + + +class S3Bucket(object): + """This class abstracts an Amazon S3 bucket. + + We currently only support upload functionality. + + """ + def __init__(self, access_key, secret_key, base_url, bucket_name): + self.conn = S3Connection(access_key, secret_key) + self.bucket = self.conn.get_bucket(bucket_name, validate=False) + self.base_url = base_url + if not base_url.endswith('/'): + self.base_url += '/' + self.name = bucket_name + + def upload_from_file(self, key_name, fp, metadata=None, public=True): + """Uploads data from the file object fp to a new key named + key_name. metadata, if not None, must be a dict of metadata key / value + pairs which will be added to the key. + + If public is True, the key will be made public after the upload. + + Returns the URL to the uploaded file. + + """ + key = self._make_key(key_name, metadata) + key.set_contents_from_file(fp) + if public: + key.make_public() + return '{}{}/{}'.format(self.base_url, self.name, key_name) + + def upload_from_filename(self, key_name, filename, metadata=None, + public=True): + """Uploads data from the file named by filename to a new key named + key_name. metadata, if not None, must be a dict of metadata key / value + pairs which will be added to the key. + + If public is True, the key will be made public after the upload. + + Returns the URL to the uploaded file. + + """ + key = self._make_key(key_name, metadata) + key.set_contents_from_filename(filename) + if public: + key.make_public() + return '{}{}/{}'.format(self.base_url, self.name, key_name) + + def upload_from_string(self, key_name, content, metadata=None, + public=True): + """Creates a new key with the given key_name, and uploads the string + content to it. metadata, if not None, must be a dict of metadata key / + value pairs which will be added to the key. + + If public is True, the key will be made public after the upload. + + Returns the URL to the uploaded file. + + """ + key = self._make_key(key_name, metadata) + key.set_contents_from_string(content) + if public: + key.make_public() + return '{}{}/{}'.format(self.base_url, self.name, key_name) + + def _make_key(self, key_name, metadata): + """Private method to create a key and optionally apply metadata to + it. + + """ + key = Key(self.bucket) + key.key = key_name + if metadata: + for k, v in metadata.iteritems(): + key.set_metadata(k, v) + return key diff -r d33bedc3be74 -r e888d627928f user_photos/forms.py --- a/user_photos/forms.py Mon Sep 09 20:53:08 2013 -0500 +++ b/user_photos/forms.py Wed Sep 11 20:31:23 2013 -0500 @@ -1,10 +1,12 @@ """Forms for the user_photos application.""" +import datetime + from django import forms from django.conf import settings +from core.s3 import S3Bucket +from core.image_uploader import upload from user_photos.models import Photo -from user_photos.images import process_file -from user_photos.s3 import S3Bucket class UploadForm(forms.Form): @@ -21,12 +23,20 @@ This function should only be called if is_valid() returns True. """ - bucket = S3Bucket(settings.USER_PHOTOS_ACCESS_KEY, - settings.USER_PHOTOS_SECRET_KEY, - settings.USER_PHOTOS_BUCKET) - url, thumb_url = process_file(self.cleaned_data['image_file'], - self.user, - bucket) + bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY, + secret_key=settings.USER_PHOTOS_SECRET_KEY, + base_url=settings.USER_PHOTOS_BASE_URL, + bucket_name=settings.USER_PHOTOS_BUCKET) + + now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + metadata = {'user': self.user.username, 'date': now} + + url, thumb_url = upload(fp=self.cleaned_data['image_file'], + bucket=bucket, + metadata=metadata, + new_size=settings.USER_PHOTOS_MAX_SIZE, + thumb_size=settings.USER_PHOTOS_THUMB_SIZE) + photo = Photo(user=self.user, url=url, thumb_url=thumb_url) photo.save() return photo diff -r d33bedc3be74 -r e888d627928f user_photos/images.py --- a/user_photos/images.py Mon Sep 09 20:53:08 2013 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,69 +0,0 @@ -"""images.py - -This module contains image processing routines for the user_photos application. - -""" -import datetime -import logging -from io import BytesIO -import os.path -import uuid - -from django.conf import settings -from PIL import Image - - -logger = logging.getLogger(__name__) - - -def process_file(f, user, bucket): - """Perform processing on the given uploaded image file: - - * The image is resized if necessary - * A thumbnail version is created - * The image and thumbnail are uploaded to an S3 bucket - * The image and thumbnail URLs are returned as a tuple - - """ - logger.info('Processing image file for {}: {}'.format(user.username, f.name)) - - unique_key = uuid.uuid4().hex - ext = os.path.splitext(f.name)[1] - filename = '/tmp/' + unique_key + ext - with open(filename, 'wb') as fp: - for chunk in f.chunks(): - fp.write(chunk) - - # Resize image if necessary - image = Image.open(filename) - if image.size > settings.USER_PHOTOS_MAX_SIZE: - logger.debug('Resizing from {} to {}'.format(image.size, settings.USER_PHOTOS_MAX_SIZE)) - image.thumbnail(settings.USER_PHOTOS_MAX_SIZE, Image.ANTIALIAS) - image.save(filename) - - # Create thumbnail - logger.debug('Creating thumbnail') - image = Image.open(filename) - image.thumbnail(settings.USER_PHOTOS_THUMB_SIZE, Image.ANTIALIAS) - thumb = BytesIO() - image.save(thumb, format=image.format) - - # Upload both images to S3 - logging.debug('Uploading image') - now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - metadata = {'user': user.username, 'date': now} - file_key = unique_key + ext - bucket.upload_from_filename(file_key, filename, metadata) - - logging.debug('Uploading thumbnail') - thumb_key = '{}t{}'.format(unique_key, ext) - bucket.upload_from_string(thumb_key, thumb.getvalue()) - - os.remove(filename) - - logger.info('Completed processing image file for {}: {}'.format(user.username, f.name)) - - url_base = '{}/{}/'.format(settings.USER_PHOTOS_BASE_URL, - settings.USER_PHOTOS_BUCKET) - - return (url_base + file_key, url_base + thumb_key) diff -r d33bedc3be74 -r e888d627928f user_photos/s3.py --- a/user_photos/s3.py Mon Sep 09 20:53:08 2013 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,59 +0,0 @@ -"""s3.py - -This module provides the necessary S3 upload functionality for the user_photos -application. - -""" -from boto.s3.connection import S3Connection -from boto.s3.key import Key - - -class S3Bucket(object): - """This class abstracts an Amazon S3 bucket. - - We currently only need upload functionality. - - """ - def __init__(self, access_key, secret_key, bucket_name): - self.conn = S3Connection(access_key, secret_key) - self.bucket = self.conn.get_bucket(bucket_name, validate=False) - - def upload_from_filename(self, key_name, filename, metadata=None, - public=True): - """Uploads data from the file named by filename to a new key named - key_name. metadata, if not None, must be a dict of metadata key / value - pairs which will be added to the key. - - If public is True, the key will be made public after the upload. - - """ - key = self._make_key(key_name, metadata) - key.set_contents_from_filename(filename) - if public: - key.make_public() - - def upload_from_string(self, key_name, content, metadata=None, - public=True): - """Creates a new key with the given key_name, and uploads the string - content to it. metadata, if not None, must be a dict of metadata key / - value pairs which will be added to the key. - - If public is True, the key will be made public after the upload. - - """ - key = self._make_key(key_name, metadata) - key.set_contents_from_string(content) - if public: - key.make_public() - - def _make_key(self, key_name, metadata): - """Private method to create a key and optionally apply metadata to - it. - - """ - key = Key(self.bucket) - key.key = key_name - if metadata: - for k, v in metadata.iteritems(): - key.set_metadata(k, v) - return key