Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 881:13f2d4393ec4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
More work on ssl_images command. Uploading now works.
author Brian Neal <bgneal@gmail.com>
date Thu, 15 Jan 2015 21:05:05 -0600
parents 6900040df0f8
children 9a3019f2c7dc
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@859 9 import logging
bgneal@859 10 from optparse import make_option
bgneal@859 11 import os.path
bgneal@863 12 import re
bgneal@863 13 import signal
bgneal@881 14 import socket
bgneal@881 15 import urllib
bgneal@868 16 import urlparse
bgneal@881 17 import uuid
bgneal@859 18
bgneal@859 19 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 20 from django.conf import settings
bgneal@863 21 import markdown.inlinepatterns
bgneal@881 22 from PIL import Image
bgneal@859 23
bgneal@860 24 from comments.models import Comment
bgneal@860 25 from forums.models import Post
bgneal@881 26 from core.s3 import S3Bucket
bgneal@860 27
bgneal@860 28
bgneal@859 29 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 30 logger = logging.getLogger(__name__)
bgneal@859 31
bgneal@871 32 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 33 re.DOTALL | re.UNICODE)
bgneal@871 34 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 35 re.DOTALL | re.UNICODE)
bgneal@863 36
bgneal@868 37 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@866 38 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 39
bgneal@881 40 PHOTO_MAX_SIZE = (660, 720)
bgneal@881 41 PHOTO_BASE_URL = 'https://s3.amazonaws.com/'
bgneal@881 42 PHOTO_BUCKET_NAME = 'sg101.forum.photos'
bgneal@881 43
bgneal@863 44 quit_flag = False
bgneal@881 45 opener = None
bgneal@881 46 bucket = None
bgneal@881 47 url_cache = {}
bgneal@863 48
bgneal@863 49
bgneal@863 50 def signal_handler(signum, frame):
bgneal@863 51 """SIGINT signal handler"""
bgneal@863 52 global quit_flag
bgneal@863 53 quit_flag = True
bgneal@863 54
bgneal@859 55
bgneal@859 56 def _setup_logging():
bgneal@859 57 logger.setLevel(logging.DEBUG)
bgneal@859 58 logger.propagate = False
bgneal@859 59 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 60 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 61 handler.setFormatter(formatter)
bgneal@859 62 logger.addHandler(handler)
bgneal@859 63
bgneal@859 64
bgneal@881 65 class ImageURLopener(urllib.FancyURLopener):
bgneal@881 66 """Our URL opener. Handles redirects as per FancyURLopener. But all other
bgneal@881 67 errors and authentication requests will raise an IOError.
bgneal@881 68 """
bgneal@881 69 HANDLED_ERRORS = set([302, 301, 303, 307])
bgneal@881 70
bgneal@881 71 def http_error_default(self, url, fp, errcode, errmsg, headers):
bgneal@881 72 return urllib.URLopener.http_error_default(self, url, fp, errcode,
bgneal@881 73 errmsg, headers)
bgneal@881 74
bgneal@881 75 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
bgneal@881 76 """Handle http errors.
bgneal@881 77 We let FancyURLopener handle the redirects, but any other error we want
bgneal@881 78 to let fail.
bgneal@881 79 """
bgneal@881 80 if errcode in self.HANDLED_ERRORS:
bgneal@881 81 name = 'http_error_%d' % errcode
bgneal@881 82 method = getattr(self, name)
bgneal@881 83 if data is None:
bgneal@881 84 result = method(url, fp, errcode, errmsg, headers)
bgneal@881 85 else:
bgneal@881 86 result = method(url, fp, errcode, errmsg, headers, data)
bgneal@881 87 if result:
bgneal@881 88 return result
bgneal@881 89 return self.http_error_default(url, fp, errcode, errmsg, headers)
bgneal@881 90
bgneal@881 91
bgneal@881 92 def download_image(src):
bgneal@881 93 """Downloads the image file from the given source URL.
bgneal@881 94
bgneal@881 95 If successful returns the path to the downloaded file. Otherwise None is
bgneal@881 96 returned.
bgneal@881 97 """
bgneal@881 98 logger.info("Retrieving %s", src)
bgneal@881 99 try:
bgneal@881 100 fn, hdrs = opener.retrieve(src)
bgneal@881 101 except IOError as ex:
bgneal@881 102 args = ex.args
bgneal@881 103 if len(args) == 4 and args[0] == 'http error':
bgneal@881 104 logger.error("http error: %d - %s", args[1], args[2])
bgneal@881 105 else:
bgneal@881 106 logger.error("%s", ex)
bgneal@881 107 return None
bgneal@881 108
bgneal@881 109 # If there is an error or timeout, sometimes there is no content-length
bgneal@881 110 # header.
bgneal@881 111 content_length = hdrs.get('content-length')
bgneal@881 112 if not content_length:
bgneal@881 113 logger.error("Bad content-length: %s", content_length)
bgneal@881 114 return None
bgneal@881 115
bgneal@881 116 # Does it look like an image?
bgneal@881 117 content_type = hdrs.get('content-type')
bgneal@881 118 if not content_type:
bgneal@881 119 logger.error("No content-type header found")
bgneal@881 120 return None
bgneal@881 121
bgneal@881 122 logger.info("Retrieved: %s bytes; content-type: %s", content_length,
bgneal@881 123 content_type)
bgneal@881 124
bgneal@881 125 parts = content_type.split('/')
bgneal@881 126 if len(parts) < 2 or parts[0] != 'image':
bgneal@881 127 logger.error("Unknown content-type: %s", content_type)
bgneal@881 128 return None
bgneal@881 129
bgneal@881 130 return fn
bgneal@881 131
bgneal@881 132
bgneal@881 133 def resize_image(img_path):
bgneal@881 134 """Resizes the image found at img_path if necessary."""
bgneal@881 135 image = Image.open(img_path)
bgneal@881 136 if image.size > PHOTO_MAX_SIZE:
bgneal@881 137 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 138 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 139 image.save(img_path)
bgneal@881 140
bgneal@881 141
bgneal@881 142 def upload_image(img_path):
bgneal@881 143 """Upload image file located at img_path to our S3 bucket.
bgneal@881 144
bgneal@881 145 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 146 """
bgneal@881 147 logger.info("upload_image starting")
bgneal@881 148 # Make a unique name for the image in the bucket
bgneal@881 149 unique_key = uuid.uuid4().hex
bgneal@881 150 ext = os.path.splitext(img_path)[1]
bgneal@881 151 file_key = unique_key + ext
bgneal@881 152 try:
bgneal@881 153 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 154 except IOError as ex:
bgneal@881 155 logger.error("Error uploading file: %s", ex)
bgneal@881 156 return None
bgneal@881 157
bgneal@881 158
bgneal@868 159 def save_image_to_cloud(src):
bgneal@881 160 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 161
bgneal@881 162 Returns the new URL or None if unsuccessful.
bgneal@881 163 """
bgneal@881 164 # Check the cache first
bgneal@881 165 new_url = url_cache.get(src)
bgneal@881 166 if new_url:
bgneal@881 167 return new_url
bgneal@881 168
bgneal@881 169 fn = download_image(src)
bgneal@881 170 if fn:
bgneal@881 171 resize_image(fn)
bgneal@881 172 new_url = upload_image(fn)
bgneal@881 173 if new_url:
bgneal@881 174 url_cache[src] = new_url
bgneal@881 175 return new_url
bgneal@881 176 return None
bgneal@868 177
bgneal@868 178
bgneal@866 179 def replace_image_markup(match):
bgneal@870 180 src_parts = match.group(8).split()
bgneal@868 181 if src_parts:
bgneal@868 182 src = src_parts[0]
bgneal@868 183 if src[0] == "<" and src[-1] == ">":
bgneal@868 184 src = src[1:-1]
bgneal@868 185 else:
bgneal@868 186 src = ''
bgneal@868 187
bgneal@868 188 title = ''
bgneal@868 189 if len(src_parts) > 1:
bgneal@868 190 title = " ".join(src_parts[1:])
bgneal@870 191 alt = match.group(1)
bgneal@868 192
bgneal@871 193 new_src = None
bgneal@868 194 if src:
bgneal@868 195 r = urlparse.urlparse(src)
bgneal@871 196 if r.hostname in SG101_HOSTS:
bgneal@871 197 new_src = r.path # convert to relative path
bgneal@871 198 elif r.scheme == 'http':
bgneal@871 199 new_src = save_image_to_cloud(src)
bgneal@868 200 elif r.scheme == 'https':
bgneal@868 201 new_src = src # already https, accept it as-is
bgneal@868 202
bgneal@868 203 if new_src:
bgneal@868 204 if title:
bgneal@871 205 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 206 else:
bgneal@868 207 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 208 else:
bgneal@868 209 # something's messed up, convert to a link using original src
bgneal@868 210 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 211
bgneal@868 212 return s
bgneal@860 213
bgneal@860 214
bgneal@866 215 def process_post(text):
bgneal@863 216 """Process the post object:
bgneal@863 217
bgneal@863 218 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 219 links, getting rid of plain old http sources; either converting to https
bgneal@863 220 or relative style links (if the link is to SG101).
bgneal@863 221
bgneal@863 222 We also do a search for Markdown image reference markup. We aren't expecting
bgneal@863 223 these, but we will log something if we see any.
bgneal@863 224
bgneal@863 225 """
bgneal@866 226 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 227
bgneal@863 228
bgneal@859 229 class Command(NoArgsCommand):
bgneal@859 230 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 231 option_list = NoArgsCommand.option_list + (
bgneal@866 232 make_option('-m', '--model',
bgneal@866 233 choices=MODEL_CHOICES,
bgneal@866 234 help="which model to update; must be one of {{{}}}".format(
bgneal@866 235 ', '.join(MODEL_CHOICES))),
bgneal@860 236 make_option('-i', '--i',
bgneal@859 237 type='int',
bgneal@863 238 help="optional first slice index; the i in [i:j]"),
bgneal@860 239 make_option('-j', '--j',
bgneal@859 240 type='int',
bgneal@863 241 help="optional second slice index; the j in [i:j]"),
bgneal@859 242 )
bgneal@859 243
bgneal@859 244 def handle_noargs(self, **options):
bgneal@859 245 _setup_logging()
bgneal@860 246 logger.info("Starting; arguments received: %s", options)
bgneal@859 247
bgneal@866 248 if options['model'] not in MODEL_CHOICES:
bgneal@866 249 raise CommandError('Please choose a --model option')
bgneal@859 250
bgneal@866 251 if options['model'] == 'comments':
bgneal@860 252 qs = Comment.objects.all()
bgneal@866 253 text_attr = 'comment'
bgneal@881 254 model_name = 'Comment'
bgneal@860 255 else:
bgneal@860 256 qs = Post.objects.all()
bgneal@866 257 text_attr = 'body'
bgneal@881 258 model_name = 'Post'
bgneal@860 259
bgneal@860 260 i, j = options['i'], options['j']
bgneal@860 261
bgneal@860 262 if i is not None and i < 0:
bgneal@860 263 raise CommandError("-i must be >= 0")
bgneal@860 264 if j is not None and j < 0:
bgneal@860 265 raise CommandError("-j must be >= 0")
bgneal@860 266 if j is not None and i is not None and j <= i:
bgneal@860 267 raise CommandError("-j must be > -i")
bgneal@860 268
bgneal@860 269 if i is not None and j is not None:
bgneal@860 270 qs = qs[i:j]
bgneal@860 271 elif i is not None and j is None:
bgneal@860 272 qs = qs[i:]
bgneal@860 273 elif i is None and j is not None:
bgneal@860 274 qs = qs[:j]
bgneal@860 275
bgneal@881 276 # Set global socket timeout
bgneal@881 277 socket.setdefaulttimeout(30)
bgneal@881 278
bgneal@863 279 # Install signal handler for ctrl-c
bgneal@863 280 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 281
bgneal@881 282 # Create URL opener to download photos
bgneal@881 283 global opener
bgneal@881 284 opener = ImageURLopener()
bgneal@881 285
bgneal@881 286 # Create bucket to upload photos
bgneal@881 287 global bucket
bgneal@881 288 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 289 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 290 base_url=PHOTO_BASE_URL,
bgneal@881 291 bucket_name=PHOTO_BUCKET_NAME)
bgneal@860 292 s = []
bgneal@881 293 for n, model in enumerate(qs.iterator()):
bgneal@863 294 if quit_flag:
bgneal@863 295 logger.warning("SIGINT received, exiting")
bgneal@881 296 break
bgneal@881 297 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 298 txt = getattr(model, text_attr)
bgneal@866 299 new_txt = process_post(txt)
bgneal@881 300 if txt != new_txt:
bgneal@881 301 logger.debug("content changed")
bgneal@881 302 logger.debug("original: %s", txt)
bgneal@881 303 logger.debug("changed: %s", new_txt)
bgneal@866 304 s.append(new_txt)
bgneal@860 305
bgneal@860 306 import pprint
bgneal@860 307 pprint.pprint(s)