Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 897:49ebeb54990a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Record if an image could not be retrieved. Added some additional stats at the end.
author Brian Neal <bgneal@gmail.com>
date Sat, 28 Feb 2015 13:52:46 -0600
parents 0054a4a88c1c
children 8fcd278d8987
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@882 9 import base64
bgneal@895 10 import datetime
bgneal@859 11 import logging
bgneal@859 12 from optparse import make_option
bgneal@888 13 import os
bgneal@863 14 import re
bgneal@863 15 import signal
bgneal@881 16 import socket
bgneal@881 17 import urllib
bgneal@868 18 import urlparse
bgneal@881 19 import uuid
bgneal@859 20
bgneal@859 21 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 22 from django.conf import settings
bgneal@894 23 from lxml import etree
bgneal@863 24 import markdown.inlinepatterns
bgneal@881 25 from PIL import Image
bgneal@859 26
bgneal@860 27 from comments.models import Comment
bgneal@860 28 from forums.models import Post
bgneal@881 29 from core.s3 import S3Bucket
bgneal@860 30
bgneal@860 31
bgneal@859 32 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 33 logger = logging.getLogger(__name__)
bgneal@859 34
bgneal@871 35 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 36 re.DOTALL | re.UNICODE)
bgneal@871 37 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 38 re.DOTALL | re.UNICODE)
bgneal@863 39
bgneal@868 40 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@866 41 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 42
bgneal@881 43 PHOTO_MAX_SIZE = (660, 720)
bgneal@881 44 PHOTO_BASE_URL = 'https://s3.amazonaws.com/'
bgneal@881 45 PHOTO_BUCKET_NAME = 'sg101.forum.photos'
bgneal@881 46
bgneal@863 47 quit_flag = False
bgneal@881 48 opener = None
bgneal@881 49 bucket = None
bgneal@881 50 url_cache = {}
bgneal@863 51
bgneal@863 52
bgneal@863 53 def signal_handler(signum, frame):
bgneal@863 54 """SIGINT signal handler"""
bgneal@863 55 global quit_flag
bgneal@863 56 quit_flag = True
bgneal@863 57
bgneal@859 58
bgneal@859 59 def _setup_logging():
bgneal@859 60 logger.setLevel(logging.DEBUG)
bgneal@859 61 logger.propagate = False
bgneal@859 62 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 63 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 64 handler.setFormatter(formatter)
bgneal@859 65 logger.addHandler(handler)
bgneal@859 66
bgneal@859 67
bgneal@881 68 class ImageURLopener(urllib.FancyURLopener):
bgneal@881 69 """Our URL opener. Handles redirects as per FancyURLopener. But all other
bgneal@881 70 errors and authentication requests will raise an IOError.
bgneal@881 71 """
bgneal@881 72 HANDLED_ERRORS = set([302, 301, 303, 307])
bgneal@881 73
bgneal@881 74 def http_error_default(self, url, fp, errcode, errmsg, headers):
bgneal@881 75 return urllib.URLopener.http_error_default(self, url, fp, errcode,
bgneal@881 76 errmsg, headers)
bgneal@881 77
bgneal@881 78 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
bgneal@881 79 """Handle http errors.
bgneal@881 80 We let FancyURLopener handle the redirects, but any other error we want
bgneal@881 81 to let fail.
bgneal@881 82 """
bgneal@881 83 if errcode in self.HANDLED_ERRORS:
bgneal@881 84 name = 'http_error_%d' % errcode
bgneal@881 85 method = getattr(self, name)
bgneal@881 86 if data is None:
bgneal@881 87 result = method(url, fp, errcode, errmsg, headers)
bgneal@881 88 else:
bgneal@881 89 result = method(url, fp, errcode, errmsg, headers, data)
bgneal@881 90 if result:
bgneal@881 91 return result
bgneal@881 92 return self.http_error_default(url, fp, errcode, errmsg, headers)
bgneal@881 93
bgneal@881 94
bgneal@881 95 def download_image(src):
bgneal@881 96 """Downloads the image file from the given source URL.
bgneal@881 97
bgneal@881 98 If successful returns the path to the downloaded file. Otherwise None is
bgneal@881 99 returned.
bgneal@881 100 """
bgneal@881 101 logger.info("Retrieving %s", src)
bgneal@881 102 try:
bgneal@881 103 fn, hdrs = opener.retrieve(src)
bgneal@881 104 except IOError as ex:
bgneal@881 105 args = ex.args
bgneal@881 106 if len(args) == 4 and args[0] == 'http error':
bgneal@881 107 logger.error("http error: %d - %s", args[1], args[2])
bgneal@881 108 else:
bgneal@881 109 logger.error("%s", ex)
bgneal@881 110 return None
bgneal@881 111
bgneal@881 112 # Does it look like an image?
bgneal@881 113 content_type = hdrs.get('content-type')
bgneal@881 114 if not content_type:
bgneal@881 115 logger.error("No content-type header found")
bgneal@881 116 return None
bgneal@881 117
bgneal@888 118 file_size = os.stat(fn).st_size
bgneal@888 119 logger.info("Retrieved: %s bytes; content-type: %s", file_size, content_type)
bgneal@881 120
bgneal@881 121 parts = content_type.split('/')
bgneal@881 122 if len(parts) < 2 or parts[0] != 'image':
bgneal@881 123 logger.error("Unknown content-type: %s", content_type)
bgneal@881 124 return None
bgneal@881 125
bgneal@881 126 return fn
bgneal@881 127
bgneal@881 128
bgneal@881 129 def resize_image(img_path):
bgneal@881 130 """Resizes the image found at img_path if necessary."""
bgneal@881 131 image = Image.open(img_path)
bgneal@881 132 if image.size > PHOTO_MAX_SIZE:
bgneal@881 133 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 134 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 135 image.save(img_path)
bgneal@881 136
bgneal@881 137
bgneal@882 138 def gen_key():
bgneal@882 139 """Return a random key."""
bgneal@882 140 return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=')
bgneal@882 141
bgneal@882 142
bgneal@881 143 def upload_image(img_path):
bgneal@881 144 """Upload image file located at img_path to our S3 bucket.
bgneal@881 145
bgneal@881 146 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 147 """
bgneal@881 148 logger.info("upload_image starting")
bgneal@881 149 # Make a unique name for the image in the bucket
bgneal@881 150 ext = os.path.splitext(img_path)[1]
bgneal@882 151 file_key = gen_key() + ext
bgneal@881 152 try:
bgneal@881 153 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 154 except IOError as ex:
bgneal@881 155 logger.error("Error uploading file: %s", ex)
bgneal@881 156 return None
bgneal@881 157
bgneal@881 158
bgneal@888 159 def convert_to_ssl(parsed_url):
bgneal@888 160 """Top-level function for moving an image to SSL."""
bgneal@888 161
bgneal@888 162 src = parsed_url.geturl()
bgneal@888 163
bgneal@888 164 # Check the cache first
bgneal@897 165 try:
bgneal@897 166 new_url = url_cache[src]
bgneal@897 167 except KeyError:
bgneal@897 168 # cache miss, try to get the file
bgneal@897 169 new_url = save_image_to_cloud(src)
bgneal@897 170 url_cache[src] = new_url
bgneal@897 171 else:
bgneal@897 172 if new_url:
bgneal@897 173 logger.info("Found URL in cache: %s => %s", src, new_url)
bgneal@897 174 else:
bgneal@897 175 logger.info("URL known to be bad, skipping: %s", src)
bgneal@888 176
bgneal@889 177 return new_url
bgneal@888 178
bgneal@888 179
bgneal@868 180 def save_image_to_cloud(src):
bgneal@881 181 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 182
bgneal@881 183 Returns the new URL or None if unsuccessful.
bgneal@881 184 """
bgneal@881 185 fn = download_image(src)
bgneal@881 186 if fn:
bgneal@881 187 resize_image(fn)
bgneal@889 188 return upload_image(fn)
bgneal@881 189 return None
bgneal@868 190
bgneal@868 191
bgneal@866 192 def replace_image_markup(match):
bgneal@870 193 src_parts = match.group(8).split()
bgneal@868 194 if src_parts:
bgneal@868 195 src = src_parts[0]
bgneal@868 196 if src[0] == "<" and src[-1] == ">":
bgneal@868 197 src = src[1:-1]
bgneal@868 198 else:
bgneal@868 199 src = ''
bgneal@868 200
bgneal@868 201 title = ''
bgneal@868 202 if len(src_parts) > 1:
bgneal@868 203 title = " ".join(src_parts[1:])
bgneal@870 204 alt = match.group(1)
bgneal@868 205
bgneal@871 206 new_src = None
bgneal@868 207 if src:
bgneal@868 208 r = urlparse.urlparse(src)
bgneal@871 209 if r.hostname in SG101_HOSTS:
bgneal@871 210 new_src = r.path # convert to relative path
bgneal@871 211 elif r.scheme == 'http':
bgneal@888 212 # Try a few things to get this on ssl:
bgneal@888 213 new_src = convert_to_ssl(r)
bgneal@868 214 elif r.scheme == 'https':
bgneal@868 215 new_src = src # already https, accept it as-is
bgneal@868 216
bgneal@868 217 if new_src:
bgneal@868 218 if title:
bgneal@871 219 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 220 else:
bgneal@868 221 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 222 else:
bgneal@868 223 # something's messed up, convert to a link using original src
bgneal@868 224 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 225
bgneal@868 226 return s
bgneal@860 227
bgneal@860 228
bgneal@887 229 def warn_if_image_refs(text, model_name, pk):
bgneal@887 230 """Search text for Markdown image reference markup.
bgneal@887 231
bgneal@887 232 We aren't expecting these, but we will log something if we see any.
bgneal@887 233 """
bgneal@887 234 if IMAGE_REF_RE.search(text):
bgneal@887 235 logger.warning("Image reference found in %s pk = #%d", model_name, pk)
bgneal@887 236
bgneal@887 237
bgneal@866 238 def process_post(text):
bgneal@863 239 """Process the post object:
bgneal@863 240
bgneal@863 241 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 242 links, getting rid of plain old http sources; either converting to https
bgneal@863 243 or relative style links (if the link is to SG101).
bgneal@863 244
bgneal@863 245 """
bgneal@866 246 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 247
bgneal@863 248
bgneal@894 249 def html_check(html):
bgneal@894 250 """Return True if the given HTML fragment has <img> tags with src attributes
bgneal@894 251 that use http, and False otherwise.
bgneal@894 252 """
bgneal@894 253 if not html:
bgneal@894 254 return False
bgneal@894 255
bgneal@894 256 root = etree.HTML(html)
bgneal@894 257 for img in root.iter('img'):
bgneal@894 258 src = img.get('src')
bgneal@894 259 if src and src.lower().startswith('http:'):
bgneal@894 260 return True
bgneal@894 261 return False
bgneal@894 262
bgneal@894 263
bgneal@859 264 class Command(NoArgsCommand):
bgneal@859 265 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 266 option_list = NoArgsCommand.option_list + (
bgneal@866 267 make_option('-m', '--model',
bgneal@866 268 choices=MODEL_CHOICES,
bgneal@866 269 help="which model to update; must be one of {{{}}}".format(
bgneal@866 270 ', '.join(MODEL_CHOICES))),
bgneal@860 271 make_option('-i', '--i',
bgneal@859 272 type='int',
bgneal@863 273 help="optional first slice index; the i in [i:j]"),
bgneal@860 274 make_option('-j', '--j',
bgneal@859 275 type='int',
bgneal@863 276 help="optional second slice index; the j in [i:j]"),
bgneal@859 277 )
bgneal@859 278
bgneal@859 279 def handle_noargs(self, **options):
bgneal@895 280 time_started = datetime.datetime.now()
bgneal@859 281 _setup_logging()
bgneal@860 282 logger.info("Starting; arguments received: %s", options)
bgneal@859 283
bgneal@866 284 if options['model'] not in MODEL_CHOICES:
bgneal@866 285 raise CommandError('Please choose a --model option')
bgneal@859 286
bgneal@866 287 if options['model'] == 'comments':
bgneal@860 288 qs = Comment.objects.all()
bgneal@866 289 text_attr = 'comment'
bgneal@881 290 model_name = 'Comment'
bgneal@860 291 else:
bgneal@860 292 qs = Post.objects.all()
bgneal@866 293 text_attr = 'body'
bgneal@881 294 model_name = 'Post'
bgneal@860 295
bgneal@860 296 i, j = options['i'], options['j']
bgneal@860 297
bgneal@860 298 if i is not None and i < 0:
bgneal@860 299 raise CommandError("-i must be >= 0")
bgneal@860 300 if j is not None and j < 0:
bgneal@860 301 raise CommandError("-j must be >= 0")
bgneal@860 302 if j is not None and i is not None and j <= i:
bgneal@860 303 raise CommandError("-j must be > -i")
bgneal@860 304
bgneal@860 305 if i is not None and j is not None:
bgneal@860 306 qs = qs[i:j]
bgneal@860 307 elif i is not None and j is None:
bgneal@860 308 qs = qs[i:]
bgneal@860 309 elif i is None and j is not None:
bgneal@860 310 qs = qs[:j]
bgneal@860 311
bgneal@881 312 # Set global socket timeout
bgneal@881 313 socket.setdefaulttimeout(30)
bgneal@881 314
bgneal@863 315 # Install signal handler for ctrl-c
bgneal@863 316 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 317
bgneal@881 318 # Create URL opener to download photos
bgneal@881 319 global opener
bgneal@881 320 opener = ImageURLopener()
bgneal@881 321
bgneal@881 322 # Create bucket to upload photos
bgneal@881 323 global bucket
bgneal@881 324 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 325 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 326 base_url=PHOTO_BASE_URL,
bgneal@881 327 bucket_name=PHOTO_BUCKET_NAME)
bgneal@887 328
bgneal@887 329 if i is None:
bgneal@887 330 i = 0
bgneal@887 331
bgneal@895 332 count = 0
bgneal@881 333 for n, model in enumerate(qs.iterator()):
bgneal@863 334 if quit_flag:
bgneal@863 335 logger.warning("SIGINT received, exiting")
bgneal@881 336 break
bgneal@881 337 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 338 txt = getattr(model, text_attr)
bgneal@887 339 warn_if_image_refs(txt, model_name, model.pk)
bgneal@866 340 new_txt = process_post(txt)
bgneal@881 341 if txt != new_txt:
bgneal@889 342 logger.info("Content changed on %s #%d (pk = %d)",
bgneal@887 343 model_name, n + i, model.pk)
bgneal@881 344 logger.debug("original: %s", txt)
bgneal@881 345 logger.debug("changed: %s", new_txt)
bgneal@887 346 setattr(model, text_attr, new_txt)
bgneal@887 347 model.save()
bgneal@894 348 elif html_check(model.html):
bgneal@894 349 # Check for content generated with older smiley code that used
bgneal@894 350 # absolute URLs for the smiley images. If True, then just save
bgneal@894 351 # the model again to force updated HTML to be created.
bgneal@894 352 logger.info("Older Smiley HTML detected, forcing a save")
bgneal@894 353 model.save()
bgneal@895 354 count += 1
bgneal@860 355
bgneal@895 356 time_finished = datetime.datetime.now()
bgneal@895 357 elapsed = time_finished - time_started
bgneal@895 358 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s",
bgneal@895 359 count, elapsed)
bgneal@897 360
bgneal@897 361 http_images = len(url_cache)
bgneal@897 362 https_images = sum(1 for v in url_cache.itervalues() if v)
bgneal@897 363 bad_images = http_images - https_images
bgneal@897 364 if http_images > 0:
bgneal@897 365 pct_saved = float(https_images) / http_images * 100.0
bgneal@897 366 else:
bgneal@897 367 pct_saved = 0.0
bgneal@897 368
bgneal@897 369 logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%",
bgneal@897 370 http_images, https_images, bad_images, pct_saved)