Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 894:101728976f9c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Check html for <img src="http:...">. Older Smiley code generated absolute URLs for smiley images. Check for this and if found, save the model to force regeneration of HTML.
author Brian Neal <bgneal@gmail.com>
date Wed, 18 Feb 2015 21:20:31 -0600
parents ae146e30d588
children e7c549e4dbf7
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@882 9 import base64
bgneal@888 10 import httplib
bgneal@859 11 import logging
bgneal@859 12 from optparse import make_option
bgneal@888 13 import os
bgneal@863 14 import re
bgneal@863 15 import signal
bgneal@881 16 import socket
bgneal@881 17 import urllib
bgneal@868 18 import urlparse
bgneal@881 19 import uuid
bgneal@859 20
bgneal@859 21 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 22 from django.conf import settings
bgneal@894 23 from lxml import etree
bgneal@863 24 import markdown.inlinepatterns
bgneal@881 25 from PIL import Image
bgneal@859 26
bgneal@860 27 from comments.models import Comment
bgneal@860 28 from forums.models import Post
bgneal@881 29 from core.s3 import S3Bucket
bgneal@860 30
bgneal@860 31
bgneal@859 32 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 33 logger = logging.getLogger(__name__)
bgneal@859 34
bgneal@871 35 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 36 re.DOTALL | re.UNICODE)
bgneal@871 37 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 38 re.DOTALL | re.UNICODE)
bgneal@863 39
bgneal@868 40 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@866 41 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 42
bgneal@881 43 PHOTO_MAX_SIZE = (660, 720)
bgneal@881 44 PHOTO_BASE_URL = 'https://s3.amazonaws.com/'
bgneal@881 45 PHOTO_BUCKET_NAME = 'sg101.forum.photos'
bgneal@881 46
bgneal@863 47 quit_flag = False
bgneal@881 48 opener = None
bgneal@881 49 bucket = None
bgneal@881 50 url_cache = {}
bgneal@863 51
bgneal@863 52
bgneal@863 53 def signal_handler(signum, frame):
bgneal@863 54 """SIGINT signal handler"""
bgneal@863 55 global quit_flag
bgneal@863 56 quit_flag = True
bgneal@863 57
bgneal@859 58
bgneal@859 59 def _setup_logging():
bgneal@859 60 logger.setLevel(logging.DEBUG)
bgneal@859 61 logger.propagate = False
bgneal@859 62 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 63 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 64 handler.setFormatter(formatter)
bgneal@859 65 logger.addHandler(handler)
bgneal@859 66
bgneal@859 67
bgneal@881 68 class ImageURLopener(urllib.FancyURLopener):
bgneal@881 69 """Our URL opener. Handles redirects as per FancyURLopener. But all other
bgneal@881 70 errors and authentication requests will raise an IOError.
bgneal@881 71 """
bgneal@881 72 HANDLED_ERRORS = set([302, 301, 303, 307])
bgneal@881 73
bgneal@881 74 def http_error_default(self, url, fp, errcode, errmsg, headers):
bgneal@881 75 return urllib.URLopener.http_error_default(self, url, fp, errcode,
bgneal@881 76 errmsg, headers)
bgneal@881 77
bgneal@881 78 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
bgneal@881 79 """Handle http errors.
bgneal@881 80 We let FancyURLopener handle the redirects, but any other error we want
bgneal@881 81 to let fail.
bgneal@881 82 """
bgneal@881 83 if errcode in self.HANDLED_ERRORS:
bgneal@881 84 name = 'http_error_%d' % errcode
bgneal@881 85 method = getattr(self, name)
bgneal@881 86 if data is None:
bgneal@881 87 result = method(url, fp, errcode, errmsg, headers)
bgneal@881 88 else:
bgneal@881 89 result = method(url, fp, errcode, errmsg, headers, data)
bgneal@881 90 if result:
bgneal@881 91 return result
bgneal@881 92 return self.http_error_default(url, fp, errcode, errmsg, headers)
bgneal@881 93
bgneal@881 94
bgneal@881 95 def download_image(src):
bgneal@881 96 """Downloads the image file from the given source URL.
bgneal@881 97
bgneal@881 98 If successful returns the path to the downloaded file. Otherwise None is
bgneal@881 99 returned.
bgneal@881 100 """
bgneal@881 101 logger.info("Retrieving %s", src)
bgneal@881 102 try:
bgneal@881 103 fn, hdrs = opener.retrieve(src)
bgneal@881 104 except IOError as ex:
bgneal@881 105 args = ex.args
bgneal@881 106 if len(args) == 4 and args[0] == 'http error':
bgneal@881 107 logger.error("http error: %d - %s", args[1], args[2])
bgneal@881 108 else:
bgneal@881 109 logger.error("%s", ex)
bgneal@881 110 return None
bgneal@881 111
bgneal@881 112 # Does it look like an image?
bgneal@881 113 content_type = hdrs.get('content-type')
bgneal@881 114 if not content_type:
bgneal@881 115 logger.error("No content-type header found")
bgneal@881 116 return None
bgneal@881 117
bgneal@888 118 file_size = os.stat(fn).st_size
bgneal@888 119 logger.info("Retrieved: %s bytes; content-type: %s", file_size, content_type)
bgneal@881 120
bgneal@881 121 parts = content_type.split('/')
bgneal@881 122 if len(parts) < 2 or parts[0] != 'image':
bgneal@881 123 logger.error("Unknown content-type: %s", content_type)
bgneal@881 124 return None
bgneal@881 125
bgneal@881 126 return fn
bgneal@881 127
bgneal@881 128
bgneal@881 129 def resize_image(img_path):
bgneal@881 130 """Resizes the image found at img_path if necessary."""
bgneal@881 131 image = Image.open(img_path)
bgneal@881 132 if image.size > PHOTO_MAX_SIZE:
bgneal@881 133 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 134 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 135 image.save(img_path)
bgneal@881 136
bgneal@881 137
bgneal@882 138 def gen_key():
bgneal@882 139 """Return a random key."""
bgneal@882 140 return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=')
bgneal@882 141
bgneal@882 142
bgneal@881 143 def upload_image(img_path):
bgneal@881 144 """Upload image file located at img_path to our S3 bucket.
bgneal@881 145
bgneal@881 146 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 147 """
bgneal@881 148 logger.info("upload_image starting")
bgneal@881 149 # Make a unique name for the image in the bucket
bgneal@881 150 ext = os.path.splitext(img_path)[1]
bgneal@882 151 file_key = gen_key() + ext
bgneal@881 152 try:
bgneal@881 153 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 154 except IOError as ex:
bgneal@881 155 logger.error("Error uploading file: %s", ex)
bgneal@881 156 return None
bgneal@881 157
bgneal@881 158
bgneal@888 159 def convert_to_ssl(parsed_url):
bgneal@888 160 """Top-level function for moving an image to SSL."""
bgneal@888 161
bgneal@888 162 src = parsed_url.geturl()
bgneal@888 163
bgneal@888 164 # Check the cache first
bgneal@888 165 new_url = url_cache.get(src)
bgneal@888 166 if new_url:
bgneal@888 167 logger.info("Found URL in cache: %s => %s", src, new_url)
bgneal@888 168 return new_url
bgneal@888 169
bgneal@888 170 # It has been observed that at least 2 different services
bgneal@888 171 # serve up the same image on https: with the URL otherwise the same.
bgneal@888 172 # Check to see if the image is available via https first.
bgneal@888 173 new_url = check_https_availability(parsed_url)
bgneal@888 174 if new_url:
bgneal@889 175 url_cache[src] = new_url
bgneal@888 176 return new_url
bgneal@888 177
bgneal@888 178 # If none of the above worked, try to download and upload to our S3 bucket
bgneal@889 179 new_url = save_image_to_cloud(src)
bgneal@889 180 if new_url:
bgneal@889 181 url_cache[src] = new_url
bgneal@889 182 return new_url
bgneal@888 183
bgneal@888 184
bgneal@888 185 def check_https_availability(parsed_url):
bgneal@888 186 """Given a urlparse.urlparse() result, perform a HEAD request over https
bgneal@888 187 using the same net location and path. If we get a response that indicates an
bgneal@888 188 image is available, return the url of the image over https. Otherwise return
bgneal@888 189 None.
bgneal@888 190 """
bgneal@888 191 logger.info("Checking https availability for %s", parsed_url.geturl())
bgneal@888 192 con = httplib.HTTPSConnection(parsed_url.netloc)
bgneal@888 193 try:
bgneal@888 194 con.request('HEAD', parsed_url.path)
bgneal@888 195 except (httplib.HTTPException, socket.timeout) as ex:
bgneal@888 196 logger.info("https HEAD request failed: %s", ex)
bgneal@888 197 return None
bgneal@888 198
bgneal@888 199 content_type = None
bgneal@888 200 response = con.getresponse()
bgneal@888 201 if response.status == 200:
bgneal@888 202 content_type = response.getheader('content-type')
bgneal@888 203 if content_type:
bgneal@888 204 parts = content_type.split('/')
bgneal@888 205 if len(parts) >= 2 and parts[0] == 'image':
bgneal@888 206 url = urlparse.urlunparse(('https', ) + parsed_url[1:])
bgneal@888 207 logger.info("Image is available at %s", url)
bgneal@888 208 return url
bgneal@888 209
bgneal@888 210 logger.info('https HEAD request failed; status = %d, content-type = %s',
bgneal@888 211 response.status, content_type)
bgneal@888 212 return None
bgneal@888 213
bgneal@888 214
bgneal@868 215 def save_image_to_cloud(src):
bgneal@881 216 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 217
bgneal@881 218 Returns the new URL or None if unsuccessful.
bgneal@881 219 """
bgneal@881 220 fn = download_image(src)
bgneal@881 221 if fn:
bgneal@881 222 resize_image(fn)
bgneal@889 223 return upload_image(fn)
bgneal@881 224 return None
bgneal@868 225
bgneal@868 226
bgneal@866 227 def replace_image_markup(match):
bgneal@870 228 src_parts = match.group(8).split()
bgneal@868 229 if src_parts:
bgneal@868 230 src = src_parts[0]
bgneal@868 231 if src[0] == "<" and src[-1] == ">":
bgneal@868 232 src = src[1:-1]
bgneal@868 233 else:
bgneal@868 234 src = ''
bgneal@868 235
bgneal@868 236 title = ''
bgneal@868 237 if len(src_parts) > 1:
bgneal@868 238 title = " ".join(src_parts[1:])
bgneal@870 239 alt = match.group(1)
bgneal@868 240
bgneal@871 241 new_src = None
bgneal@868 242 if src:
bgneal@868 243 r = urlparse.urlparse(src)
bgneal@871 244 if r.hostname in SG101_HOSTS:
bgneal@871 245 new_src = r.path # convert to relative path
bgneal@871 246 elif r.scheme == 'http':
bgneal@888 247 # Try a few things to get this on ssl:
bgneal@888 248 new_src = convert_to_ssl(r)
bgneal@868 249 elif r.scheme == 'https':
bgneal@868 250 new_src = src # already https, accept it as-is
bgneal@868 251
bgneal@868 252 if new_src:
bgneal@868 253 if title:
bgneal@871 254 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 255 else:
bgneal@868 256 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 257 else:
bgneal@868 258 # something's messed up, convert to a link using original src
bgneal@868 259 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 260
bgneal@868 261 return s
bgneal@860 262
bgneal@860 263
bgneal@887 264 def warn_if_image_refs(text, model_name, pk):
bgneal@887 265 """Search text for Markdown image reference markup.
bgneal@887 266
bgneal@887 267 We aren't expecting these, but we will log something if we see any.
bgneal@887 268 """
bgneal@887 269 if IMAGE_REF_RE.search(text):
bgneal@887 270 logger.warning("Image reference found in %s pk = #%d", model_name, pk)
bgneal@887 271
bgneal@887 272
bgneal@866 273 def process_post(text):
bgneal@863 274 """Process the post object:
bgneal@863 275
bgneal@863 276 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 277 links, getting rid of plain old http sources; either converting to https
bgneal@863 278 or relative style links (if the link is to SG101).
bgneal@863 279
bgneal@863 280 """
bgneal@866 281 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 282
bgneal@863 283
bgneal@894 284 def html_check(html):
bgneal@894 285 """Return True if the given HTML fragment has <img> tags with src attributes
bgneal@894 286 that use http, and False otherwise.
bgneal@894 287 """
bgneal@894 288 if not html:
bgneal@894 289 return False
bgneal@894 290
bgneal@894 291 root = etree.HTML(html)
bgneal@894 292 for img in root.iter('img'):
bgneal@894 293 src = img.get('src')
bgneal@894 294 if src and src.lower().startswith('http:'):
bgneal@894 295 return True
bgneal@894 296 return False
bgneal@894 297
bgneal@894 298
bgneal@859 299 class Command(NoArgsCommand):
bgneal@859 300 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 301 option_list = NoArgsCommand.option_list + (
bgneal@866 302 make_option('-m', '--model',
bgneal@866 303 choices=MODEL_CHOICES,
bgneal@866 304 help="which model to update; must be one of {{{}}}".format(
bgneal@866 305 ', '.join(MODEL_CHOICES))),
bgneal@860 306 make_option('-i', '--i',
bgneal@859 307 type='int',
bgneal@863 308 help="optional first slice index; the i in [i:j]"),
bgneal@860 309 make_option('-j', '--j',
bgneal@859 310 type='int',
bgneal@863 311 help="optional second slice index; the j in [i:j]"),
bgneal@859 312 )
bgneal@859 313
bgneal@859 314 def handle_noargs(self, **options):
bgneal@859 315 _setup_logging()
bgneal@860 316 logger.info("Starting; arguments received: %s", options)
bgneal@859 317
bgneal@866 318 if options['model'] not in MODEL_CHOICES:
bgneal@866 319 raise CommandError('Please choose a --model option')
bgneal@859 320
bgneal@866 321 if options['model'] == 'comments':
bgneal@860 322 qs = Comment.objects.all()
bgneal@866 323 text_attr = 'comment'
bgneal@881 324 model_name = 'Comment'
bgneal@860 325 else:
bgneal@860 326 qs = Post.objects.all()
bgneal@866 327 text_attr = 'body'
bgneal@881 328 model_name = 'Post'
bgneal@860 329
bgneal@860 330 i, j = options['i'], options['j']
bgneal@860 331
bgneal@860 332 if i is not None and i < 0:
bgneal@860 333 raise CommandError("-i must be >= 0")
bgneal@860 334 if j is not None and j < 0:
bgneal@860 335 raise CommandError("-j must be >= 0")
bgneal@860 336 if j is not None and i is not None and j <= i:
bgneal@860 337 raise CommandError("-j must be > -i")
bgneal@860 338
bgneal@860 339 if i is not None and j is not None:
bgneal@860 340 qs = qs[i:j]
bgneal@860 341 elif i is not None and j is None:
bgneal@860 342 qs = qs[i:]
bgneal@860 343 elif i is None and j is not None:
bgneal@860 344 qs = qs[:j]
bgneal@860 345
bgneal@881 346 # Set global socket timeout
bgneal@881 347 socket.setdefaulttimeout(30)
bgneal@881 348
bgneal@863 349 # Install signal handler for ctrl-c
bgneal@863 350 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 351
bgneal@881 352 # Create URL opener to download photos
bgneal@881 353 global opener
bgneal@881 354 opener = ImageURLopener()
bgneal@881 355
bgneal@881 356 # Create bucket to upload photos
bgneal@881 357 global bucket
bgneal@881 358 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 359 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 360 base_url=PHOTO_BASE_URL,
bgneal@881 361 bucket_name=PHOTO_BUCKET_NAME)
bgneal@887 362
bgneal@887 363 if i is None:
bgneal@887 364 i = 0
bgneal@887 365
bgneal@881 366 for n, model in enumerate(qs.iterator()):
bgneal@863 367 if quit_flag:
bgneal@863 368 logger.warning("SIGINT received, exiting")
bgneal@881 369 break
bgneal@881 370 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 371 txt = getattr(model, text_attr)
bgneal@887 372 warn_if_image_refs(txt, model_name, model.pk)
bgneal@866 373 new_txt = process_post(txt)
bgneal@881 374 if txt != new_txt:
bgneal@889 375 logger.info("Content changed on %s #%d (pk = %d)",
bgneal@887 376 model_name, n + i, model.pk)
bgneal@881 377 logger.debug("original: %s", txt)
bgneal@881 378 logger.debug("changed: %s", new_txt)
bgneal@887 379 setattr(model, text_attr, new_txt)
bgneal@887 380 model.save()
bgneal@894 381 elif html_check(model.html):
bgneal@894 382 # Check for content generated with older smiley code that used
bgneal@894 383 # absolute URLs for the smiley images. If True, then just save
bgneal@894 384 # the model again to force updated HTML to be created.
bgneal@894 385 logger.info("Older Smiley HTML detected, forcing a save")
bgneal@894 386 model.save()
bgneal@860 387
bgneal@887 388 logger.info("ssl_images exiting")