Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 943:cf9918328c64

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Haystack tweaks for Django 1.7.7. I had to upgrade to Haystack 2.3.1 to get it to work with Django 1.7.7. I also had to update the Xapian backend. But I ran into problems. On my laptop anyway (Ubuntu 14.0.4), xapian gets mad when search terms are greater than 245 chars (or something) when indexing. So I created a custom field that would simply omit terms greater than 64 chars and used this field everywhere I previously used a CharField. Secondly, the custom search form was broken now. Something changed in the Xapian backend and exact searches stopped working. Fortunately the auto_query (which I was using originally and broke during an upgrade) started working again. So I cut the search form back over to doing an auto_query. I kept the form the same (3 fields) because I didn't want to change the form and I think it's better that way.
author Brian Neal <bgneal@gmail.com>
date Wed, 13 May 2015 20:25:07 -0500
parents 62cd07bb891c
children 4619290d171d
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@882 9 import base64
bgneal@895 10 import datetime
bgneal@899 11 import json
bgneal@859 12 import logging
bgneal@859 13 from optparse import make_option
bgneal@888 14 import os
bgneal@863 15 import re
bgneal@863 16 import signal
bgneal@881 17 import socket
bgneal@881 18 import urllib
bgneal@868 19 import urlparse
bgneal@881 20 import uuid
bgneal@859 21
bgneal@859 22 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 23 from django.conf import settings
bgneal@894 24 from lxml import etree
bgneal@863 25 import markdown.inlinepatterns
bgneal@881 26 from PIL import Image
bgneal@859 27
bgneal@860 28 from comments.models import Comment
bgneal@860 29 from forums.models import Post
bgneal@881 30 from core.s3 import S3Bucket
bgneal@860 31
bgneal@860 32
bgneal@859 33 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 34 logger = logging.getLogger(__name__)
bgneal@859 35
bgneal@871 36 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 37 re.DOTALL | re.UNICODE)
bgneal@871 38 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 39 re.DOTALL | re.UNICODE)
bgneal@863 40
bgneal@868 41 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@866 42 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 43
bgneal@881 44 PHOTO_MAX_SIZE = (660, 720)
bgneal@881 45 PHOTO_BASE_URL = 'https://s3.amazonaws.com/'
bgneal@881 46 PHOTO_BUCKET_NAME = 'sg101.forum.photos'
bgneal@881 47
bgneal@899 48 CACHE_FILENAME = 'ssl_images_cache.json'
bgneal@899 49
bgneal@863 50 quit_flag = False
bgneal@881 51 opener = None
bgneal@881 52 bucket = None
bgneal@881 53 url_cache = {}
bgneal@899 54 bad_hosts = set()
bgneal@863 55
bgneal@863 56
bgneal@863 57 def signal_handler(signum, frame):
bgneal@863 58 """SIGINT signal handler"""
bgneal@863 59 global quit_flag
bgneal@863 60 quit_flag = True
bgneal@863 61
bgneal@859 62
bgneal@859 63 def _setup_logging():
bgneal@859 64 logger.setLevel(logging.DEBUG)
bgneal@859 65 logger.propagate = False
bgneal@859 66 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 67 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 68 handler.setFormatter(formatter)
bgneal@859 69 logger.addHandler(handler)
bgneal@859 70
bgneal@859 71
bgneal@881 72 class ImageURLopener(urllib.FancyURLopener):
bgneal@881 73 """Our URL opener. Handles redirects as per FancyURLopener. But all other
bgneal@881 74 errors and authentication requests will raise an IOError.
bgneal@881 75 """
bgneal@881 76 HANDLED_ERRORS = set([302, 301, 303, 307])
bgneal@881 77
bgneal@881 78 def http_error_default(self, url, fp, errcode, errmsg, headers):
bgneal@881 79 return urllib.URLopener.http_error_default(self, url, fp, errcode,
bgneal@881 80 errmsg, headers)
bgneal@881 81
bgneal@881 82 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
bgneal@881 83 """Handle http errors.
bgneal@881 84 We let FancyURLopener handle the redirects, but any other error we want
bgneal@881 85 to let fail.
bgneal@881 86 """
bgneal@881 87 if errcode in self.HANDLED_ERRORS:
bgneal@881 88 name = 'http_error_%d' % errcode
bgneal@881 89 method = getattr(self, name)
bgneal@881 90 if data is None:
bgneal@881 91 result = method(url, fp, errcode, errmsg, headers)
bgneal@881 92 else:
bgneal@881 93 result = method(url, fp, errcode, errmsg, headers, data)
bgneal@881 94 if result:
bgneal@881 95 return result
bgneal@881 96 return self.http_error_default(url, fp, errcode, errmsg, headers)
bgneal@881 97
bgneal@881 98
bgneal@899 99 def download_image(parsed_url):
bgneal@881 100 """Downloads the image file from the given source URL.
bgneal@881 101
bgneal@881 102 If successful returns the path to the downloaded file. Otherwise None is
bgneal@881 103 returned.
bgneal@881 104 """
bgneal@899 105 src = parsed_url.geturl()
bgneal@881 106 logger.info("Retrieving %s", src)
bgneal@881 107 try:
bgneal@881 108 fn, hdrs = opener.retrieve(src)
bgneal@881 109 except IOError as ex:
bgneal@899 110 args = ex.args if ex.args else []
bgneal@881 111 if len(args) == 4 and args[0] == 'http error':
bgneal@881 112 logger.error("http error: %d - %s", args[1], args[2])
bgneal@899 113 elif len(args) == 2 and isinstance(args[1], socket.gaierror):
bgneal@899 114 logger.error("gaierror, ignoring host %s", parsed_url.hostname)
bgneal@899 115 bad_hosts.add(parsed_url.hostname)
bgneal@881 116 else:
bgneal@881 117 logger.error("%s", ex)
bgneal@881 118 return None
bgneal@881 119
bgneal@881 120 # Does it look like an image?
bgneal@881 121 content_type = hdrs.get('content-type')
bgneal@881 122 if not content_type:
bgneal@881 123 logger.error("No content-type header found")
bgneal@881 124 return None
bgneal@881 125
bgneal@888 126 file_size = os.stat(fn).st_size
bgneal@888 127 logger.info("Retrieved: %s bytes; content-type: %s", file_size, content_type)
bgneal@881 128
bgneal@881 129 parts = content_type.split('/')
bgneal@881 130 if len(parts) < 2 or parts[0] != 'image':
bgneal@881 131 logger.error("Unknown content-type: %s", content_type)
bgneal@881 132 return None
bgneal@881 133
bgneal@881 134 return fn
bgneal@881 135
bgneal@881 136
bgneal@881 137 def resize_image(img_path):
bgneal@881 138 """Resizes the image found at img_path if necessary."""
bgneal@881 139 image = Image.open(img_path)
bgneal@881 140 if image.size > PHOTO_MAX_SIZE:
bgneal@881 141 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 142 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 143 image.save(img_path)
bgneal@881 144
bgneal@881 145
bgneal@882 146 def gen_key():
bgneal@882 147 """Return a random key."""
bgneal@882 148 return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=')
bgneal@882 149
bgneal@882 150
bgneal@881 151 def upload_image(img_path):
bgneal@881 152 """Upload image file located at img_path to our S3 bucket.
bgneal@881 153
bgneal@881 154 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 155 """
bgneal@881 156 logger.info("upload_image starting")
bgneal@881 157 # Make a unique name for the image in the bucket
bgneal@881 158 ext = os.path.splitext(img_path)[1]
bgneal@882 159 file_key = gen_key() + ext
bgneal@881 160 try:
bgneal@881 161 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 162 except IOError as ex:
bgneal@881 163 logger.error("Error uploading file: %s", ex)
bgneal@881 164 return None
bgneal@881 165
bgneal@881 166
bgneal@888 167 def convert_to_ssl(parsed_url):
bgneal@888 168 """Top-level function for moving an image to SSL."""
bgneal@888 169
bgneal@888 170 src = parsed_url.geturl()
bgneal@888 171
bgneal@899 172 if parsed_url.hostname in bad_hosts:
bgneal@899 173 logger.info("Host known to be bad, skipping: %s", src)
bgneal@899 174 return None
bgneal@899 175
bgneal@899 176 # Check the cache
bgneal@897 177 try:
bgneal@897 178 new_url = url_cache[src]
bgneal@897 179 except KeyError:
bgneal@897 180 # cache miss, try to get the file
bgneal@899 181 new_url = save_image_to_cloud(parsed_url)
bgneal@897 182 url_cache[src] = new_url
bgneal@897 183 else:
bgneal@897 184 if new_url:
bgneal@897 185 logger.info("Found URL in cache: %s => %s", src, new_url)
bgneal@897 186 else:
bgneal@897 187 logger.info("URL known to be bad, skipping: %s", src)
bgneal@888 188
bgneal@889 189 return new_url
bgneal@888 190
bgneal@888 191
bgneal@899 192 def save_image_to_cloud(parsed_url):
bgneal@881 193 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 194
bgneal@881 195 Returns the new URL or None if unsuccessful.
bgneal@881 196 """
bgneal@899 197 fn = download_image(parsed_url)
bgneal@881 198 if fn:
bgneal@881 199 resize_image(fn)
bgneal@889 200 return upload_image(fn)
bgneal@881 201 return None
bgneal@868 202
bgneal@868 203
bgneal@866 204 def replace_image_markup(match):
bgneal@870 205 src_parts = match.group(8).split()
bgneal@868 206 if src_parts:
bgneal@868 207 src = src_parts[0]
bgneal@868 208 if src[0] == "<" and src[-1] == ">":
bgneal@868 209 src = src[1:-1]
bgneal@868 210 else:
bgneal@868 211 src = ''
bgneal@868 212
bgneal@868 213 title = ''
bgneal@868 214 if len(src_parts) > 1:
bgneal@868 215 title = " ".join(src_parts[1:])
bgneal@870 216 alt = match.group(1)
bgneal@868 217
bgneal@871 218 new_src = None
bgneal@868 219 if src:
bgneal@868 220 r = urlparse.urlparse(src)
bgneal@871 221 if r.hostname in SG101_HOSTS:
bgneal@871 222 new_src = r.path # convert to relative path
bgneal@871 223 elif r.scheme == 'http':
bgneal@888 224 # Try a few things to get this on ssl:
bgneal@888 225 new_src = convert_to_ssl(r)
bgneal@868 226 elif r.scheme == 'https':
bgneal@868 227 new_src = src # already https, accept it as-is
bgneal@868 228
bgneal@868 229 if new_src:
bgneal@868 230 if title:
bgneal@871 231 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 232 else:
bgneal@868 233 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 234 else:
bgneal@868 235 # something's messed up, convert to a link using original src
bgneal@868 236 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 237
bgneal@868 238 return s
bgneal@860 239
bgneal@860 240
bgneal@887 241 def warn_if_image_refs(text, model_name, pk):
bgneal@887 242 """Search text for Markdown image reference markup.
bgneal@887 243
bgneal@887 244 We aren't expecting these, but we will log something if we see any.
bgneal@887 245 """
bgneal@887 246 if IMAGE_REF_RE.search(text):
bgneal@887 247 logger.warning("Image reference found in %s pk = #%d", model_name, pk)
bgneal@887 248
bgneal@887 249
bgneal@866 250 def process_post(text):
bgneal@863 251 """Process the post object:
bgneal@863 252
bgneal@863 253 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 254 links, getting rid of plain old http sources; either converting to https
bgneal@863 255 or relative style links (if the link is to SG101).
bgneal@863 256
bgneal@863 257 """
bgneal@866 258 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 259
bgneal@863 260
bgneal@894 261 def html_check(html):
bgneal@894 262 """Return True if the given HTML fragment has <img> tags with src attributes
bgneal@894 263 that use http, and False otherwise.
bgneal@894 264 """
bgneal@894 265 if not html:
bgneal@894 266 return False
bgneal@894 267
bgneal@894 268 root = etree.HTML(html)
bgneal@894 269 for img in root.iter('img'):
bgneal@894 270 src = img.get('src')
bgneal@894 271 if src and src.lower().startswith('http:'):
bgneal@894 272 return True
bgneal@894 273 return False
bgneal@894 274
bgneal@894 275
bgneal@859 276 class Command(NoArgsCommand):
bgneal@859 277 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 278 option_list = NoArgsCommand.option_list + (
bgneal@866 279 make_option('-m', '--model',
bgneal@866 280 choices=MODEL_CHOICES,
bgneal@866 281 help="which model to update; must be one of {{{}}}".format(
bgneal@866 282 ', '.join(MODEL_CHOICES))),
bgneal@860 283 make_option('-i', '--i',
bgneal@859 284 type='int',
bgneal@863 285 help="optional first slice index; the i in [i:j]"),
bgneal@860 286 make_option('-j', '--j',
bgneal@859 287 type='int',
bgneal@863 288 help="optional second slice index; the j in [i:j]"),
bgneal@898 289 make_option('-t', '--timeout',
bgneal@898 290 type='int',
bgneal@898 291 help="optional socket timeout (secs)"),
bgneal@859 292 )
bgneal@859 293
bgneal@859 294 def handle_noargs(self, **options):
bgneal@895 295 time_started = datetime.datetime.now()
bgneal@859 296 _setup_logging()
bgneal@860 297 logger.info("Starting; arguments received: %s", options)
bgneal@859 298
bgneal@866 299 if options['model'] not in MODEL_CHOICES:
bgneal@866 300 raise CommandError('Please choose a --model option')
bgneal@859 301
bgneal@866 302 if options['model'] == 'comments':
bgneal@860 303 qs = Comment.objects.all()
bgneal@866 304 text_attr = 'comment'
bgneal@881 305 model_name = 'Comment'
bgneal@860 306 else:
bgneal@860 307 qs = Post.objects.all()
bgneal@866 308 text_attr = 'body'
bgneal@881 309 model_name = 'Post'
bgneal@860 310
bgneal@860 311 i, j = options['i'], options['j']
bgneal@860 312
bgneal@860 313 if i is not None and i < 0:
bgneal@860 314 raise CommandError("-i must be >= 0")
bgneal@860 315 if j is not None and j < 0:
bgneal@860 316 raise CommandError("-j must be >= 0")
bgneal@860 317 if j is not None and i is not None and j <= i:
bgneal@860 318 raise CommandError("-j must be > -i")
bgneal@860 319
bgneal@860 320 if i is not None and j is not None:
bgneal@860 321 qs = qs[i:j]
bgneal@860 322 elif i is not None and j is None:
bgneal@860 323 qs = qs[i:]
bgneal@860 324 elif i is None and j is not None:
bgneal@860 325 qs = qs[:j]
bgneal@860 326
bgneal@881 327 # Set global socket timeout
bgneal@898 328 timeout = options.get('timeout', 30)
bgneal@898 329 logger.info("Setting socket timeout to %d", timeout)
bgneal@898 330 socket.setdefaulttimeout(timeout)
bgneal@881 331
bgneal@863 332 # Install signal handler for ctrl-c
bgneal@863 333 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 334
bgneal@881 335 # Create URL opener to download photos
bgneal@881 336 global opener
bgneal@881 337 opener = ImageURLopener()
bgneal@881 338
bgneal@881 339 # Create bucket to upload photos
bgneal@881 340 global bucket
bgneal@881 341 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 342 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 343 base_url=PHOTO_BASE_URL,
bgneal@881 344 bucket_name=PHOTO_BUCKET_NAME)
bgneal@887 345
bgneal@899 346 # Load cached info from previous runs
bgneal@899 347 load_cache()
bgneal@899 348
bgneal@887 349 if i is None:
bgneal@887 350 i = 0
bgneal@887 351
bgneal@895 352 count = 0
bgneal@881 353 for n, model in enumerate(qs.iterator()):
bgneal@863 354 if quit_flag:
bgneal@863 355 logger.warning("SIGINT received, exiting")
bgneal@881 356 break
bgneal@881 357 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 358 txt = getattr(model, text_attr)
bgneal@887 359 warn_if_image_refs(txt, model_name, model.pk)
bgneal@866 360 new_txt = process_post(txt)
bgneal@881 361 if txt != new_txt:
bgneal@889 362 logger.info("Content changed on %s #%d (pk = %d)",
bgneal@887 363 model_name, n + i, model.pk)
bgneal@881 364 logger.debug("original: %s", txt)
bgneal@881 365 logger.debug("changed: %s", new_txt)
bgneal@887 366 setattr(model, text_attr, new_txt)
bgneal@887 367 model.save()
bgneal@894 368 elif html_check(model.html):
bgneal@894 369 # Check for content generated with older smiley code that used
bgneal@894 370 # absolute URLs for the smiley images. If True, then just save
bgneal@894 371 # the model again to force updated HTML to be created.
bgneal@894 372 logger.info("Older Smiley HTML detected, forcing a save")
bgneal@894 373 model.save()
bgneal@895 374 count += 1
bgneal@860 375
bgneal@895 376 time_finished = datetime.datetime.now()
bgneal@895 377 elapsed = time_finished - time_started
bgneal@895 378 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s",
bgneal@895 379 count, elapsed)
bgneal@897 380
bgneal@897 381 http_images = len(url_cache)
bgneal@897 382 https_images = sum(1 for v in url_cache.itervalues() if v)
bgneal@897 383 bad_images = http_images - https_images
bgneal@897 384 if http_images > 0:
bgneal@897 385 pct_saved = float(https_images) / http_images * 100.0
bgneal@897 386 else:
bgneal@897 387 pct_saved = 0.0
bgneal@897 388
bgneal@897 389 logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%",
bgneal@897 390 http_images, https_images, bad_images, pct_saved)
bgneal@899 391
bgneal@899 392 save_cache()
bgneal@899 393 logger.info("ssl_images done")
bgneal@899 394
bgneal@899 395
bgneal@899 396 def load_cache():
bgneal@899 397 """Load cache from previous runs."""
bgneal@899 398 logger.info("Loading cached information")
bgneal@899 399 try:
bgneal@899 400 with open(CACHE_FILENAME, 'r') as fp:
bgneal@899 401 d = json.load(fp)
bgneal@899 402 except IOError as ex:
bgneal@899 403 logger.error("Cache file (%s) IOError: %s", CACHE_FILENAME, ex)
bgneal@899 404 return
bgneal@899 405 except ValueError:
bgneal@899 406 logger.error("Mangled cache file: %s", CACHE_FILENAME)
bgneal@899 407 return
bgneal@899 408
bgneal@899 409 global bad_hosts, url_cache
bgneal@899 410 try:
bgneal@899 411 bad_hosts = set(d['bad_hosts'])
bgneal@899 412 url_cache = d['url_cache']
bgneal@899 413 except KeyError:
bgneal@899 414 logger.error("Malformed cache file: %s", CACHE_FILENAME)
bgneal@899 415
bgneal@899 416
bgneal@899 417 def save_cache():
bgneal@899 418 """Save our cache to a file for future runs."""
bgneal@899 419 logger.info("Saving cached information")
bgneal@899 420 d = {'bad_hosts': list(bad_hosts), 'url_cache': url_cache}
bgneal@899 421 with open(CACHE_FILENAME, 'w') as fp:
bgneal@899 422 json.dump(d, fp, indent=4)