Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 980:3ebde23a59d0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Pass timeout to requests instead of global timeout. Global timeout didn't seem to work once we started using requests.
author Brian Neal <bgneal@gmail.com>
date Thu, 15 Oct 2015 18:35:56 -0500
parents a6331579ff43
children ef1558941bc9
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@882 9 import base64
bgneal@895 10 import datetime
bgneal@899 11 import json
bgneal@859 12 import logging
bgneal@859 13 from optparse import make_option
bgneal@888 14 import os
bgneal@863 15 import re
bgneal@863 16 import signal
bgneal@868 17 import urlparse
bgneal@881 18 import uuid
bgneal@859 19
bgneal@859 20 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 21 from django.conf import settings
bgneal@894 22 from lxml import etree
bgneal@863 23 import markdown.inlinepatterns
bgneal@881 24 from PIL import Image
bgneal@979 25 import requests
bgneal@859 26
bgneal@860 27 from comments.models import Comment
bgneal@860 28 from forums.models import Post
bgneal@979 29 from core.download import download_file
bgneal@979 30 from core.functions import remove_file
bgneal@881 31 from core.s3 import S3Bucket
bgneal@860 32
bgneal@860 33
bgneal@859 34 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 35 logger = logging.getLogger(__name__)
bgneal@859 36
bgneal@871 37 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 38 re.DOTALL | re.UNICODE)
bgneal@871 39 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 40 re.DOTALL | re.UNICODE)
bgneal@863 41
bgneal@868 42 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@963 43 WHITELIST_HOSTS = set(settings.USER_IMAGES_SOURCES)
bgneal@866 44 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 45
bgneal@881 46 PHOTO_MAX_SIZE = (660, 720)
bgneal@979 47 PHOTO_BASE_URL = settings.HOT_LINK_PHOTOS_BASE_URL
bgneal@979 48 PHOTO_BUCKET_NAME = settings.HOT_LINK_PHOTOS_BUCKET
bgneal@881 49
bgneal@899 50 CACHE_FILENAME = 'ssl_images_cache.json'
bgneal@899 51
bgneal@863 52 quit_flag = False
bgneal@881 53 bucket = None
bgneal@881 54 url_cache = {}
bgneal@899 55 bad_hosts = set()
bgneal@980 56 request_timeout = None
bgneal@863 57
bgneal@863 58
bgneal@863 59 def signal_handler(signum, frame):
bgneal@863 60 """SIGINT signal handler"""
bgneal@863 61 global quit_flag
bgneal@863 62 quit_flag = True
bgneal@863 63
bgneal@859 64
bgneal@859 65 def _setup_logging():
bgneal@859 66 logger.setLevel(logging.DEBUG)
bgneal@859 67 logger.propagate = False
bgneal@859 68 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 69 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 70 handler.setFormatter(formatter)
bgneal@859 71 logger.addHandler(handler)
bgneal@859 72
bgneal@859 73
bgneal@979 74 def resize_image(img_path):
bgneal@979 75 """Resizes the image found at img_path if necessary.
bgneal@979 76
bgneal@979 77 Returns True if the image was resized or resizing wasn't necessary.
bgneal@979 78 Returns False if the image could not be read or processed.
bgneal@881 79 """
bgneal@979 80 try:
bgneal@979 81 image = Image.open(img_path)
bgneal@979 82 except IOError as ex:
bgneal@979 83 logger.error("Error opening %s: %s", img_path, ex)
bgneal@979 84 return False
bgneal@881 85
bgneal@881 86 if image.size > PHOTO_MAX_SIZE:
bgneal@881 87 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 88 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 89 image.save(img_path)
bgneal@881 90
bgneal@979 91 return True
bgneal@979 92
bgneal@881 93
bgneal@882 94 def gen_key():
bgneal@882 95 """Return a random key."""
bgneal@882 96 return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=')
bgneal@882 97
bgneal@882 98
bgneal@881 99 def upload_image(img_path):
bgneal@881 100 """Upload image file located at img_path to our S3 bucket.
bgneal@881 101
bgneal@881 102 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 103 """
bgneal@881 104 logger.info("upload_image starting")
bgneal@881 105 # Make a unique name for the image in the bucket
bgneal@881 106 ext = os.path.splitext(img_path)[1]
bgneal@882 107 file_key = gen_key() + ext
bgneal@881 108 try:
bgneal@881 109 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 110 except IOError as ex:
bgneal@881 111 logger.error("Error uploading file: %s", ex)
bgneal@881 112 return None
bgneal@881 113
bgneal@881 114
bgneal@888 115 def convert_to_ssl(parsed_url):
bgneal@888 116 """Top-level function for moving an image to SSL."""
bgneal@888 117
bgneal@888 118 src = parsed_url.geturl()
bgneal@888 119
bgneal@899 120 if parsed_url.hostname in bad_hosts:
bgneal@899 121 logger.info("Host known to be bad, skipping: %s", src)
bgneal@899 122 return None
bgneal@899 123
bgneal@899 124 # Check the cache
bgneal@897 125 try:
bgneal@897 126 new_url = url_cache[src]
bgneal@897 127 except KeyError:
bgneal@897 128 # cache miss, try to get the file
bgneal@899 129 new_url = save_image_to_cloud(parsed_url)
bgneal@897 130 url_cache[src] = new_url
bgneal@897 131 else:
bgneal@897 132 if new_url:
bgneal@897 133 logger.info("Found URL in cache: %s => %s", src, new_url)
bgneal@897 134 else:
bgneal@897 135 logger.info("URL known to be bad, skipping: %s", src)
bgneal@888 136
bgneal@889 137 return new_url
bgneal@888 138
bgneal@888 139
bgneal@899 140 def save_image_to_cloud(parsed_url):
bgneal@881 141 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 142
bgneal@881 143 Returns the new URL or None if unsuccessful.
bgneal@881 144 """
bgneal@979 145 url = parsed_url.geturl()
bgneal@979 146 fn = None
bgneal@979 147 try:
bgneal@980 148 fn = download_file(url, timeout=request_timeout)
bgneal@979 149 except requests.ConnectionError as ex:
bgneal@979 150 logger.error("ConnectionError, ignoring host %s", parsed_url.hostname)
bgneal@979 151 bad_hosts.add(parsed_url.hostname)
bgneal@979 152 except requests.RequestException as ex:
bgneal@979 153 logger.error("%s", ex)
bgneal@979 154 except Exception as ex:
bgneal@979 155 logger.exception("%s", ex)
bgneal@979 156
bgneal@881 157 if fn:
bgneal@979 158 with remove_file(fn):
bgneal@979 159 if resize_image(fn):
bgneal@979 160 return upload_image(fn)
bgneal@881 161 return None
bgneal@868 162
bgneal@868 163
bgneal@866 164 def replace_image_markup(match):
bgneal@870 165 src_parts = match.group(8).split()
bgneal@868 166 if src_parts:
bgneal@868 167 src = src_parts[0]
bgneal@868 168 if src[0] == "<" and src[-1] == ">":
bgneal@868 169 src = src[1:-1]
bgneal@868 170 else:
bgneal@868 171 src = ''
bgneal@868 172
bgneal@868 173 title = ''
bgneal@868 174 if len(src_parts) > 1:
bgneal@868 175 title = " ".join(src_parts[1:])
bgneal@870 176 alt = match.group(1)
bgneal@868 177
bgneal@871 178 new_src = None
bgneal@868 179 if src:
bgneal@868 180 r = urlparse.urlparse(src)
bgneal@871 181 if r.hostname in SG101_HOSTS:
bgneal@871 182 new_src = r.path # convert to relative path
bgneal@871 183 elif r.scheme == 'http':
bgneal@888 184 # Try a few things to get this on ssl:
bgneal@888 185 new_src = convert_to_ssl(r)
bgneal@868 186 elif r.scheme == 'https':
bgneal@963 187 if r.hostname in WHITELIST_HOSTS:
bgneal@963 188 new_src = src # already in whitelist
bgneal@963 189 else:
bgneal@963 190 new_src = convert_to_ssl(r)
bgneal@868 191
bgneal@868 192 if new_src:
bgneal@868 193 if title:
bgneal@871 194 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 195 else:
bgneal@868 196 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 197 else:
bgneal@868 198 # something's messed up, convert to a link using original src
bgneal@868 199 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 200
bgneal@868 201 return s
bgneal@860 202
bgneal@860 203
bgneal@887 204 def warn_if_image_refs(text, model_name, pk):
bgneal@887 205 """Search text for Markdown image reference markup.
bgneal@887 206
bgneal@887 207 We aren't expecting these, but we will log something if we see any.
bgneal@887 208 """
bgneal@887 209 if IMAGE_REF_RE.search(text):
bgneal@887 210 logger.warning("Image reference found in %s pk = #%d", model_name, pk)
bgneal@887 211
bgneal@887 212
bgneal@866 213 def process_post(text):
bgneal@863 214 """Process the post object:
bgneal@863 215
bgneal@863 216 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 217 links, getting rid of plain old http sources; either converting to https
bgneal@863 218 or relative style links (if the link is to SG101).
bgneal@863 219
bgneal@863 220 """
bgneal@866 221 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 222
bgneal@863 223
bgneal@894 224 def html_check(html):
bgneal@894 225 """Return True if the given HTML fragment has <img> tags with src attributes
bgneal@894 226 that use http, and False otherwise.
bgneal@894 227 """
bgneal@894 228 if not html:
bgneal@894 229 return False
bgneal@894 230
bgneal@894 231 root = etree.HTML(html)
bgneal@894 232 for img in root.iter('img'):
bgneal@894 233 src = img.get('src')
bgneal@894 234 if src and src.lower().startswith('http:'):
bgneal@894 235 return True
bgneal@894 236 return False
bgneal@894 237
bgneal@894 238
bgneal@859 239 class Command(NoArgsCommand):
bgneal@859 240 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 241 option_list = NoArgsCommand.option_list + (
bgneal@866 242 make_option('-m', '--model',
bgneal@866 243 choices=MODEL_CHOICES,
bgneal@866 244 help="which model to update; must be one of {{{}}}".format(
bgneal@866 245 ', '.join(MODEL_CHOICES))),
bgneal@860 246 make_option('-i', '--i',
bgneal@859 247 type='int',
bgneal@863 248 help="optional first slice index; the i in [i:j]"),
bgneal@860 249 make_option('-j', '--j',
bgneal@859 250 type='int',
bgneal@863 251 help="optional second slice index; the j in [i:j]"),
bgneal@898 252 make_option('-t', '--timeout',
bgneal@980 253 type='float',
bgneal@979 254 help="optional socket timeout (secs)",
bgneal@980 255 default=30.0),
bgneal@859 256 )
bgneal@859 257
bgneal@859 258 def handle_noargs(self, **options):
bgneal@895 259 time_started = datetime.datetime.now()
bgneal@859 260 _setup_logging()
bgneal@860 261 logger.info("Starting; arguments received: %s", options)
bgneal@859 262
bgneal@866 263 if options['model'] not in MODEL_CHOICES:
bgneal@866 264 raise CommandError('Please choose a --model option')
bgneal@859 265
bgneal@866 266 if options['model'] == 'comments':
bgneal@860 267 qs = Comment.objects.all()
bgneal@866 268 text_attr = 'comment'
bgneal@881 269 model_name = 'Comment'
bgneal@860 270 else:
bgneal@860 271 qs = Post.objects.all()
bgneal@866 272 text_attr = 'body'
bgneal@881 273 model_name = 'Post'
bgneal@860 274
bgneal@860 275 i, j = options['i'], options['j']
bgneal@860 276
bgneal@860 277 if i is not None and i < 0:
bgneal@860 278 raise CommandError("-i must be >= 0")
bgneal@860 279 if j is not None and j < 0:
bgneal@860 280 raise CommandError("-j must be >= 0")
bgneal@860 281 if j is not None and i is not None and j <= i:
bgneal@860 282 raise CommandError("-j must be > -i")
bgneal@860 283
bgneal@860 284 if i is not None and j is not None:
bgneal@860 285 qs = qs[i:j]
bgneal@860 286 elif i is not None and j is None:
bgneal@860 287 qs = qs[i:]
bgneal@860 288 elif i is None and j is not None:
bgneal@860 289 qs = qs[:j]
bgneal@860 290
bgneal@881 291 # Set global socket timeout
bgneal@980 292 global request_timeout
bgneal@980 293 request_timeout = options.get('timeout')
bgneal@980 294 logger.info("Using socket timeout of %4.2f", request_timeout)
bgneal@881 295
bgneal@863 296 # Install signal handler for ctrl-c
bgneal@863 297 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 298
bgneal@881 299 # Create bucket to upload photos
bgneal@881 300 global bucket
bgneal@881 301 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 302 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 303 base_url=PHOTO_BASE_URL,
bgneal@881 304 bucket_name=PHOTO_BUCKET_NAME)
bgneal@887 305
bgneal@899 306 # Load cached info from previous runs
bgneal@899 307 load_cache()
bgneal@899 308
bgneal@887 309 if i is None:
bgneal@887 310 i = 0
bgneal@887 311
bgneal@895 312 count = 0
bgneal@881 313 for n, model in enumerate(qs.iterator()):
bgneal@863 314 if quit_flag:
bgneal@863 315 logger.warning("SIGINT received, exiting")
bgneal@881 316 break
bgneal@881 317 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 318 txt = getattr(model, text_attr)
bgneal@887 319 warn_if_image_refs(txt, model_name, model.pk)
bgneal@866 320 new_txt = process_post(txt)
bgneal@881 321 if txt != new_txt:
bgneal@889 322 logger.info("Content changed on %s #%d (pk = %d)",
bgneal@887 323 model_name, n + i, model.pk)
bgneal@881 324 logger.debug("original: %s", txt)
bgneal@881 325 logger.debug("changed: %s", new_txt)
bgneal@887 326 setattr(model, text_attr, new_txt)
bgneal@887 327 model.save()
bgneal@894 328 elif html_check(model.html):
bgneal@894 329 # Check for content generated with older smiley code that used
bgneal@894 330 # absolute URLs for the smiley images. If True, then just save
bgneal@894 331 # the model again to force updated HTML to be created.
bgneal@894 332 logger.info("Older Smiley HTML detected, forcing a save")
bgneal@894 333 model.save()
bgneal@895 334 count += 1
bgneal@860 335
bgneal@895 336 time_finished = datetime.datetime.now()
bgneal@895 337 elapsed = time_finished - time_started
bgneal@895 338 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s",
bgneal@895 339 count, elapsed)
bgneal@897 340
bgneal@897 341 http_images = len(url_cache)
bgneal@897 342 https_images = sum(1 for v in url_cache.itervalues() if v)
bgneal@897 343 bad_images = http_images - https_images
bgneal@897 344 if http_images > 0:
bgneal@897 345 pct_saved = float(https_images) / http_images * 100.0
bgneal@897 346 else:
bgneal@897 347 pct_saved = 0.0
bgneal@897 348
bgneal@897 349 logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%",
bgneal@897 350 http_images, https_images, bad_images, pct_saved)
bgneal@899 351
bgneal@899 352 save_cache()
bgneal@899 353 logger.info("ssl_images done")
bgneal@899 354
bgneal@899 355
bgneal@899 356 def load_cache():
bgneal@899 357 """Load cache from previous runs."""
bgneal@899 358 logger.info("Loading cached information")
bgneal@899 359 try:
bgneal@899 360 with open(CACHE_FILENAME, 'r') as fp:
bgneal@899 361 d = json.load(fp)
bgneal@899 362 except IOError as ex:
bgneal@899 363 logger.error("Cache file (%s) IOError: %s", CACHE_FILENAME, ex)
bgneal@899 364 return
bgneal@899 365 except ValueError:
bgneal@899 366 logger.error("Mangled cache file: %s", CACHE_FILENAME)
bgneal@899 367 return
bgneal@899 368
bgneal@899 369 global bad_hosts, url_cache
bgneal@899 370 try:
bgneal@899 371 bad_hosts = set(d['bad_hosts'])
bgneal@899 372 url_cache = d['url_cache']
bgneal@899 373 except KeyError:
bgneal@899 374 logger.error("Malformed cache file: %s", CACHE_FILENAME)
bgneal@899 375
bgneal@899 376
bgneal@899 377 def save_cache():
bgneal@899 378 """Save our cache to a file for future runs."""
bgneal@899 379 logger.info("Saving cached information")
bgneal@899 380 d = {'bad_hosts': list(bad_hosts), 'url_cache': url_cache}
bgneal@899 381 with open(CACHE_FILENAME, 'w') as fp:
bgneal@899 382 json.dump(d, fp, indent=4)