Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 979:a6331579ff43

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Convert ssl_images to use download_file.
author Brian Neal <bgneal@gmail.com>
date Fri, 09 Oct 2015 22:20:32 -0500
parents 4619290d171d
children 3ebde23a59d0
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@882 9 import base64
bgneal@895 10 import datetime
bgneal@899 11 import json
bgneal@859 12 import logging
bgneal@859 13 from optparse import make_option
bgneal@888 14 import os
bgneal@863 15 import re
bgneal@863 16 import signal
bgneal@881 17 import socket
bgneal@868 18 import urlparse
bgneal@881 19 import uuid
bgneal@859 20
bgneal@859 21 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 22 from django.conf import settings
bgneal@894 23 from lxml import etree
bgneal@863 24 import markdown.inlinepatterns
bgneal@881 25 from PIL import Image
bgneal@979 26 import requests
bgneal@859 27
bgneal@860 28 from comments.models import Comment
bgneal@860 29 from forums.models import Post
bgneal@979 30 from core.download import download_file
bgneal@979 31 from core.functions import remove_file
bgneal@881 32 from core.s3 import S3Bucket
bgneal@860 33
bgneal@860 34
bgneal@859 35 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 36 logger = logging.getLogger(__name__)
bgneal@859 37
bgneal@871 38 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 39 re.DOTALL | re.UNICODE)
bgneal@871 40 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 41 re.DOTALL | re.UNICODE)
bgneal@863 42
bgneal@868 43 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@963 44 WHITELIST_HOSTS = set(settings.USER_IMAGES_SOURCES)
bgneal@866 45 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 46
bgneal@881 47 PHOTO_MAX_SIZE = (660, 720)
bgneal@979 48 PHOTO_BASE_URL = settings.HOT_LINK_PHOTOS_BASE_URL
bgneal@979 49 PHOTO_BUCKET_NAME = settings.HOT_LINK_PHOTOS_BUCKET
bgneal@881 50
bgneal@899 51 CACHE_FILENAME = 'ssl_images_cache.json'
bgneal@899 52
bgneal@863 53 quit_flag = False
bgneal@881 54 bucket = None
bgneal@881 55 url_cache = {}
bgneal@899 56 bad_hosts = set()
bgneal@863 57
bgneal@863 58
bgneal@863 59 def signal_handler(signum, frame):
bgneal@863 60 """SIGINT signal handler"""
bgneal@863 61 global quit_flag
bgneal@863 62 quit_flag = True
bgneal@863 63
bgneal@859 64
bgneal@859 65 def _setup_logging():
bgneal@859 66 logger.setLevel(logging.DEBUG)
bgneal@859 67 logger.propagate = False
bgneal@859 68 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 69 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 70 handler.setFormatter(formatter)
bgneal@859 71 logger.addHandler(handler)
bgneal@859 72
bgneal@859 73
bgneal@979 74 def resize_image(img_path):
bgneal@979 75 """Resizes the image found at img_path if necessary.
bgneal@979 76
bgneal@979 77 Returns True if the image was resized or resizing wasn't necessary.
bgneal@979 78 Returns False if the image could not be read or processed.
bgneal@881 79 """
bgneal@979 80 try:
bgneal@979 81 image = Image.open(img_path)
bgneal@979 82 except IOError as ex:
bgneal@979 83 logger.error("Error opening %s: %s", img_path, ex)
bgneal@979 84 return False
bgneal@881 85
bgneal@881 86 if image.size > PHOTO_MAX_SIZE:
bgneal@881 87 logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE)
bgneal@881 88 image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS)
bgneal@881 89 image.save(img_path)
bgneal@881 90
bgneal@979 91 return True
bgneal@979 92
bgneal@881 93
bgneal@882 94 def gen_key():
bgneal@882 95 """Return a random key."""
bgneal@882 96 return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=')
bgneal@882 97
bgneal@882 98
bgneal@881 99 def upload_image(img_path):
bgneal@881 100 """Upload image file located at img_path to our S3 bucket.
bgneal@881 101
bgneal@881 102 Returns the URL of the image in the bucket or None if an error occurs.
bgneal@881 103 """
bgneal@881 104 logger.info("upload_image starting")
bgneal@881 105 # Make a unique name for the image in the bucket
bgneal@881 106 ext = os.path.splitext(img_path)[1]
bgneal@882 107 file_key = gen_key() + ext
bgneal@881 108 try:
bgneal@881 109 return bucket.upload_from_filename(file_key, img_path, public=True)
bgneal@881 110 except IOError as ex:
bgneal@881 111 logger.error("Error uploading file: %s", ex)
bgneal@881 112 return None
bgneal@881 113
bgneal@881 114
bgneal@888 115 def convert_to_ssl(parsed_url):
bgneal@888 116 """Top-level function for moving an image to SSL."""
bgneal@888 117
bgneal@888 118 src = parsed_url.geturl()
bgneal@888 119
bgneal@899 120 if parsed_url.hostname in bad_hosts:
bgneal@899 121 logger.info("Host known to be bad, skipping: %s", src)
bgneal@899 122 return None
bgneal@899 123
bgneal@899 124 # Check the cache
bgneal@897 125 try:
bgneal@897 126 new_url = url_cache[src]
bgneal@897 127 except KeyError:
bgneal@897 128 # cache miss, try to get the file
bgneal@899 129 new_url = save_image_to_cloud(parsed_url)
bgneal@897 130 url_cache[src] = new_url
bgneal@897 131 else:
bgneal@897 132 if new_url:
bgneal@897 133 logger.info("Found URL in cache: %s => %s", src, new_url)
bgneal@897 134 else:
bgneal@897 135 logger.info("URL known to be bad, skipping: %s", src)
bgneal@888 136
bgneal@889 137 return new_url
bgneal@888 138
bgneal@888 139
bgneal@899 140 def save_image_to_cloud(parsed_url):
bgneal@881 141 """Downloads an image at a given source URL. Uploads it to cloud storage.
bgneal@881 142
bgneal@881 143 Returns the new URL or None if unsuccessful.
bgneal@881 144 """
bgneal@979 145 url = parsed_url.geturl()
bgneal@979 146 fn = None
bgneal@979 147 try:
bgneal@979 148 fn = download_file(url)
bgneal@979 149 except requests.ConnectionError as ex:
bgneal@979 150 logger.error("ConnectionError, ignoring host %s", parsed_url.hostname)
bgneal@979 151 bad_hosts.add(parsed_url.hostname)
bgneal@979 152 except requests.RequestException as ex:
bgneal@979 153 logger.error("%s", ex)
bgneal@979 154 except Exception as ex:
bgneal@979 155 logger.exception("%s", ex)
bgneal@979 156
bgneal@881 157 if fn:
bgneal@979 158 with remove_file(fn):
bgneal@979 159 if resize_image(fn):
bgneal@979 160 return upload_image(fn)
bgneal@881 161 return None
bgneal@868 162
bgneal@868 163
bgneal@866 164 def replace_image_markup(match):
bgneal@870 165 src_parts = match.group(8).split()
bgneal@868 166 if src_parts:
bgneal@868 167 src = src_parts[0]
bgneal@868 168 if src[0] == "<" and src[-1] == ">":
bgneal@868 169 src = src[1:-1]
bgneal@868 170 else:
bgneal@868 171 src = ''
bgneal@868 172
bgneal@868 173 title = ''
bgneal@868 174 if len(src_parts) > 1:
bgneal@868 175 title = " ".join(src_parts[1:])
bgneal@870 176 alt = match.group(1)
bgneal@868 177
bgneal@871 178 new_src = None
bgneal@868 179 if src:
bgneal@868 180 r = urlparse.urlparse(src)
bgneal@871 181 if r.hostname in SG101_HOSTS:
bgneal@871 182 new_src = r.path # convert to relative path
bgneal@871 183 elif r.scheme == 'http':
bgneal@888 184 # Try a few things to get this on ssl:
bgneal@888 185 new_src = convert_to_ssl(r)
bgneal@868 186 elif r.scheme == 'https':
bgneal@963 187 if r.hostname in WHITELIST_HOSTS:
bgneal@963 188 new_src = src # already in whitelist
bgneal@963 189 else:
bgneal@963 190 new_src = convert_to_ssl(r)
bgneal@868 191
bgneal@868 192 if new_src:
bgneal@868 193 if title:
bgneal@871 194 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 195 else:
bgneal@868 196 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 197 else:
bgneal@868 198 # something's messed up, convert to a link using original src
bgneal@868 199 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 200
bgneal@868 201 return s
bgneal@860 202
bgneal@860 203
bgneal@887 204 def warn_if_image_refs(text, model_name, pk):
bgneal@887 205 """Search text for Markdown image reference markup.
bgneal@887 206
bgneal@887 207 We aren't expecting these, but we will log something if we see any.
bgneal@887 208 """
bgneal@887 209 if IMAGE_REF_RE.search(text):
bgneal@887 210 logger.warning("Image reference found in %s pk = #%d", model_name, pk)
bgneal@887 211
bgneal@887 212
bgneal@866 213 def process_post(text):
bgneal@863 214 """Process the post object:
bgneal@863 215
bgneal@863 216 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 217 links, getting rid of plain old http sources; either converting to https
bgneal@863 218 or relative style links (if the link is to SG101).
bgneal@863 219
bgneal@863 220 """
bgneal@866 221 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 222
bgneal@863 223
bgneal@894 224 def html_check(html):
bgneal@894 225 """Return True if the given HTML fragment has <img> tags with src attributes
bgneal@894 226 that use http, and False otherwise.
bgneal@894 227 """
bgneal@894 228 if not html:
bgneal@894 229 return False
bgneal@894 230
bgneal@894 231 root = etree.HTML(html)
bgneal@894 232 for img in root.iter('img'):
bgneal@894 233 src = img.get('src')
bgneal@894 234 if src and src.lower().startswith('http:'):
bgneal@894 235 return True
bgneal@894 236 return False
bgneal@894 237
bgneal@894 238
bgneal@859 239 class Command(NoArgsCommand):
bgneal@859 240 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 241 option_list = NoArgsCommand.option_list + (
bgneal@866 242 make_option('-m', '--model',
bgneal@866 243 choices=MODEL_CHOICES,
bgneal@866 244 help="which model to update; must be one of {{{}}}".format(
bgneal@866 245 ', '.join(MODEL_CHOICES))),
bgneal@860 246 make_option('-i', '--i',
bgneal@859 247 type='int',
bgneal@863 248 help="optional first slice index; the i in [i:j]"),
bgneal@860 249 make_option('-j', '--j',
bgneal@859 250 type='int',
bgneal@863 251 help="optional second slice index; the j in [i:j]"),
bgneal@898 252 make_option('-t', '--timeout',
bgneal@898 253 type='int',
bgneal@979 254 help="optional socket timeout (secs)",
bgneal@979 255 default=30),
bgneal@859 256 )
bgneal@859 257
bgneal@859 258 def handle_noargs(self, **options):
bgneal@895 259 time_started = datetime.datetime.now()
bgneal@859 260 _setup_logging()
bgneal@860 261 logger.info("Starting; arguments received: %s", options)
bgneal@859 262
bgneal@866 263 if options['model'] not in MODEL_CHOICES:
bgneal@866 264 raise CommandError('Please choose a --model option')
bgneal@859 265
bgneal@866 266 if options['model'] == 'comments':
bgneal@860 267 qs = Comment.objects.all()
bgneal@866 268 text_attr = 'comment'
bgneal@881 269 model_name = 'Comment'
bgneal@860 270 else:
bgneal@860 271 qs = Post.objects.all()
bgneal@866 272 text_attr = 'body'
bgneal@881 273 model_name = 'Post'
bgneal@860 274
bgneal@860 275 i, j = options['i'], options['j']
bgneal@860 276
bgneal@860 277 if i is not None and i < 0:
bgneal@860 278 raise CommandError("-i must be >= 0")
bgneal@860 279 if j is not None and j < 0:
bgneal@860 280 raise CommandError("-j must be >= 0")
bgneal@860 281 if j is not None and i is not None and j <= i:
bgneal@860 282 raise CommandError("-j must be > -i")
bgneal@860 283
bgneal@860 284 if i is not None and j is not None:
bgneal@860 285 qs = qs[i:j]
bgneal@860 286 elif i is not None and j is None:
bgneal@860 287 qs = qs[i:]
bgneal@860 288 elif i is None and j is not None:
bgneal@860 289 qs = qs[:j]
bgneal@860 290
bgneal@881 291 # Set global socket timeout
bgneal@979 292 timeout = options.get('timeout')
bgneal@898 293 logger.info("Setting socket timeout to %d", timeout)
bgneal@898 294 socket.setdefaulttimeout(timeout)
bgneal@881 295
bgneal@863 296 # Install signal handler for ctrl-c
bgneal@863 297 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 298
bgneal@881 299 # Create bucket to upload photos
bgneal@881 300 global bucket
bgneal@881 301 bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY,
bgneal@881 302 secret_key=settings.USER_PHOTOS_SECRET_KEY,
bgneal@881 303 base_url=PHOTO_BASE_URL,
bgneal@881 304 bucket_name=PHOTO_BUCKET_NAME)
bgneal@887 305
bgneal@899 306 # Load cached info from previous runs
bgneal@899 307 load_cache()
bgneal@899 308
bgneal@887 309 if i is None:
bgneal@887 310 i = 0
bgneal@887 311
bgneal@895 312 count = 0
bgneal@881 313 for n, model in enumerate(qs.iterator()):
bgneal@863 314 if quit_flag:
bgneal@863 315 logger.warning("SIGINT received, exiting")
bgneal@881 316 break
bgneal@881 317 logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk)
bgneal@866 318 txt = getattr(model, text_attr)
bgneal@887 319 warn_if_image_refs(txt, model_name, model.pk)
bgneal@866 320 new_txt = process_post(txt)
bgneal@881 321 if txt != new_txt:
bgneal@889 322 logger.info("Content changed on %s #%d (pk = %d)",
bgneal@887 323 model_name, n + i, model.pk)
bgneal@881 324 logger.debug("original: %s", txt)
bgneal@881 325 logger.debug("changed: %s", new_txt)
bgneal@887 326 setattr(model, text_attr, new_txt)
bgneal@887 327 model.save()
bgneal@894 328 elif html_check(model.html):
bgneal@894 329 # Check for content generated with older smiley code that used
bgneal@894 330 # absolute URLs for the smiley images. If True, then just save
bgneal@894 331 # the model again to force updated HTML to be created.
bgneal@894 332 logger.info("Older Smiley HTML detected, forcing a save")
bgneal@894 333 model.save()
bgneal@895 334 count += 1
bgneal@860 335
bgneal@895 336 time_finished = datetime.datetime.now()
bgneal@895 337 elapsed = time_finished - time_started
bgneal@895 338 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s",
bgneal@895 339 count, elapsed)
bgneal@897 340
bgneal@897 341 http_images = len(url_cache)
bgneal@897 342 https_images = sum(1 for v in url_cache.itervalues() if v)
bgneal@897 343 bad_images = http_images - https_images
bgneal@897 344 if http_images > 0:
bgneal@897 345 pct_saved = float(https_images) / http_images * 100.0
bgneal@897 346 else:
bgneal@897 347 pct_saved = 0.0
bgneal@897 348
bgneal@897 349 logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%",
bgneal@897 350 http_images, https_images, bad_images, pct_saved)
bgneal@899 351
bgneal@899 352 save_cache()
bgneal@899 353 logger.info("ssl_images done")
bgneal@899 354
bgneal@899 355
bgneal@899 356 def load_cache():
bgneal@899 357 """Load cache from previous runs."""
bgneal@899 358 logger.info("Loading cached information")
bgneal@899 359 try:
bgneal@899 360 with open(CACHE_FILENAME, 'r') as fp:
bgneal@899 361 d = json.load(fp)
bgneal@899 362 except IOError as ex:
bgneal@899 363 logger.error("Cache file (%s) IOError: %s", CACHE_FILENAME, ex)
bgneal@899 364 return
bgneal@899 365 except ValueError:
bgneal@899 366 logger.error("Mangled cache file: %s", CACHE_FILENAME)
bgneal@899 367 return
bgneal@899 368
bgneal@899 369 global bad_hosts, url_cache
bgneal@899 370 try:
bgneal@899 371 bad_hosts = set(d['bad_hosts'])
bgneal@899 372 url_cache = d['url_cache']
bgneal@899 373 except KeyError:
bgneal@899 374 logger.error("Malformed cache file: %s", CACHE_FILENAME)
bgneal@899 375
bgneal@899 376
bgneal@899 377 def save_cache():
bgneal@899 378 """Save our cache to a file for future runs."""
bgneal@899 379 logger.info("Saving cached information")
bgneal@899 380 d = {'bad_hosts': list(bad_hosts), 'url_cache': url_cache}
bgneal@899 381 with open(CACHE_FILENAME, 'w') as fp:
bgneal@899 382 json.dump(d, fp, indent=4)