Mercurial > public > sg101
changeset 909:90632d090bbc
Merge with upstream.
| author | Brian Neal <bgneal@gmail.com> |
|---|---|
| date | Mon, 16 Mar 2015 21:03:37 -0500 |
| parents | 4dee923a2f6d (diff) 2181da65c98b (current diff) |
| children | 90acb29478e9 |
| files | |
| diffstat | 5 files changed, 794 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Mon Mar 16 20:58:40 2015 -0500 +++ b/.hgignore Mon Mar 16 21:03:37 2015 -0500 @@ -7,6 +7,7 @@ secrets.json *.db *.mp3 +.tags static_serve media/avatars/users media/badges
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/management/commands/ssl_images.py Mon Mar 16 21:03:37 2015 -0500 @@ -0,0 +1,422 @@ +""" +ssl_images is a custom manage.py command to convert forum post and comment +images to https. It does this by rewriting the markup: + - Images with src = http://surfguitar101.com/something are rewritten to be + /something. + - Non SG101 images that use http: are downloaded, resized, and uploaded to + an S3 bucket. The src attribute is replaced with the new S3 URL. +""" +import base64 +import datetime +import json +import logging +from optparse import make_option +import os +import re +import signal +import socket +import urllib +import urlparse +import uuid + +from django.core.management.base import NoArgsCommand, CommandError +from django.conf import settings +from lxml import etree +import markdown.inlinepatterns +from PIL import Image + +from comments.models import Comment +from forums.models import Post +from core.s3 import S3Bucket + + +LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log') +logger = logging.getLogger(__name__) + +IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE, + re.DOTALL | re.UNICODE) +IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE, + re.DOTALL | re.UNICODE) + +SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com']) +MODEL_CHOICES = ['comments', 'posts'] + +PHOTO_MAX_SIZE = (660, 720) +PHOTO_BASE_URL = 'https://s3.amazonaws.com/' +PHOTO_BUCKET_NAME = 'sg101.forum.photos' + +CACHE_FILENAME = 'ssl_images_cache.json' + +quit_flag = False +opener = None +bucket = None +url_cache = {} +bad_hosts = set() + + +def signal_handler(signum, frame): + """SIGINT signal handler""" + global quit_flag + quit_flag = True + + +def _setup_logging(): + logger.setLevel(logging.DEBUG) + logger.propagate = False + handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8') + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + +class ImageURLopener(urllib.FancyURLopener): + """Our URL opener. Handles redirects as per FancyURLopener. But all other + errors and authentication requests will raise an IOError. + """ + HANDLED_ERRORS = set([302, 301, 303, 307]) + + def http_error_default(self, url, fp, errcode, errmsg, headers): + return urllib.URLopener.http_error_default(self, url, fp, errcode, + errmsg, headers) + + def http_error(self, url, fp, errcode, errmsg, headers, data=None): + """Handle http errors. + We let FancyURLopener handle the redirects, but any other error we want + to let fail. + """ + if errcode in self.HANDLED_ERRORS: + name = 'http_error_%d' % errcode + method = getattr(self, name) + if data is None: + result = method(url, fp, errcode, errmsg, headers) + else: + result = method(url, fp, errcode, errmsg, headers, data) + if result: + return result + return self.http_error_default(url, fp, errcode, errmsg, headers) + + +def download_image(parsed_url): + """Downloads the image file from the given source URL. + + If successful returns the path to the downloaded file. Otherwise None is + returned. + """ + src = parsed_url.geturl() + logger.info("Retrieving %s", src) + try: + fn, hdrs = opener.retrieve(src) + except IOError as ex: + args = ex.args if ex.args else [] + if len(args) == 4 and args[0] == 'http error': + logger.error("http error: %d - %s", args[1], args[2]) + elif len(args) == 2 and isinstance(args[1], socket.gaierror): + logger.error("gaierror, ignoring host %s", parsed_url.hostname) + bad_hosts.add(parsed_url.hostname) + else: + logger.error("%s", ex) + return None + + # Does it look like an image? + content_type = hdrs.get('content-type') + if not content_type: + logger.error("No content-type header found") + return None + + file_size = os.stat(fn).st_size + logger.info("Retrieved: %s bytes; content-type: %s", file_size, content_type) + + parts = content_type.split('/') + if len(parts) < 2 or parts[0] != 'image': + logger.error("Unknown content-type: %s", content_type) + return None + + return fn + + +def resize_image(img_path): + """Resizes the image found at img_path if necessary.""" + image = Image.open(img_path) + if image.size > PHOTO_MAX_SIZE: + logger.info('Resizing from %s to %s', image.size, PHOTO_MAX_SIZE) + image.thumbnail(PHOTO_MAX_SIZE, Image.ANTIALIAS) + image.save(img_path) + + +def gen_key(): + """Return a random key.""" + return base64.b64encode(uuid.uuid4().bytes, '-_').rstrip('=') + + +def upload_image(img_path): + """Upload image file located at img_path to our S3 bucket. + + Returns the URL of the image in the bucket or None if an error occurs. + """ + logger.info("upload_image starting") + # Make a unique name for the image in the bucket + ext = os.path.splitext(img_path)[1] + file_key = gen_key() + ext + try: + return bucket.upload_from_filename(file_key, img_path, public=True) + except IOError as ex: + logger.error("Error uploading file: %s", ex) + return None + + +def convert_to_ssl(parsed_url): + """Top-level function for moving an image to SSL.""" + + src = parsed_url.geturl() + + if parsed_url.hostname in bad_hosts: + logger.info("Host known to be bad, skipping: %s", src) + return None + + # Check the cache + try: + new_url = url_cache[src] + except KeyError: + # cache miss, try to get the file + new_url = save_image_to_cloud(parsed_url) + url_cache[src] = new_url + else: + if new_url: + logger.info("Found URL in cache: %s => %s", src, new_url) + else: + logger.info("URL known to be bad, skipping: %s", src) + + return new_url + + +def save_image_to_cloud(parsed_url): + """Downloads an image at a given source URL. Uploads it to cloud storage. + + Returns the new URL or None if unsuccessful. + """ + fn = download_image(parsed_url) + if fn: + resize_image(fn) + return upload_image(fn) + return None + + +def replace_image_markup(match): + src_parts = match.group(8).split() + if src_parts: + src = src_parts[0] + if src[0] == "<" and src[-1] == ">": + src = src[1:-1] + else: + src = '' + + title = '' + if len(src_parts) > 1: + title = " ".join(src_parts[1:]) + alt = match.group(1) + + new_src = None + if src: + r = urlparse.urlparse(src) + if r.hostname in SG101_HOSTS: + new_src = r.path # convert to relative path + elif r.scheme == 'http': + # Try a few things to get this on ssl: + new_src = convert_to_ssl(r) + elif r.scheme == 'https': + new_src = src # already https, accept it as-is + + if new_src: + if title: + s = u''.format(alt=alt, src=new_src, title=title) + else: + s = u''.format(alt=alt, src=new_src) + else: + # something's messed up, convert to a link using original src + s = u'[{alt}]({src})'.format(alt=alt, src=src) + + return s + + +def warn_if_image_refs(text, model_name, pk): + """Search text for Markdown image reference markup. + + We aren't expecting these, but we will log something if we see any. + """ + if IMAGE_REF_RE.search(text): + logger.warning("Image reference found in %s pk = #%d", model_name, pk) + + +def process_post(text): + """Process the post object: + + A regex substitution is run on the post's text field. This fixes up image + links, getting rid of plain old http sources; either converting to https + or relative style links (if the link is to SG101). + + """ + return IMAGE_LINK_RE.sub(replace_image_markup, text) + + +def html_check(html): + """Return True if the given HTML fragment has <img> tags with src attributes + that use http, and False otherwise. + """ + if not html: + return False + + root = etree.HTML(html) + for img in root.iter('img'): + src = img.get('src') + if src and src.lower().startswith('http:'): + return True + return False + + +class Command(NoArgsCommand): + help = "Rewrite forum posts and comments to not use http for images" + option_list = NoArgsCommand.option_list + ( + make_option('-m', '--model', + choices=MODEL_CHOICES, + help="which model to update; must be one of {{{}}}".format( + ', '.join(MODEL_CHOICES))), + make_option('-i', '--i', + type='int', + help="optional first slice index; the i in [i:j]"), + make_option('-j', '--j', + type='int', + help="optional second slice index; the j in [i:j]"), + make_option('-t', '--timeout', + type='int', + help="optional socket timeout (secs)"), + ) + + def handle_noargs(self, **options): + time_started = datetime.datetime.now() + _setup_logging() + logger.info("Starting; arguments received: %s", options) + + if options['model'] not in MODEL_CHOICES: + raise CommandError('Please choose a --model option') + + if options['model'] == 'comments': + qs = Comment.objects.all() + text_attr = 'comment' + model_name = 'Comment' + else: + qs = Post.objects.all() + text_attr = 'body' + model_name = 'Post' + + i, j = options['i'], options['j'] + + if i is not None and i < 0: + raise CommandError("-i must be >= 0") + if j is not None and j < 0: + raise CommandError("-j must be >= 0") + if j is not None and i is not None and j <= i: + raise CommandError("-j must be > -i") + + if i is not None and j is not None: + qs = qs[i:j] + elif i is not None and j is None: + qs = qs[i:] + elif i is None and j is not None: + qs = qs[:j] + + # Set global socket timeout + timeout = options.get('timeout', 30) + logger.info("Setting socket timeout to %d", timeout) + socket.setdefaulttimeout(timeout) + + # Install signal handler for ctrl-c + signal.signal(signal.SIGINT, signal_handler) + + # Create URL opener to download photos + global opener + opener = ImageURLopener() + + # Create bucket to upload photos + global bucket + bucket = S3Bucket(access_key=settings.USER_PHOTOS_ACCESS_KEY, + secret_key=settings.USER_PHOTOS_SECRET_KEY, + base_url=PHOTO_BASE_URL, + bucket_name=PHOTO_BUCKET_NAME) + + # Load cached info from previous runs + load_cache() + + if i is None: + i = 0 + + count = 0 + for n, model in enumerate(qs.iterator()): + if quit_flag: + logger.warning("SIGINT received, exiting") + break + logger.info("Processing %s #%d (pk = %d)", model_name, n + i, model.pk) + txt = getattr(model, text_attr) + warn_if_image_refs(txt, model_name, model.pk) + new_txt = process_post(txt) + if txt != new_txt: + logger.info("Content changed on %s #%d (pk = %d)", + model_name, n + i, model.pk) + logger.debug("original: %s", txt) + logger.debug("changed: %s", new_txt) + setattr(model, text_attr, new_txt) + model.save() + elif html_check(model.html): + # Check for content generated with older smiley code that used + # absolute URLs for the smiley images. If True, then just save + # the model again to force updated HTML to be created. + logger.info("Older Smiley HTML detected, forcing a save") + model.save() + count += 1 + + time_finished = datetime.datetime.now() + elapsed = time_finished - time_started + logger.info("ssl_images exiting; number of objects: %d; elapsed: %s", + count, elapsed) + + http_images = len(url_cache) + https_images = sum(1 for v in url_cache.itervalues() if v) + bad_images = http_images - https_images + if http_images > 0: + pct_saved = float(https_images) / http_images * 100.0 + else: + pct_saved = 0.0 + + logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%", + http_images, https_images, bad_images, pct_saved) + + save_cache() + logger.info("ssl_images done") + + +def load_cache(): + """Load cache from previous runs.""" + logger.info("Loading cached information") + try: + with open(CACHE_FILENAME, 'r') as fp: + d = json.load(fp) + except IOError as ex: + logger.error("Cache file (%s) IOError: %s", CACHE_FILENAME, ex) + return + except ValueError: + logger.error("Mangled cache file: %s", CACHE_FILENAME) + return + + global bad_hosts, url_cache + try: + bad_hosts = set(d['bad_hosts']) + url_cache = d['url_cache'] + except KeyError: + logger.error("Malformed cache file: %s", CACHE_FILENAME) + + +def save_cache(): + """Save our cache to a file for future runs.""" + logger.info("Saving cached information") + d = {'bad_hosts': list(bad_hosts), 'url_cache': url_cache} + with open(CACHE_FILENAME, 'w') as fp: + json.dump(d, fp, indent=4)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/mdexts/ssl_images.py Mon Mar 16 21:03:37 2015 -0500 @@ -0,0 +1,28 @@ +""" +A python-markdown extension to turn <img> tags with http: source attributes into +<a> tags. +""" +from urlparse import urlparse + +import markdown + + +class SslImagesTreeprocessor(markdown.treeprocessors.Treeprocessor): + + def run(self, root): + for node in root.iter('img'): + src = node.get('src') + if src: + url = urlparse(src) + if url.scheme == 'http': + node.clear() + node.tag = 'a' + node.text = 'Click for image' + node.set('href', url.geturl()) + + +class SslImagesExtension(markdown.Extension): + + def extendMarkdown(self, md, md_globals): + tree_proc = SslImagesTreeprocessor() + md.treeprocessors.add('ssl_images', tree_proc, '>inline')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/tests/test_mdexts.py Mon Mar 16 21:03:37 2015 -0500 @@ -0,0 +1,27 @@ +"""Testing our custom Markdown extensions.""" + +import unittest + +import markdown + +from core.mdexts.ssl_images import SslImagesExtension + + +class SslImagesExtTestCase(unittest.TestCase): + """Tests for the SslImagesExtension.""" + + def setUp(self): + self.md = markdown.Markdown(extensions=[SslImagesExtension()]) + + def test_simple(self): + self.assertEqual(self.md.convert(''), '') + self.assertEqual(self.md.convert('1'), '<p>1</p>') + + def test_no_change(self): + self.assertEqual(self.md.convert(''), + u'<p><img alt="image" src="https://example.com/1.jpg" /></p>') + + def test_change(self): + text = u'' + html = u'<p><a href="http://example.com/1.jpg">Click for image</a></p>' + self.assertEqual(self.md.convert(text), html)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/tests/test_ssl_images.py Mon Mar 16 21:03:37 2015 -0500 @@ -0,0 +1,316 @@ +"""Unit tests for the ssl_images management command.""" +import re +import unittest +from urlparse import urlparse + +import mock + +from core.management.commands.ssl_images import html_check +from core.management.commands.ssl_images import process_post +import core.management.commands.ssl_images + + +class ProcessPostTestCase(unittest.TestCase): + + SG101_RE = re.compile(r'http://(?:www\.)?surfguitar101.com/', re.I) + + def tearDown(self): + core.management.commands.ssl_images.url_cache = {} + + def test_empty_string(self): + s = process_post('') + self.assertEqual(s, '') + + def test_no_matches(self): + test_str = """Here is a post that doesn't contain any image links at + all. It also spans lines. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_sg101_images(self): + test_str = """An image:  + And another: . + More stuff here.""" + expected = self.SG101_RE.sub('/', test_str) + result = process_post(test_str) + self.assertNotEqual(test_str, expected) + self.assertEqual(expected, result) + + def test_sg101_with_newlines(self): + test_str = """An image:  + with trailing text.""" + expected = """An image:  + with trailing text.""" + result = process_post(test_str) + self.assertNotEqual(test_str, expected) + self.assertEqual(expected, result) + + def test_https_already(self): + test_str = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_https_sg101(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_multiple_non_http(self): + test_str = """An image:  + And another: . + More stuff here.""" + expected = """An image:  + And another: . + More stuff here.""" + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_already_with_title(self): + test_str = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_sg101_with_title(self): + test_str = """An image on SG101: +  + It's cool. + """ + expected = """An image on SG101: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_sg101_brackets(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_already_brackets(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + @mock.patch('core.management.commands.ssl_images.check_https_availability', + new=lambda r: None) + def test_simple_replacement(self, upload_mock): + old_src = 'http://example.com/images/my_image.jpg' + new_src = 'https://cloud.com/ABCDEF.jpg' + test_str = """Here is a really cool http: based image: +  + Cool, right?""".format(old_src) + expected = """Here is a really cool http: based image: +  + Cool, right?""".format(new_src) + + upload_mock.return_value = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + upload_mock.assert_called_once_with(old_src) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + @mock.patch('core.management.commands.ssl_images.check_https_availability', + new=lambda r: None) + def test_multiple_replacement(self, upload_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'http://example.com/static/wow.gif', + 'http://example.com/media/a/b/c/pic.png', + ] + new_src = [ + 'https://cloud.com/some/path/012345.jpg', + 'https://cloud.com/some/path/6789AB.gif', + 'https://cloud.com/some/path/CDEF01.png', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another one:  + And finally +  + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + upload_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + expected_args = [mock.call(c) for c in old_src] + self.assertEqual(upload_mock.call_args_list, expected_args) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + @mock.patch('core.management.commands.ssl_images.check_https_availability', + new=lambda r: None) + def test_multiple_replacement_2(self, upload_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'https://example.com/static/wow.gif', + 'http://www.surfguitar101.com/media/a/b/c/pic.png', + 'http://surfguitar101.com/media/a/b/c/pic2.png', + ] + new_src = [ + 'https://cloud.com/some/path/012345.jpg', + 'https://example.com/static/wow.gif', + '/media/a/b/c/pic.png', + '/media/a/b/c/pic2.png', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another two:   + And finally + . + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + upload_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + upload_mock.assert_called_once_with(old_src[0]) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + @mock.patch('core.management.commands.ssl_images.check_https_availability', + new=lambda r: None) + def test_caching(self, upload_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'http://example.com/static/wow.gif', + 'http://example.com/images/my_image.jpg', + ] + new_src = [ + 'https://cloud.com/some/path/012345.jpg', + 'https://cloud.com/some/path/6789AB.gif', + 'https://cloud.com/some/path/012345.jpg', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another one:  + And finally +  + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + upload_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + expected_args = [mock.call(c) for c in old_src[:2]] + self.assertEqual(upload_mock.call_args_list, expected_args) + + @mock.patch('core.management.commands.ssl_images.check_https_availability') + def test_https_availability(self, check_https_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'http://example.com/static/wow.gif', + 'http://example.com/images/another_image.jpg', + ] + new_src = [ + 'https://example.com/images/my_image.jpg', + 'https://example.com/static/wow.gif', + 'https://example.com/images/another_image.jpg', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another one:  + And finally +  + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + check_https_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + expected_args = [mock.call(urlparse(c)) for c in old_src] + self.assertEqual(check_https_mock.call_args_list, expected_args) + + +class HtmlCheckTestCase(unittest.TestCase): + + def test_empty(self): + self.assertFalse(html_check('')) + + def test_no_images(self): + self.assertFalse(html_check('<p>Hi there!</p>')) + self.assertFalse(html_check('<p>Hi <b>there</b>!</p>')) + + def test_safe_image(self): + self.assertFalse(html_check('<img src="https://a.jpg" />')) + self.assertFalse(html_check('<img src="" alt="stuff" />')) + self.assertFalse(html_check('<img src="HTTPS://a.jpg" />')) + self.assertFalse(html_check(""" + <div> + <p>Look: <img src="https://a.jpg" alt="a" /></p> + <p>Look again: <img src="https://b.jpg" alt="b" /></p> + </div> + """)) + + def test_one_image(self): + self.assertTrue(html_check('<img src="http://a.jpg" alt="a" />')) + self.assertTrue(html_check( + '<p>Look: <img src="http://a.jpg" alt="a" /></p>')) + + def test_two_images(self): + self.assertTrue(html_check(""" + <p>Look: <img src="https://a.jpg" alt="a" /></p> + <p>Look again: <img src="http://b.jpg" alt="b" /></p> + """)) + self.assertTrue(html_check(""" + <p>Look: <img src="http://a.jpg" alt="a" /></p> + <p>Look again: <img src="http://b.jpg" alt="b" /></p> + """)) + self.assertTrue(html_check(""" + <div> + <p>Look: <img src="http://a.jpg" alt="a" /></p> + <p>Look again: <img src="http://b.jpg" alt="b" /></p> + </div> + """)) + self.assertTrue(html_check(""" + <div> + <p>Look: <img src="http://a.jpg" alt="a" /></p> + <p>Look again: <img src="https://b.jpg" alt="b" /></p> + </div> + """))