Mercurial > public > sg101
changeset 875:d5d8e90d08b5
Merge with upstream.
| author | Brian Neal <bgneal@gmail.com> |
|---|---|
| date | Thu, 25 Dec 2014 17:28:48 -0600 |
| parents | 9676833dfdca (diff) b59c154d0163 (current diff) |
| children | bab6b1eac1e2 |
| files | |
| diffstat | 3 files changed, 360 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Thu Dec 25 17:20:28 2014 -0600 +++ b/.hgignore Thu Dec 25 17:28:48 2014 -0600 @@ -7,6 +7,7 @@ secrets.json *.db *.mp3 +.tags static_serve media/avatars/users media/badges
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/management/commands/ssl_images.py Thu Dec 25 17:28:48 2014 -0600 @@ -0,0 +1,165 @@ +""" +ssl_images is a custom manage.py command to convert forum post and comment +images to https. It does this by rewriting the markup: + - Images with src = http://surfguitar101.com/something are rewritten to be + /something. + - Non SG101 images that use http: are downloaded, resized, and uploaded to + an S3 bucket. The src attribute is replaced with the new S3 URL. +""" +import logging +from optparse import make_option +import os.path +import re +import signal +import urlparse + +from django.core.management.base import NoArgsCommand, CommandError +from django.conf import settings +import markdown.inlinepatterns + +from comments.models import Comment +from forums.models import Post + + +LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log') +logger = logging.getLogger(__name__) + +IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE, + re.DOTALL | re.UNICODE) +IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE, + re.DOTALL | re.UNICODE) + +SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com']) +MODEL_CHOICES = ['comments', 'posts'] + +quit_flag = False + + +def signal_handler(signum, frame): + """SIGINT signal handler""" + global quit_flag + quit_flag = True + + +def _setup_logging(): + logger.setLevel(logging.DEBUG) + logger.propagate = False + handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8') + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + +def save_image_to_cloud(src): + # TODO + return src + + +def replace_image_markup(match): + src_parts = match.group(8).split() + if src_parts: + src = src_parts[0] + if src[0] == "<" and src[-1] == ">": + src = src[1:-1] + else: + src = '' + + title = '' + if len(src_parts) > 1: + title = " ".join(src_parts[1:]) + alt = match.group(1) + + new_src = None + if src: + r = urlparse.urlparse(src) + if r.hostname in SG101_HOSTS: + new_src = r.path # convert to relative path + elif r.scheme == 'http': + new_src = save_image_to_cloud(src) + elif r.scheme == 'https': + new_src = src # already https, accept it as-is + + if new_src: + if title: + s = u''.format(alt=alt, src=new_src, title=title) + else: + s = u''.format(alt=alt, src=new_src) + else: + # something's messed up, convert to a link using original src + s = u'[{alt}]({src})'.format(alt=alt, src=src) + + return s + + +def process_post(text): + """Process the post object: + + A regex substitution is run on the post's text field. This fixes up image + links, getting rid of plain old http sources; either converting to https + or relative style links (if the link is to SG101). + + We also do a search for Markdown image reference markup. We aren't expecting + these, but we will log something if we see any. + + """ + return IMAGE_LINK_RE.sub(replace_image_markup, text) + + +class Command(NoArgsCommand): + help = "Rewrite forum posts and comments to not use http for images" + option_list = NoArgsCommand.option_list + ( + make_option('-m', '--model', + choices=MODEL_CHOICES, + help="which model to update; must be one of {{{}}}".format( + ', '.join(MODEL_CHOICES))), + make_option('-i', '--i', + type='int', + help="optional first slice index; the i in [i:j]"), + make_option('-j', '--j', + type='int', + help="optional second slice index; the j in [i:j]"), + ) + + def handle_noargs(self, **options): + _setup_logging() + logger.info("Starting; arguments received: %s", options) + + if options['model'] not in MODEL_CHOICES: + raise CommandError('Please choose a --model option') + + if options['model'] == 'comments': + qs = Comment.objects.all() + text_attr = 'comment' + else: + qs = Post.objects.all() + text_attr = 'body' + + i, j = options['i'], options['j'] + + if i is not None and i < 0: + raise CommandError("-i must be >= 0") + if j is not None and j < 0: + raise CommandError("-j must be >= 0") + if j is not None and i is not None and j <= i: + raise CommandError("-j must be > -i") + + if i is not None and j is not None: + qs = qs[i:j] + elif i is not None and j is None: + qs = qs[i:] + elif i is None and j is not None: + qs = qs[:j] + + # Install signal handler for ctrl-c + signal.signal(signal.SIGINT, signal_handler) + + s = [] + for model in qs.iterator(): + if quit_flag: + logger.warning("SIGINT received, exiting") + txt = getattr(model, text_attr) + new_txt = process_post(txt) + s.append(new_txt) + + import pprint + pprint.pprint(s)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/tests/test_ssl_images.py Thu Dec 25 17:28:48 2014 -0600 @@ -0,0 +1,194 @@ +"""Unit tests for the ssl_images management command.""" +import re +import unittest + +import mock + +from core.management.commands.ssl_images import process_post + + +class ProcessPostTestCase(unittest.TestCase): + + SG101_RE = re.compile(r'http://(?:www\.)?surfguitar101.com/', re.I) + + def test_empty_string(self): + s = process_post('') + self.assertEqual(s, '') + + def test_no_matches(self): + test_str = """Here is a post that doesn't contain any image links at + all. It also spans lines. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_sg101_images(self): + test_str = """An image:  + And another: . + More stuff here.""" + expected = self.SG101_RE.sub('/', test_str) + result = process_post(test_str) + self.assertNotEqual(test_str, expected) + self.assertEqual(expected, result) + + def test_sg101_with_newlines(self): + test_str = """An image:  + with trailing text.""" + expected = """An image:  + with trailing text.""" + result = process_post(test_str) + self.assertNotEqual(test_str, expected) + self.assertEqual(expected, result) + + def test_https_already(self): + test_str = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_https_sg101(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_multiple_non_http(self): + test_str = """An image:  + And another: . + More stuff here.""" + expected = """An image:  + And another: . + More stuff here.""" + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_already_with_title(self): + test_str = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(test_str, result) + + def test_sg101_with_title(self): + test_str = """An image on SG101: +  + It's cool. + """ + expected = """An image on SG101: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_sg101_brackets(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + def test_https_already_brackets(self): + test_str = """An image that is already using https: +  + It's cool. + """ + expected = """An image that is already using https: +  + It's cool. + """ + result = process_post(test_str) + self.assertEqual(expected, result) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + def test_simple_replacement(self, upload_mock): + old_src = 'http://example.com/images/my_image.jpg' + new_src = 'https://cloud.com/ABCDEF.jpg' + test_str = """Here is a really cool http: based image: +  + Cool, right?""".format(old_src) + expected = """Here is a really cool http: based image: +  + Cool, right?""".format(new_src) + + upload_mock.return_value = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + upload_mock.assert_called_once_with(old_src) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + def test_multiple_replacement(self, upload_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'http://example.com/static/wow.gif', + 'http://example.com/media/a/b/c/pic.png', + ] + new_src = [ + 'https://cloud.com/some/path/012345.jpg', + 'https://cloud.com/some/path/6789AB.gif', + 'https://cloud.com/some/path/CDEF01.png', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another one:  + And finally +  + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + upload_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + expected_args = [mock.call(c) for c in old_src] + self.assertEqual(upload_mock.call_args_list, expected_args) + + @mock.patch('core.management.commands.ssl_images.save_image_to_cloud') + def test_multiple_replacement_2(self, upload_mock): + old_src = [ + 'http://example.com/images/my_image.jpg', + 'https://example.com/static/wow.gif', + 'http://www.surfguitar101.com/media/a/b/c/pic.png', + 'http://surfguitar101.com/media/a/b/c/pic2.png', + ] + new_src = [ + 'https://cloud.com/some/path/012345.jpg', + 'https://example.com/static/wow.gif', + '/media/a/b/c/pic.png', + '/media/a/b/c/pic2.png', + ] + + template = """Here is a really cool http: based image: +  + Cool, right? + Another two:   + And finally + . + """ + + test_str = template.format(*old_src) + expected = template.format(*new_src) + + upload_mock.side_effect = new_src + result = process_post(test_str) + self.assertEqual(expected, result) + upload_mock.assert_called_once_with(old_src[0])