Mercurial > public > sg101

annotate core/management/commands/ssl_images.py @ 871:6900040df0f8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
More WIP on ssl_images command & unit test.
author Brian Neal <bgneal@gmail.com>
date Tue, 16 Dec 2014 20:59:49 -0600
parents ee56c8c8cf98
children 13f2d4393ec4
rev   line source
bgneal@859 1 """
bgneal@859 2 ssl_images is a custom manage.py command to convert forum post and comment
bgneal@859 3 images to https. It does this by rewriting the markup:
bgneal@859 4 - Images with src = http://surfguitar101.com/something are rewritten to be
bgneal@859 5 /something.
bgneal@859 6 - Non SG101 images that use http: are downloaded, resized, and uploaded to
bgneal@859 7 an S3 bucket. The src attribute is replaced with the new S3 URL.
bgneal@859 8 """
bgneal@859 9 import logging
bgneal@859 10 from optparse import make_option
bgneal@859 11 import os.path
bgneal@863 12 import re
bgneal@863 13 import signal
bgneal@868 14 import urlparse
bgneal@859 15
bgneal@859 16 from django.core.management.base import NoArgsCommand, CommandError
bgneal@859 17 from django.conf import settings
bgneal@863 18 import markdown.inlinepatterns
bgneal@859 19
bgneal@860 20 from comments.models import Comment
bgneal@860 21 from forums.models import Post
bgneal@860 22
bgneal@860 23
bgneal@859 24 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'ssl_images.log')
bgneal@859 25 logger = logging.getLogger(__name__)
bgneal@859 26
bgneal@871 27 IMAGE_LINK_RE = re.compile(markdown.inlinepatterns.IMAGE_LINK_RE,
bgneal@871 28 re.DOTALL | re.UNICODE)
bgneal@871 29 IMAGE_REF_RE = re.compile(markdown.inlinepatterns.IMAGE_REFERENCE_RE,
bgneal@871 30 re.DOTALL | re.UNICODE)
bgneal@863 31
bgneal@868 32 SG101_HOSTS = set(['www.surfguitar101.com', 'surfguitar101.com'])
bgneal@866 33 MODEL_CHOICES = ['comments', 'posts']
bgneal@866 34
bgneal@863 35 quit_flag = False
bgneal@863 36
bgneal@863 37
bgneal@863 38 def signal_handler(signum, frame):
bgneal@863 39 """SIGINT signal handler"""
bgneal@863 40 global quit_flag
bgneal@863 41 quit_flag = True
bgneal@863 42
bgneal@859 43
bgneal@859 44 def _setup_logging():
bgneal@859 45 logger.setLevel(logging.DEBUG)
bgneal@859 46 logger.propagate = False
bgneal@859 47 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@859 48 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@859 49 handler.setFormatter(formatter)
bgneal@859 50 logger.addHandler(handler)
bgneal@859 51
bgneal@859 52
bgneal@868 53 def save_image_to_cloud(src):
bgneal@868 54 # TODO
bgneal@868 55 return src
bgneal@868 56
bgneal@868 57
bgneal@866 58 def replace_image_markup(match):
bgneal@870 59 src_parts = match.group(8).split()
bgneal@868 60 if src_parts:
bgneal@868 61 src = src_parts[0]
bgneal@868 62 if src[0] == "<" and src[-1] == ">":
bgneal@868 63 src = src[1:-1]
bgneal@868 64 else:
bgneal@868 65 src = ''
bgneal@868 66
bgneal@868 67 title = ''
bgneal@868 68 if len(src_parts) > 1:
bgneal@868 69 title = " ".join(src_parts[1:])
bgneal@870 70 alt = match.group(1)
bgneal@868 71
bgneal@871 72 new_src = None
bgneal@868 73 if src:
bgneal@868 74 r = urlparse.urlparse(src)
bgneal@871 75 if r.hostname in SG101_HOSTS:
bgneal@871 76 new_src = r.path # convert to relative path
bgneal@871 77 elif r.scheme == 'http':
bgneal@871 78 new_src = save_image_to_cloud(src)
bgneal@868 79 elif r.scheme == 'https':
bgneal@868 80 new_src = src # already https, accept it as-is
bgneal@868 81
bgneal@868 82 if new_src:
bgneal@868 83 if title:
bgneal@871 84 s = u'![{alt}]({src} {title})'.format(alt=alt, src=new_src, title=title)
bgneal@868 85 else:
bgneal@868 86 s = u'![{alt}]({src})'.format(alt=alt, src=new_src)
bgneal@868 87 else:
bgneal@868 88 # something's messed up, convert to a link using original src
bgneal@868 89 s = u'[{alt}]({src})'.format(alt=alt, src=src)
bgneal@868 90
bgneal@868 91 return s
bgneal@860 92
bgneal@860 93
bgneal@866 94 def process_post(text):
bgneal@863 95 """Process the post object:
bgneal@863 96
bgneal@863 97 A regex substitution is run on the post's text field. This fixes up image
bgneal@863 98 links, getting rid of plain old http sources; either converting to https
bgneal@863 99 or relative style links (if the link is to SG101).
bgneal@863 100
bgneal@863 101 We also do a search for Markdown image reference markup. We aren't expecting
bgneal@863 102 these, but we will log something if we see any.
bgneal@863 103
bgneal@863 104 """
bgneal@866 105 return IMAGE_LINK_RE.sub(replace_image_markup, text)
bgneal@863 106
bgneal@863 107
bgneal@859 108 class Command(NoArgsCommand):
bgneal@859 109 help = "Rewrite forum posts and comments to not use http for images"
bgneal@859 110 option_list = NoArgsCommand.option_list + (
bgneal@866 111 make_option('-m', '--model',
bgneal@866 112 choices=MODEL_CHOICES,
bgneal@866 113 help="which model to update; must be one of {{{}}}".format(
bgneal@866 114 ', '.join(MODEL_CHOICES))),
bgneal@860 115 make_option('-i', '--i',
bgneal@859 116 type='int',
bgneal@863 117 help="optional first slice index; the i in [i:j]"),
bgneal@860 118 make_option('-j', '--j',
bgneal@859 119 type='int',
bgneal@863 120 help="optional second slice index; the j in [i:j]"),
bgneal@859 121 )
bgneal@859 122
bgneal@859 123 def handle_noargs(self, **options):
bgneal@859 124 _setup_logging()
bgneal@860 125 logger.info("Starting; arguments received: %s", options)
bgneal@859 126
bgneal@866 127 if options['model'] not in MODEL_CHOICES:
bgneal@866 128 raise CommandError('Please choose a --model option')
bgneal@859 129
bgneal@866 130 if options['model'] == 'comments':
bgneal@860 131 qs = Comment.objects.all()
bgneal@866 132 text_attr = 'comment'
bgneal@860 133 else:
bgneal@860 134 qs = Post.objects.all()
bgneal@866 135 text_attr = 'body'
bgneal@860 136
bgneal@860 137 i, j = options['i'], options['j']
bgneal@860 138
bgneal@860 139 if i is not None and i < 0:
bgneal@860 140 raise CommandError("-i must be >= 0")
bgneal@860 141 if j is not None and j < 0:
bgneal@860 142 raise CommandError("-j must be >= 0")
bgneal@860 143 if j is not None and i is not None and j <= i:
bgneal@860 144 raise CommandError("-j must be > -i")
bgneal@860 145
bgneal@860 146 if i is not None and j is not None:
bgneal@860 147 qs = qs[i:j]
bgneal@860 148 elif i is not None and j is None:
bgneal@860 149 qs = qs[i:]
bgneal@860 150 elif i is None and j is not None:
bgneal@860 151 qs = qs[:j]
bgneal@860 152
bgneal@863 153 # Install signal handler for ctrl-c
bgneal@863 154 signal.signal(signal.SIGINT, signal_handler)
bgneal@863 155
bgneal@860 156 s = []
bgneal@860 157 for model in qs.iterator():
bgneal@863 158 if quit_flag:
bgneal@863 159 logger.warning("SIGINT received, exiting")
bgneal@866 160 txt = getattr(model, text_attr)
bgneal@866 161 new_txt = process_post(txt)
bgneal@866 162 s.append(new_txt)
bgneal@860 163
bgneal@860 164 import pprint
bgneal@860 165 pprint.pprint(s)