Mercurial > public > sg101

annotate oembed/management/commands/oembed_refresh.py @ 911:d3f6e9cb1f39

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
First take at an oembed refresh command.
author Brian Neal <bgneal@gmail.com>
date Tue, 17 Mar 2015 21:24:45 -0500
parents
children 602639621257
rev   line source
bgneal@911 1 """oembed_refresh -
bgneal@911 2 A custom management command to refresh the oEmbed HTML for the oEmbed objects in
bgneal@911 3 our database.
bgneal@911 4 This command is currently designed to fix up SSL mixed content issues. In the
bgneal@911 5 future it would be nice to rework this command to refresh oEmbed objects if they
bgneal@911 6 are older than some date.
bgneal@911 7 """
bgneal@911 8 import datetime
bgneal@911 9 import logging
bgneal@911 10 import os.path
bgneal@911 11 import re
bgneal@911 12 import urllib2
bgneal@911 13
bgneal@911 14 from django.core.management.base import NoArgsCommand
bgneal@911 15 from django.conf import settings
bgneal@911 16
bgneal@911 17 from oembed.core import get_oembed
bgneal@911 18 from oembed.models import Oembed
bgneal@911 19 from oembed.models import Provider
bgneal@911 20
bgneal@911 21
bgneal@911 22 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'oembed_refresh.log')
bgneal@911 23 logger = logging.getLogger(__name__)
bgneal@911 24
bgneal@911 25
bgneal@911 26 def _setup_logging():
bgneal@911 27 logger.setLevel(logging.DEBUG)
bgneal@911 28 logger.propagate = False
bgneal@911 29 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@911 30 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@911 31 handler.setFormatter(formatter)
bgneal@911 32 logger.addHandler(handler)
bgneal@911 33
bgneal@911 34
bgneal@911 35 def error_html(oembed):
bgneal@911 36 """Returns a string of HTML to be used when we can't retrieve the oEmbed
bgneal@911 37 data.
bgneal@911 38 """
bgneal@911 39 return u'<p><a href="{url}">{title}</a></p>'.format(url=oembed.url, title=oembed.title)
bgneal@911 40
bgneal@911 41
bgneal@911 42 def refresh_html(providers, oembed):
bgneal@911 43 """Refresh the oEmbed HTML for the given oembed object."""
bgneal@911 44
bgneal@911 45 # Find provider
bgneal@911 46 for p in providers:
bgneal@911 47 if re.match(p.url_regex, oembed.url):
bgneal@911 48 endpoint = p.api_endpoint
bgneal@911 49 break
bgneal@911 50 else:
bgneal@911 51 logger.error("No provider found for %s", oembed)
bgneal@911 52 return
bgneal@911 53
bgneal@911 54 html = None
bgneal@911 55 try:
bgneal@911 56 result = get_oembed(endpoint, oembed.url, fmt='json',
bgneal@911 57 maxwidth=settings.OEMBED_MAXWIDTH,
bgneal@911 58 maxheight=settings.OEMBED_MAXHEIGHT,
bgneal@911 59 scheme='https')
bgneal@911 60 except urllib2.HTTPError as ex:
bgneal@911 61 if 400 <= ex.code < 500:
bgneal@911 62 logger.error("Server could not handle request for %s: %d", oembed, ex.code)
bgneal@911 63 html = error_html(oembed)
bgneal@911 64 else:
bgneal@911 65 logger.critical("Server error during request for %s: %d", oembed, ex.code)
bgneal@911 66
bgneal@911 67 except urllib2.URLError as ex:
bgneal@911 68 logger.critical("Failed to reach provider for %s: %s", oembed, ex.reason)
bgneal@911 69 else:
bgneal@911 70 html = result['html']
bgneal@911 71
bgneal@911 72 if html:
bgneal@911 73 logger.info("Updating %s", oembed)
bgneal@911 74 oembed.html = html
bgneal@911 75 oembed.save()
bgneal@911 76
bgneal@911 77
bgneal@911 78 class Command(NoArgsCommand):
bgneal@911 79 help = "Refresh oEmbed objects by requesting new data from providers"""
bgneal@911 80
bgneal@911 81 def handle_noargs(self, **options):
bgneal@911 82 time_started = datetime.datetime.now()
bgneal@911 83 _setup_logging()
bgneal@911 84 logger.info("Starting; arguments received: %s", options)
bgneal@911 85
bgneal@911 86 providers = list(Provider.objects.all())
bgneal@911 87
bgneal@911 88 qs = Oembed.objects.all()
bgneal@911 89 for oembed in qs.iterator():
bgneal@911 90 if 'https:' not in oembed.html:
bgneal@911 91 refresh_html(providers, oembed)
bgneal@911 92
bgneal@911 93 time_finished = datetime.datetime.now()
bgneal@911 94 elapsed = time_finished - time_started
bgneal@911 95 logger.info("Finished; elapsed time: %s", elapsed)