Mercurial > public > sg101

annotate oembed/management/commands/oembed_refresh.py @ 912:602639621257

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Tweak HTML for videos that are no longer found.
author Brian Neal <bgneal@gmail.com>
date Sat, 21 Mar 2015 09:56:40 -0500
parents d3f6e9cb1f39
children f2fbe2b0d25d
rev   line source
bgneal@911 1 """oembed_refresh -
bgneal@911 2 A custom management command to refresh the oEmbed HTML for the oEmbed objects in
bgneal@911 3 our database.
bgneal@911 4 This command is currently designed to fix up SSL mixed content issues. In the
bgneal@911 5 future it would be nice to rework this command to refresh oEmbed objects if they
bgneal@911 6 are older than some date.
bgneal@911 7 """
bgneal@911 8 import datetime
bgneal@911 9 import logging
bgneal@911 10 import os.path
bgneal@911 11 import re
bgneal@911 12 import urllib2
bgneal@911 13
bgneal@911 14 from django.core.management.base import NoArgsCommand
bgneal@911 15 from django.conf import settings
bgneal@911 16
bgneal@911 17 from oembed.core import get_oembed
bgneal@911 18 from oembed.models import Oembed
bgneal@911 19 from oembed.models import Provider
bgneal@911 20
bgneal@911 21
bgneal@911 22 LOGFILE = os.path.join(settings.PROJECT_PATH, 'logs', 'oembed_refresh.log')
bgneal@911 23 logger = logging.getLogger(__name__)
bgneal@911 24
bgneal@911 25
bgneal@911 26 def _setup_logging():
bgneal@911 27 logger.setLevel(logging.DEBUG)
bgneal@911 28 logger.propagate = False
bgneal@911 29 handler = logging.FileHandler(filename=LOGFILE, encoding='utf-8')
bgneal@911 30 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
bgneal@911 31 handler.setFormatter(formatter)
bgneal@911 32 logger.addHandler(handler)
bgneal@911 33
bgneal@911 34
bgneal@911 35 def error_html(oembed):
bgneal@911 36 """Returns a string of HTML to be used when we can't retrieve the oEmbed
bgneal@911 37 data.
bgneal@911 38 """
bgneal@912 39 return u'<p>Video: <a href="{url}">{title}</a></p>'.format(
bgneal@912 40 url=oembed.url, title=oembed.title)
bgneal@911 41
bgneal@911 42
bgneal@911 43 def refresh_html(providers, oembed):
bgneal@911 44 """Refresh the oEmbed HTML for the given oembed object."""
bgneal@911 45
bgneal@911 46 # Find provider
bgneal@911 47 for p in providers:
bgneal@911 48 if re.match(p.url_regex, oembed.url):
bgneal@911 49 endpoint = p.api_endpoint
bgneal@911 50 break
bgneal@911 51 else:
bgneal@911 52 logger.error("No provider found for %s", oembed)
bgneal@911 53 return
bgneal@911 54
bgneal@911 55 html = None
bgneal@911 56 try:
bgneal@911 57 result = get_oembed(endpoint, oembed.url, fmt='json',
bgneal@911 58 maxwidth=settings.OEMBED_MAXWIDTH,
bgneal@911 59 maxheight=settings.OEMBED_MAXHEIGHT,
bgneal@911 60 scheme='https')
bgneal@911 61 except urllib2.HTTPError as ex:
bgneal@911 62 if 400 <= ex.code < 500:
bgneal@911 63 logger.error("Server could not handle request for %s: %d", oembed, ex.code)
bgneal@911 64 html = error_html(oembed)
bgneal@911 65 else:
bgneal@911 66 logger.critical("Server error during request for %s: %d", oembed, ex.code)
bgneal@911 67
bgneal@911 68 except urllib2.URLError as ex:
bgneal@911 69 logger.critical("Failed to reach provider for %s: %s", oembed, ex.reason)
bgneal@911 70 else:
bgneal@911 71 html = result['html']
bgneal@911 72
bgneal@911 73 if html:
bgneal@911 74 logger.info("Updating %s", oembed)
bgneal@911 75 oembed.html = html
bgneal@911 76 oembed.save()
bgneal@911 77
bgneal@911 78
bgneal@911 79 class Command(NoArgsCommand):
bgneal@911 80 help = "Refresh oEmbed objects by requesting new data from providers"""
bgneal@911 81
bgneal@911 82 def handle_noargs(self, **options):
bgneal@911 83 time_started = datetime.datetime.now()
bgneal@911 84 _setup_logging()
bgneal@911 85 logger.info("Starting; arguments received: %s", options)
bgneal@911 86
bgneal@911 87 providers = list(Provider.objects.all())
bgneal@911 88
bgneal@911 89 qs = Oembed.objects.all()
bgneal@911 90 for oembed in qs.iterator():
bgneal@911 91 if 'https:' not in oembed.html:
bgneal@911 92 refresh_html(providers, oembed)
bgneal@911 93
bgneal@911 94 time_finished = datetime.datetime.now()
bgneal@911 95 elapsed = time_finished - time_started
bgneal@911 96 logger.info("Finished; elapsed time: %s", elapsed)