Mercurial > public > sg101

comparison core/management/commands/ssl_images.py @ 980:3ebde23a59d0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Pass timeout to requests instead of global timeout. Global timeout didn't seem to work once we started using requests.
author Brian Neal <bgneal@gmail.com>
date Thu, 15 Oct 2015 18:35:56 -0500
parents a6331579ff43
children ef1558941bc9
comparison
equal deleted inserted replaced
979:a6331579ff43 980:3ebde23a59d0
12 import logging 12 import logging
13 from optparse import make_option 13 from optparse import make_option
14 import os 14 import os
15 import re 15 import re
16 import signal 16 import signal
17 import socket
18 import urlparse 17 import urlparse
19 import uuid 18 import uuid
20 19
21 from django.core.management.base import NoArgsCommand, CommandError 20 from django.core.management.base import NoArgsCommand, CommandError
22 from django.conf import settings 21 from django.conf import settings
52 51
53 quit_flag = False 52 quit_flag = False
54 bucket = None 53 bucket = None
55 url_cache = {} 54 url_cache = {}
56 bad_hosts = set() 55 bad_hosts = set()
56 request_timeout = None
57 57
58 58
59 def signal_handler(signum, frame): 59 def signal_handler(signum, frame):
60 """SIGINT signal handler""" 60 """SIGINT signal handler"""
61 global quit_flag 61 global quit_flag
143 Returns the new URL or None if unsuccessful. 143 Returns the new URL or None if unsuccessful.
144 """ 144 """
145 url = parsed_url.geturl() 145 url = parsed_url.geturl()
146 fn = None 146 fn = None
147 try: 147 try:
148 fn = download_file(url) 148 fn = download_file(url, timeout=request_timeout)
149 except requests.ConnectionError as ex: 149 except requests.ConnectionError as ex:
150 logger.error("ConnectionError, ignoring host %s", parsed_url.hostname) 150 logger.error("ConnectionError, ignoring host %s", parsed_url.hostname)
151 bad_hosts.add(parsed_url.hostname) 151 bad_hosts.add(parsed_url.hostname)
152 except requests.RequestException as ex: 152 except requests.RequestException as ex:
153 logger.error("%s", ex) 153 logger.error("%s", ex)
248 help="optional first slice index; the i in [i:j]"), 248 help="optional first slice index; the i in [i:j]"),
249 make_option('-j', '--j', 249 make_option('-j', '--j',
250 type='int', 250 type='int',
251 help="optional second slice index; the j in [i:j]"), 251 help="optional second slice index; the j in [i:j]"),
252 make_option('-t', '--timeout', 252 make_option('-t', '--timeout',
253 type='int', 253 type='float',
254 help="optional socket timeout (secs)", 254 help="optional socket timeout (secs)",
255 default=30), 255 default=30.0),
256 ) 256 )
257 257
258 def handle_noargs(self, **options): 258 def handle_noargs(self, **options):
259 time_started = datetime.datetime.now() 259 time_started = datetime.datetime.now()
260 _setup_logging() 260 _setup_logging()
287 qs = qs[i:] 287 qs = qs[i:]
288 elif i is None and j is not None: 288 elif i is None and j is not None:
289 qs = qs[:j] 289 qs = qs[:j]
290 290
291 # Set global socket timeout 291 # Set global socket timeout
292 timeout = options.get('timeout') 292 global request_timeout
293 logger.info("Setting socket timeout to %d", timeout) 293 request_timeout = options.get('timeout')
294 socket.setdefaulttimeout(timeout) 294 logger.info("Using socket timeout of %4.2f", request_timeout)
295 295
296 # Install signal handler for ctrl-c 296 # Install signal handler for ctrl-c
297 signal.signal(signal.SIGINT, signal_handler) 297 signal.signal(signal.SIGINT, signal_handler)
298 298
299 # Create bucket to upload photos 299 # Create bucket to upload photos