Mercurial > public > sg101

changeset 980:3ebde23a59d0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Pass timeout to requests instead of global timeout. Global timeout didn't seem to work once we started using requests.
author Brian Neal <bgneal@gmail.com>
date Thu, 15 Oct 2015 18:35:56 -0500
parents a6331579ff43
children ef1558941bc9
files core/download.py core/management/commands/ssl_images.py
diffstat 2 files changed, 9 insertions(+), 9 deletions(-) [+]
line wrap: on
line diff
--- a/core/download.py	Fri Oct 09 22:20:32 2015 -0500
+++ b/core/download.py	Thu Oct 15 18:35:56 2015 -0500
@@ -12,7 +12,7 @@
 logger = logging.getLogger(__name__)
 
 
-def download_file(url, path=None):
+def download_file(url, path=None, timeout=None):
     """Downloads the image file from the given source URL and stores it in the
     filename given by path. If path is None, a temporary file will be created.
 
@@ -24,7 +24,7 @@
     logger.info("download_file from %s; path=%s", url, path)
 
     try:
-        r = requests.get(url, stream=True)
+        r = requests.get(url, stream=True, timeout=timeout)
     except requests.RequestException:
         logger.exception("download_file requests.get('%s') exception", url)
         raise
--- a/core/management/commands/ssl_images.py	Fri Oct 09 22:20:32 2015 -0500
+++ b/core/management/commands/ssl_images.py	Thu Oct 15 18:35:56 2015 -0500
@@ -14,7 +14,6 @@
 import os
 import re
 import signal
-import socket
 import urlparse
 import uuid
 
@@ -54,6 +53,7 @@
 bucket = None
 url_cache = {}
 bad_hosts = set()
+request_timeout = None
 
 
 def signal_handler(signum, frame):
@@ -145,7 +145,7 @@
     url = parsed_url.geturl()
     fn = None
     try:
-        fn = download_file(url)
+        fn = download_file(url, timeout=request_timeout)
     except requests.ConnectionError as ex:
         logger.error("ConnectionError, ignoring host %s", parsed_url.hostname)
         bad_hosts.add(parsed_url.hostname)
@@ -250,9 +250,9 @@
                 type='int',
                 help="optional second slice index; the j in [i:j]"),
             make_option('-t', '--timeout',
-                type='int',
+                type='float',
                 help="optional socket timeout (secs)",
-                default=30),
+                default=30.0),
             )
 
     def handle_noargs(self, **options):
@@ -289,9 +289,9 @@
             qs = qs[:j]
 
         # Set global socket timeout
-        timeout = options.get('timeout')
-        logger.info("Setting socket timeout to %d", timeout)
-        socket.setdefaulttimeout(timeout)
+        global request_timeout
+        request_timeout = options.get('timeout')
+        logger.info("Using socket timeout of %4.2f", request_timeout)
 
         # Install signal handler for ctrl-c
         signal.signal(signal.SIGINT, signal_handler)