Mercurial > public > sg101

comparison core/download.py @ 972:7138883966b3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Started unit tests for hotlinking images.
author Brian Neal <bgneal@gmail.com>
date Wed, 23 Sep 2015 21:26:09 -0500
parents
children 6f55c086db1e
comparison
equal deleted inserted replaced
971:4f265f61874b 972:7138883966b3
1 """This module contains routines for downloading files."""
2
3 import logging
4 import os
5 import shutil
6 import tempfile
7
8 import requests
9
10
11 logger = logging.getLogger(__name__)
12
13
14 def download_file(url, path=None):
15 """Downloads the image file from the given source URL and stores it in the
16 filename given by path. If path is None, a temporary file will be created.
17
18 If successful returns the path to the downloaded file. Otherwise None is
19 returned.
20
21 This function may raise various exceptions from the requests library.
22 """
23 logger.info("download_file from %s; path=%s", url, path)
24
25 try:
26 r = requests.get(url, stream=True)
27 except requests.RequestException:
28 logger.exception("download_file requests.get('%s') exception", url)
29 raise
30
31 if r.status_code != 200:
32 logger.error("download_file from %s: error code %d", url, r.status_code)
33 return None
34
35 # Save file data
36
37 if not path:
38 fd, path = tempfile.mkstemp()
39 os.close(fd)
40
41 try:
42 with open(path, 'wb') as fp:
43 r.raw.decode_content = True
44 shutil.copyfileobj(r.raw, fp)
45 except requests.RequestException:
46 logger.exception("download_file download exception")
47 raise
48
49 file_size = os.stat(path).st_size
50 logger.info("download_file retrieved %s bytes from %s; saved to %s", file_size, url, path)
51 return path
52
53
54 if __name__ == '__main__':
55 import sys
56 s = "%(asctime)s : %(levelname)s : %(message)s"
57 logging.basicConfig(level=logging.DEBUG, format=s)
58 logging.info("argument is %s", sys.argv[1])
59 result = download_file(sys.argv[1])
60 if result:
61 print result