|
bgneal@972
|
1 """This module contains routines for downloading files."""
|
|
bgneal@972
|
2
|
|
bgneal@972
|
3 import logging
|
|
bgneal@972
|
4 import os
|
|
bgneal@972
|
5 import shutil
|
|
bgneal@972
|
6 import tempfile
|
|
bgneal@972
|
7
|
|
bgneal@972
|
8 import requests
|
|
bgneal@972
|
9
|
|
bgneal@972
|
10
|
|
bgneal@972
|
11 logger = logging.getLogger(__name__)
|
|
bgneal@972
|
12
|
|
bgneal@972
|
13
|
|
bgneal@972
|
14 def download_file(url, path=None):
|
|
bgneal@972
|
15 """Downloads the image file from the given source URL and stores it in the
|
|
bgneal@972
|
16 filename given by path. If path is None, a temporary file will be created.
|
|
bgneal@972
|
17
|
|
bgneal@972
|
18 If successful returns the path to the downloaded file. Otherwise None is
|
|
bgneal@972
|
19 returned.
|
|
bgneal@972
|
20
|
|
bgneal@972
|
21 This function may raise various exceptions from the requests library.
|
|
bgneal@972
|
22 """
|
|
bgneal@972
|
23 logger.info("download_file from %s; path=%s", url, path)
|
|
bgneal@972
|
24
|
|
bgneal@972
|
25 try:
|
|
bgneal@972
|
26 r = requests.get(url, stream=True)
|
|
bgneal@972
|
27 except requests.RequestException:
|
|
bgneal@972
|
28 logger.exception("download_file requests.get('%s') exception", url)
|
|
bgneal@972
|
29 raise
|
|
bgneal@972
|
30
|
|
bgneal@972
|
31 if r.status_code != 200:
|
|
bgneal@972
|
32 logger.error("download_file from %s: error code %d", url, r.status_code)
|
|
bgneal@972
|
33 return None
|
|
bgneal@972
|
34
|
|
bgneal@972
|
35 # Save file data
|
|
bgneal@972
|
36
|
|
bgneal@972
|
37 if not path:
|
|
bgneal@972
|
38 fd, path = tempfile.mkstemp()
|
|
bgneal@972
|
39 os.close(fd)
|
|
bgneal@972
|
40
|
|
bgneal@972
|
41 try:
|
|
bgneal@972
|
42 with open(path, 'wb') as fp:
|
|
bgneal@972
|
43 r.raw.decode_content = True
|
|
bgneal@972
|
44 shutil.copyfileobj(r.raw, fp)
|
|
bgneal@972
|
45 except requests.RequestException:
|
|
bgneal@972
|
46 logger.exception("download_file download exception")
|
|
bgneal@972
|
47 raise
|
|
bgneal@972
|
48
|
|
bgneal@972
|
49 file_size = os.stat(path).st_size
|
|
bgneal@972
|
50 logger.info("download_file retrieved %s bytes from %s; saved to %s", file_size, url, path)
|
|
bgneal@972
|
51 return path
|
|
bgneal@972
|
52
|
|
bgneal@972
|
53
|
|
bgneal@972
|
54 if __name__ == '__main__':
|
|
bgneal@972
|
55 import sys
|
|
bgneal@972
|
56 s = "%(asctime)s : %(levelname)s : %(message)s"
|
|
bgneal@972
|
57 logging.basicConfig(level=logging.DEBUG, format=s)
|
|
bgneal@972
|
58 logging.info("argument is %s", sys.argv[1])
|
|
bgneal@972
|
59 result = download_file(sys.argv[1])
|
|
bgneal@972
|
60 if result:
|
|
bgneal@972
|
61 print result
|
