Mercurial > public > sg101

comparison core/management/commands/ssl_images.py @ 897:49ebeb54990a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Record if an image could not be retrieved. Added some additional stats at the end.
author Brian Neal <bgneal@gmail.com>
date Sat, 28 Feb 2015 13:52:46 -0600
parents 0054a4a88c1c
children 8fcd278d8987
comparison
equal deleted inserted replaced
896:0054a4a88c1c 897:49ebeb54990a
160 """Top-level function for moving an image to SSL.""" 160 """Top-level function for moving an image to SSL."""
161 161
162 src = parsed_url.geturl() 162 src = parsed_url.geturl()
163 163
164 # Check the cache first 164 # Check the cache first
165 new_url = url_cache.get(src) 165 try:
166 if new_url: 166 new_url = url_cache[src]
167 logger.info("Found URL in cache: %s => %s", src, new_url) 167 except KeyError:
168 return new_url 168 # cache miss, try to get the file
169 169 new_url = save_image_to_cloud(src)
170 # Try to download and upload to our S3 bucket
171 new_url = save_image_to_cloud(src)
172 if new_url:
173 url_cache[src] = new_url 170 url_cache[src] = new_url
171 else:
172 if new_url:
173 logger.info("Found URL in cache: %s => %s", src, new_url)
174 else:
175 logger.info("URL known to be bad, skipping: %s", src)
176
174 return new_url 177 return new_url
175 178
176 179
177 def save_image_to_cloud(src): 180 def save_image_to_cloud(src):
178 """Downloads an image at a given source URL. Uploads it to cloud storage. 181 """Downloads an image at a given source URL. Uploads it to cloud storage.
352 355
353 time_finished = datetime.datetime.now() 356 time_finished = datetime.datetime.now()
354 elapsed = time_finished - time_started 357 elapsed = time_finished - time_started
355 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s", 358 logger.info("ssl_images exiting; number of objects: %d; elapsed: %s",
356 count, elapsed) 359 count, elapsed)
360
361 http_images = len(url_cache)
362 https_images = sum(1 for v in url_cache.itervalues() if v)
363 bad_images = http_images - https_images
364 if http_images > 0:
365 pct_saved = float(https_images) / http_images * 100.0
366 else:
367 pct_saved = 0.0
368
369 logger.info("Summary: http: %d; https: %d; lost: %d; saved: %3.1f %%",
370 http_images, https_images, bad_images, pct_saved)