Mercurial > public > sg101

annotate core/s3.py @ 1157:e4f2d6a4b401

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Rework S3 connection logic for latest versions of Python 2.7. Had to make these changes for Ubuntu 16.04. Seems backward compatible with production.
author Brian Neal <bgneal@gmail.com>
date Thu, 19 Jan 2017 18:35:53 -0600
parents bf5340705d0c
children
rev   line source
bgneal@697 1 """s3.py
bgneal@697 2
bgneal@700 3 This module provides Amazon S3 convenience wrappers.
bgneal@697 4
bgneal@697 5 """
bgneal@1157 6 from boto.s3 import connect_to_region
bgneal@1157 7 from boto.s3.connection import OrdinaryCallingFormat
bgneal@697 8 from boto.s3.key import Key
bgneal@697 9
bgneal@697 10
bgneal@697 11 class S3Bucket(object):
bgneal@697 12 """This class abstracts an Amazon S3 bucket.
bgneal@697 13
bgneal@697 14 """
bgneal@1157 15 def __init__(self, access_key, secret_key, base_url, bucket_name,
bgneal@1157 16 region_name='us-west-1'):
bgneal@1157 17 self.region_name = region_name
bgneal@1157 18 self.conn = self._get_connection(access_key, secret_key)
bgneal@697 19 self.bucket = self.conn.get_bucket(bucket_name, validate=False)
bgneal@700 20 self.base_url = base_url
bgneal@700 21 if not base_url.endswith('/'):
bgneal@700 22 self.base_url += '/'
bgneal@700 23 self.name = bucket_name
bgneal@700 24
bgneal@1157 25 def _get_connection(self, access_key, secret_key):
bgneal@1157 26 conn = connect_to_region(
bgneal@1157 27 self.region_name,
bgneal@1157 28 aws_access_key_id=access_key,
bgneal@1157 29 aws_secret_access_key=secret_key,
bgneal@1157 30 calling_format=OrdinaryCallingFormat())
bgneal@1157 31 return conn
bgneal@1157 32
bgneal@700 33 def upload_from_file(self, key_name, fp, metadata=None, public=True):
bgneal@700 34 """Uploads data from the file object fp to a new key named
bgneal@700 35 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@700 36 pairs which will be added to the key.
bgneal@700 37
bgneal@700 38 If public is True, the key will be made public after the upload.
bgneal@700 39
bgneal@700 40 Returns the URL to the uploaded file.
bgneal@700 41
bgneal@700 42 """
bgneal@700 43 key = self._make_key(key_name, metadata)
bgneal@700 44 key.set_contents_from_file(fp)
bgneal@700 45 if public:
bgneal@700 46 key.make_public()
bgneal@700 47 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 48
bgneal@697 49 def upload_from_filename(self, key_name, filename, metadata=None,
bgneal@697 50 public=True):
bgneal@697 51 """Uploads data from the file named by filename to a new key named
bgneal@697 52 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@697 53 pairs which will be added to the key.
bgneal@697 54
bgneal@697 55 If public is True, the key will be made public after the upload.
bgneal@697 56
bgneal@700 57 Returns the URL to the uploaded file.
bgneal@700 58
bgneal@697 59 """
bgneal@697 60 key = self._make_key(key_name, metadata)
bgneal@697 61 key.set_contents_from_filename(filename)
bgneal@697 62 if public:
bgneal@697 63 key.make_public()
bgneal@700 64 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 65
bgneal@697 66 def upload_from_string(self, key_name, content, metadata=None,
bgneal@697 67 public=True):
bgneal@697 68 """Creates a new key with the given key_name, and uploads the string
bgneal@697 69 content to it. metadata, if not None, must be a dict of metadata key /
bgneal@697 70 value pairs which will be added to the key.
bgneal@697 71
bgneal@697 72 If public is True, the key will be made public after the upload.
bgneal@697 73
bgneal@700 74 Returns the URL to the uploaded file.
bgneal@700 75
bgneal@697 76 """
bgneal@697 77 key = self._make_key(key_name, metadata)
bgneal@697 78 key.set_contents_from_string(content)
bgneal@697 79 if public:
bgneal@697 80 key.make_public()
bgneal@700 81 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 82
bgneal@718 83 def delete_keys(self, key_urls):
bgneal@718 84 """Deletes a set of keys, specified as a list of URLs. The URLs could
bgneal@718 85 have been returned by one or more of the upload_* methods.
bgneal@718 86
bgneal@718 87 Returns the number of keys that were successfully deleted.
bgneal@718 88
bgneal@718 89 """
bgneal@718 90 if len(key_urls) == 0:
bgneal@718 91 return 0
bgneal@718 92
bgneal@718 93 prefix = '{}{}/'.format(self.base_url, self.name)
bgneal@718 94 prefix_len = len(prefix)
bgneal@718 95
bgneal@718 96 keys = []
bgneal@718 97 for url in key_urls:
bgneal@718 98 if url.startswith(prefix):
bgneal@718 99 key = url[prefix_len:]
bgneal@718 100 keys.append(key)
bgneal@718 101
bgneal@718 102 response = self.bucket.delete_keys(keys, quiet=True)
bgneal@718 103 return len(key_urls) - len(response.errors)
bgneal@718 104
bgneal@697 105 def _make_key(self, key_name, metadata):
bgneal@697 106 """Private method to create a key and optionally apply metadata to
bgneal@697 107 it.
bgneal@697 108
bgneal@697 109 """
bgneal@697 110 key = Key(self.bucket)
bgneal@697 111 key.key = key_name
bgneal@697 112 if metadata:
bgneal@697 113 for k, v in metadata.iteritems():
bgneal@697 114 key.set_metadata(k, v)
bgneal@697 115 return key