Mercurial > public > sg101

annotate core/s3.py @ 1053:03237aaad839

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge with upstream.
author Brian Neal <bgneal@gmail.com>
date Thu, 03 Mar 2016 20:22:41 -0600
parents bf5340705d0c
children e4f2d6a4b401
rev   line source
bgneal@697 1 """s3.py
bgneal@697 2
bgneal@700 3 This module provides Amazon S3 convenience wrappers.
bgneal@697 4
bgneal@697 5 """
bgneal@697 6 from boto.s3.connection import S3Connection
bgneal@697 7 from boto.s3.key import Key
bgneal@697 8
bgneal@697 9
bgneal@697 10 class S3Bucket(object):
bgneal@697 11 """This class abstracts an Amazon S3 bucket.
bgneal@697 12
bgneal@697 13 """
bgneal@700 14 def __init__(self, access_key, secret_key, base_url, bucket_name):
bgneal@697 15 self.conn = S3Connection(access_key, secret_key)
bgneal@697 16 self.bucket = self.conn.get_bucket(bucket_name, validate=False)
bgneal@700 17 self.base_url = base_url
bgneal@700 18 if not base_url.endswith('/'):
bgneal@700 19 self.base_url += '/'
bgneal@700 20 self.name = bucket_name
bgneal@700 21
bgneal@700 22 def upload_from_file(self, key_name, fp, metadata=None, public=True):
bgneal@700 23 """Uploads data from the file object fp to a new key named
bgneal@700 24 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@700 25 pairs which will be added to the key.
bgneal@700 26
bgneal@700 27 If public is True, the key will be made public after the upload.
bgneal@700 28
bgneal@700 29 Returns the URL to the uploaded file.
bgneal@700 30
bgneal@700 31 """
bgneal@700 32 key = self._make_key(key_name, metadata)
bgneal@700 33 key.set_contents_from_file(fp)
bgneal@700 34 if public:
bgneal@700 35 key.make_public()
bgneal@700 36 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 37
bgneal@697 38 def upload_from_filename(self, key_name, filename, metadata=None,
bgneal@697 39 public=True):
bgneal@697 40 """Uploads data from the file named by filename to a new key named
bgneal@697 41 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@697 42 pairs which will be added to the key.
bgneal@697 43
bgneal@697 44 If public is True, the key will be made public after the upload.
bgneal@697 45
bgneal@700 46 Returns the URL to the uploaded file.
bgneal@700 47
bgneal@697 48 """
bgneal@697 49 key = self._make_key(key_name, metadata)
bgneal@697 50 key.set_contents_from_filename(filename)
bgneal@697 51 if public:
bgneal@697 52 key.make_public()
bgneal@700 53 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 54
bgneal@697 55 def upload_from_string(self, key_name, content, metadata=None,
bgneal@697 56 public=True):
bgneal@697 57 """Creates a new key with the given key_name, and uploads the string
bgneal@697 58 content to it. metadata, if not None, must be a dict of metadata key /
bgneal@697 59 value pairs which will be added to the key.
bgneal@697 60
bgneal@697 61 If public is True, the key will be made public after the upload.
bgneal@697 62
bgneal@700 63 Returns the URL to the uploaded file.
bgneal@700 64
bgneal@697 65 """
bgneal@697 66 key = self._make_key(key_name, metadata)
bgneal@697 67 key.set_contents_from_string(content)
bgneal@697 68 if public:
bgneal@697 69 key.make_public()
bgneal@700 70 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 71
bgneal@718 72 def delete_keys(self, key_urls):
bgneal@718 73 """Deletes a set of keys, specified as a list of URLs. The URLs could
bgneal@718 74 have been returned by one or more of the upload_* methods.
bgneal@718 75
bgneal@718 76 Returns the number of keys that were successfully deleted.
bgneal@718 77
bgneal@718 78 """
bgneal@718 79 if len(key_urls) == 0:
bgneal@718 80 return 0
bgneal@718 81
bgneal@718 82 prefix = '{}{}/'.format(self.base_url, self.name)
bgneal@718 83 prefix_len = len(prefix)
bgneal@718 84
bgneal@718 85 keys = []
bgneal@718 86 for url in key_urls:
bgneal@718 87 if url.startswith(prefix):
bgneal@718 88 key = url[prefix_len:]
bgneal@718 89 keys.append(key)
bgneal@718 90
bgneal@718 91 response = self.bucket.delete_keys(keys, quiet=True)
bgneal@718 92 return len(key_urls) - len(response.errors)
bgneal@718 93
bgneal@697 94 def _make_key(self, key_name, metadata):
bgneal@697 95 """Private method to create a key and optionally apply metadata to
bgneal@697 96 it.
bgneal@697 97
bgneal@697 98 """
bgneal@697 99 key = Key(self.bucket)
bgneal@697 100 key.key = key_name
bgneal@697 101 if metadata:
bgneal@697 102 for k, v in metadata.iteritems():
bgneal@697 103 key.set_metadata(k, v)
bgneal@697 104 return key