Mercurial > public > sg101

annotate core/s3.py @ 1197:ba23e79438f4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Another attempt to write BytesIO to a file.
author Brian Neal <bgneal@gmail.com>
date Sun, 07 May 2023 19:23:35 -0500
parents e4f2d6a4b401
children
rev   line source
bgneal@697 1 """s3.py
bgneal@697 2
bgneal@700 3 This module provides Amazon S3 convenience wrappers.
bgneal@697 4
bgneal@697 5 """
bgneal@1157 6 from boto.s3 import connect_to_region
bgneal@1157 7 from boto.s3.connection import OrdinaryCallingFormat
bgneal@697 8 from boto.s3.key import Key
bgneal@697 9
bgneal@697 10
bgneal@697 11 class S3Bucket(object):
bgneal@697 12 """This class abstracts an Amazon S3 bucket.
bgneal@697 13
bgneal@697 14 """
bgneal@1157 15 def __init__(self, access_key, secret_key, base_url, bucket_name,
bgneal@1157 16 region_name='us-west-1'):
bgneal@1157 17 self.region_name = region_name
bgneal@1157 18 self.conn = self._get_connection(access_key, secret_key)
bgneal@697 19 self.bucket = self.conn.get_bucket(bucket_name, validate=False)
bgneal@700 20 self.base_url = base_url
bgneal@700 21 if not base_url.endswith('/'):
bgneal@700 22 self.base_url += '/'
bgneal@700 23 self.name = bucket_name
bgneal@700 24
bgneal@1157 25 def _get_connection(self, access_key, secret_key):
bgneal@1157 26 conn = connect_to_region(
bgneal@1157 27 self.region_name,
bgneal@1157 28 aws_access_key_id=access_key,
bgneal@1157 29 aws_secret_access_key=secret_key,
bgneal@1157 30 calling_format=OrdinaryCallingFormat())
bgneal@1157 31 return conn
bgneal@1157 32
bgneal@700 33 def upload_from_file(self, key_name, fp, metadata=None, public=True):
bgneal@700 34 """Uploads data from the file object fp to a new key named
bgneal@700 35 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@700 36 pairs which will be added to the key.
bgneal@700 37
bgneal@700 38 If public is True, the key will be made public after the upload.
bgneal@700 39
bgneal@700 40 Returns the URL to the uploaded file.
bgneal@700 41
bgneal@700 42 """
bgneal@700 43 key = self._make_key(key_name, metadata)
bgneal@700 44 key.set_contents_from_file(fp)
bgneal@700 45 if public:
bgneal@700 46 key.make_public()
bgneal@700 47 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 48
bgneal@697 49 def upload_from_filename(self, key_name, filename, metadata=None,
bgneal@697 50 public=True):
bgneal@697 51 """Uploads data from the file named by filename to a new key named
bgneal@697 52 key_name. metadata, if not None, must be a dict of metadata key / value
bgneal@697 53 pairs which will be added to the key.
bgneal@697 54
bgneal@697 55 If public is True, the key will be made public after the upload.
bgneal@697 56
bgneal@700 57 Returns the URL to the uploaded file.
bgneal@700 58
bgneal@697 59 """
bgneal@697 60 key = self._make_key(key_name, metadata)
bgneal@697 61 key.set_contents_from_filename(filename)
bgneal@697 62 if public:
bgneal@697 63 key.make_public()
bgneal@700 64 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 65
bgneal@697 66 def upload_from_string(self, key_name, content, metadata=None,
bgneal@697 67 public=True):
bgneal@697 68 """Creates a new key with the given key_name, and uploads the string
bgneal@697 69 content to it. metadata, if not None, must be a dict of metadata key /
bgneal@697 70 value pairs which will be added to the key.
bgneal@697 71
bgneal@697 72 If public is True, the key will be made public after the upload.
bgneal@697 73
bgneal@700 74 Returns the URL to the uploaded file.
bgneal@700 75
bgneal@697 76 """
bgneal@697 77 key = self._make_key(key_name, metadata)
bgneal@697 78 key.set_contents_from_string(content)
bgneal@697 79 if public:
bgneal@697 80 key.make_public()
bgneal@700 81 return '{}{}/{}'.format(self.base_url, self.name, key_name)
bgneal@697 82
bgneal@718 83 def delete_keys(self, key_urls):
bgneal@718 84 """Deletes a set of keys, specified as a list of URLs. The URLs could
bgneal@718 85 have been returned by one or more of the upload_* methods.
bgneal@718 86
bgneal@718 87 Returns the number of keys that were successfully deleted.
bgneal@718 88
bgneal@718 89 """
bgneal@718 90 if len(key_urls) == 0:
bgneal@718 91 return 0
bgneal@718 92
bgneal@718 93 prefix = '{}{}/'.format(self.base_url, self.name)
bgneal@718 94 prefix_len = len(prefix)
bgneal@718 95
bgneal@718 96 keys = []
bgneal@718 97 for url in key_urls:
bgneal@718 98 if url.startswith(prefix):
bgneal@718 99 key = url[prefix_len:]
bgneal@718 100 keys.append(key)
bgneal@718 101
bgneal@718 102 response = self.bucket.delete_keys(keys, quiet=True)
bgneal@718 103 return len(key_urls) - len(response.errors)
bgneal@718 104
bgneal@697 105 def _make_key(self, key_name, metadata):
bgneal@697 106 """Private method to create a key and optionally apply metadata to
bgneal@697 107 it.
bgneal@697 108
bgneal@697 109 """
bgneal@697 110 key = Key(self.bucket)
bgneal@697 111 key.key = key_name
bgneal@697 112 if metadata:
bgneal@697 113 for k, v in metadata.iteritems():
bgneal@697 114 key.set_metadata(k, v)
bgneal@697 115 return key