Skip to content

Commit

Permalink
Reformat with Black.�
Browse files Browse the repository at this point in the history
  • Loading branch information
movestill committed Nov 13, 2020
1 parent 369b64e commit 7d9eb09
Show file tree
Hide file tree
Showing 61 changed files with 4,274 additions and 3,534 deletions.
1 change: 0 additions & 1 deletion __init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

1 change: 0 additions & 1 deletion ndbucket/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

272 changes: 147 additions & 125 deletions ndbucket/cuboidbucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,139 +16,161 @@
from __future__ import print_function
from __future__ import absolute_import
from ndingest.settings.settings import Settings

settings = Settings.load()
import hashlib
import boto3
import botocore
from ndingest.util.util import Util

UtilClass = Util.load()


class CuboidBucket:
"""Cuboid bucket management.
This class is not used by the Boss since it manages its cuboid bucket
via CloudFormation.
"""

def __init__(self, project_name, region_name=settings.REGION_NAME, endpoint_url=None):
"""Create resource for the cuboid queue"""

bucket_name = CuboidBucket.getBucketName()
self.project_name = project_name
self.s3 = boto3.resource(
's3', region_name=region_name, endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
try:
self.bucket = self.s3.Bucket(bucket_name)
except botocore.exceptions.ClientError as e:
print (e)
raise

@staticmethod
def createBucket(region_name=settings.REGION_NAME, endpoint_url=None):
"""Create the cuboid bucket"""

bucket_name = CuboidBucket.getBucketName()
s3 = boto3.resource(
's3', region_name=region_name, endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
bucket = s3.Bucket(bucket_name)
try:
# creating the bucket
response = bucket.create(
ACL = 'private'
)
except Exception as e:
print (e)
raise

@staticmethod
def deleteBucket(region_name=settings.REGION_NAME, endpoint_url=None):
"""Delete the cuboid bucket"""

bucket_name = CuboidBucket.getBucketName()
s3 = boto3.resource(
's3', region_name=region_name, endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
bucket = s3.Bucket(bucket_name)
try:
# deleting the bucket
response = bucket.delete()
except Exception as e:
print (e)
raise

@staticmethod
def getBucketName():
"""Generate the Bucket Name"""
return settings.S3_CUBOID_BUCKET


def putObject(self, channel_name, resolution, morton_index, cube_data, time_index=0):
"""Put object in the cuboid bucket.
Not supported by the Boss. Use putObjectByKey() instead.
"""
supercuboid_key = self.generateSupercuboidKey(channel_name, resolution, morton_index, time_index)
return self.putObjectByKey(supercuboid_key, cube_data)

def putObjectByKey(self, supercuboid_key, cube_data):
"""Put object in the cuboid bucket by key"""

try:
response = self.bucket.put_object(
ACL = 'private',
Body = cube_data,
Key = supercuboid_key,
StorageClass = 'STANDARD'
)
return response
except Exception as e:
print (e)
raise

def getObjectByKey(self, supercuboid_key):
"""Get an object from the cuboid bucket based on key. """

try:
s3_obj = self.s3.Object(self.bucket.name, supercuboid_key)
response = s3_obj.get()
return response['Body'].read()
except Exception as e:
print (e)
raise

def getObject(self, channel_name, resolution, morton_index, time_index=0):
"""Get object from the cuboid bucket based on parameters.
Not supported by the Boss. Use getObjectByKey() instead.
"""
"""Cuboid bucket management.
supercuboid_key = self.generateSupercuboidKey(channel_name, resolution, morton_index, time_index)
return self.getObjectByKey(supercuboid_key)

def generateSupercuboidKey(self, channel_name, resolution, morton_index, time_index=0):
"""Generate the supercuboid key"""
return UtilClass.generateCuboidKey(self.project_name, channel_name, resolution, morton_index, time_index)

def deleteObject(self, supercuboid_key):
"""Delete object from the upload bucket"""

try:
s3_obj = self.s3.Object(self.bucket.name, supercuboid_key)
response = s3_obj.delete()
return response
except Exception as e:
print (e)
raise

def getAllObjects(self):
"""Get a collection of ObjectSummary for all objects in the bucket."""

try:
return self.bucket.objects.all()
except Exception as e:
print (e)
raise
This class is not used by the Boss since it manages its cuboid bucket
via CloudFormation.
"""

def __init__(
self, project_name, region_name=settings.REGION_NAME, endpoint_url=None
):
"""Create resource for the cuboid queue"""

bucket_name = CuboidBucket.getBucketName()
self.project_name = project_name
self.s3 = boto3.resource(
"s3",
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
)
try:
self.bucket = self.s3.Bucket(bucket_name)
except botocore.exceptions.ClientError as e:
print(e)
raise

@staticmethod
def createBucket(region_name=settings.REGION_NAME, endpoint_url=None):
"""Create the cuboid bucket"""

bucket_name = CuboidBucket.getBucketName()
s3 = boto3.resource(
"s3",
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
)
bucket = s3.Bucket(bucket_name)
try:
# creating the bucket
response = bucket.create(ACL="private")
except Exception as e:
print(e)
raise

@staticmethod
def deleteBucket(region_name=settings.REGION_NAME, endpoint_url=None):
"""Delete the cuboid bucket"""

bucket_name = CuboidBucket.getBucketName()
s3 = boto3.resource(
"s3",
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
)
bucket = s3.Bucket(bucket_name)
try:
# deleting the bucket
response = bucket.delete()
except Exception as e:
print(e)
raise

@staticmethod
def getBucketName():
"""Generate the Bucket Name"""
return settings.S3_CUBOID_BUCKET

def putObject(
self, channel_name, resolution, morton_index, cube_data, time_index=0
):
"""Put object in the cuboid bucket.
Not supported by the Boss. Use putObjectByKey() instead.
"""
supercuboid_key = self.generateSupercuboidKey(
channel_name, resolution, morton_index, time_index
)
return self.putObjectByKey(supercuboid_key, cube_data)

def putObjectByKey(self, supercuboid_key, cube_data):
"""Put object in the cuboid bucket by key"""

try:
response = self.bucket.put_object(
ACL="private",
Body=cube_data,
Key=supercuboid_key,
StorageClass="STANDARD",
)
return response
except Exception as e:
print(e)
raise

def getObjectByKey(self, supercuboid_key):
"""Get an object from the cuboid bucket based on key. """

try:
s3_obj = self.s3.Object(self.bucket.name, supercuboid_key)
response = s3_obj.get()
return response["Body"].read()
except Exception as e:
print(e)
raise

def getObject(self, channel_name, resolution, morton_index, time_index=0):
"""Get object from the cuboid bucket based on parameters.
Not supported by the Boss. Use getObjectByKey() instead.
"""

supercuboid_key = self.generateSupercuboidKey(
channel_name, resolution, morton_index, time_index
)
return self.getObjectByKey(supercuboid_key)

def generateSupercuboidKey(
self, channel_name, resolution, morton_index, time_index=0
):
"""Generate the supercuboid key"""
return UtilClass.generateCuboidKey(
self.project_name, channel_name, resolution, morton_index, time_index
)

def deleteObject(self, supercuboid_key):
"""Delete object from the upload bucket"""

try:
s3_obj = self.s3.Object(self.bucket.name, supercuboid_key)
response = s3_obj.delete()
return response
except Exception as e:
print(e)
raise

def getAllObjects(self):
"""Get a collection of ObjectSummary for all objects in the bucket."""

try:
return self.bucket.objects.all()
except Exception as e:
print(e)
raise
Loading

0 comments on commit 7d9eb09

Please sign in to comment.