* use of botornado for async requests (just like willtrking#14)

* hashed path in S3 prevent to serach by prefix in keys. Without real advantage (not a real FS)
* add vows based storage tests
* add .gitignore for *.pyc

.gitignore

@@ -0,0 +1 @@
+*.pyc

thumbor_aws/__init__.py

@@ -1 +1,14 @@
-# coding: utf-8
+# coding: utf-8
+from thumbor.config import Config
+Config.define('STORAGE_BUCKET', 'thumbor-images','S3 bucket for Storage', 'S3 Storage')
+Config.define('RESULT_STORAGE_BUCKET', 'thumbor-result', 'S3 bucket for result Storage', 'S3 Result Storage')
+Config.define('S3_LOADER_BUCKET','thumbor-images','S3 bucket for loader', 'S3 Loader')
+Config.define('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH','', 'S3 path prefix', 'S3 Storage')
+Config.define('STORAGE_EXPIRATION_SECONDS', 3600, 'S3 expiration', 'S3 Storage')
+Config.define('S3_STORAGE_SSE', False, 'S3 encriptipon key', 'S3 Storage')
+Config.define('S3_STORAGE_RRS', False, 'S3 redundency', 'S3 Storage')
+Config.define('S3_ALLOWED_BUCKETS', False, 'List of allowed bucket to be requeted', 'S3 Loader')
+
+Config.define('AWS_ACCESS_KEY', None, 'AWS Access key, if None use environment AWS_ACCESS_KEY_ID', 'AWS')
+Config.define('AWS_SECRET_KEY', None, 'AWS Secret key, if None use environment AWS_SECRET_ACCESS_KEY', 'AWS')
+

thumbor_aws/result_storages/s3_storage.py

@@ -46,8 +46,8 @@ def put(self, bytes):
         file_key.key = file_abspath
 
         file_key.set_contents_from_string(bytes, 
-            encrypt_key = self.context.config.get('S3_STORAGE_SSE', default=False),
-            reduced_redundancy = self.context.config.get('S3_STORAGE_RRS', default=False)
+            encrypt_key = self.context.config.S3_STORAGE_SSE,
+            reduced_redundancy = self.context.config.S3_STORAGE_RRS
         )
 
     def get(self):
@@ -61,12 +61,14 @@ def get(self):
         return file_key.read()
 
     def normalize_path(self, path):
-        root_path = self.context.config.get('RESULT_STORAGE_AWS_STORAGE_ROOT_PATH', default='thumbor/result_storage/')
+        root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
         path_segments = [path]
         if self.is_auto_webp:
             path_segments.append("webp")
-        digest = hashlib.sha1(".".join(path_segments).encode('utf-8')).hexdigest()
-        return os.path.join(root_path, digest)
+        #digest = hashlib.sha1(".".join(path_segments).encode('utf-8')).hexdigest()
+        #return os.path.join(root_path, digest)
+        print os.path.join(root_path, *path_segments)
+        return os.path.join(root_path, *path_segments)
 
     def is_expired(self, key):
         if key:

thumbor_aws/storages/s3_storage.py

@@ -5,7 +5,7 @@
 import hashlib
 from json import loads, dumps
 
-from os.path import splitext
+from os.path import splitext, join
 
 from thumbor.storages import BaseStorage
 from thumbor.utils import logger
@@ -125,8 +125,10 @@ def exists(self, path):
         return True
 
     def normalize_path(self, path):
-        digest = hashlib.sha1(path.encode('utf-8')).hexdigest()
-        return "thumbor/storage/"+digest
+        root_path = self.context.config.RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
+        path_segments = [path]
+        return join(root_path, *path_segments)
+
 
     def is_expired(self, key):
         if key:
@@ -158,4 +160,6 @@ def utc_to_local(self, utc_dt):
         assert utc_dt.resolution >= timedelta(microseconds=1)
         return local_dt.replace(microsecond=utc_dt.microsecond)
 
+    def resolve_original_photo_path(self,filename):
+        return filename
 

vows/fixtures/storage_fixture.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/globocom/thumbor/wiki
+
+# Licensed under the MIT license: 
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com [email protected]
+
+from os.path import join, abspath, dirname
+
+from thumbor.context import ServerParameters, Context
+from thumbor.config import Config
+from thumbor.importer import Importer
+
+IMAGE_URL = 's.glbimg.com/some/image_%s.jpg'
+IMAGE_PATH = join(abspath(dirname(__file__)), 'image.jpg')
+
+with open(IMAGE_PATH, 'r') as img:
+    IMAGE_BYTES = img.read()
+
+def get_server(key=None):
+    server_params = ServerParameters(8888, 'localhost', 'thumbor.conf', None, 'info', None)
+    server_params.security_key = key
+    return server_params
+
+def get_context(server=None, config=None, importer=None):
+    if not server:
+        server = get_server()
+
+    if not config:
+        config = Config()
+
+    if not importer:
+        importer = Importer(config)
+
+    ctx = Context(server=server, config=config, importer=importer)
+    return ctx

vows/storage_vows.py

@@ -0,0 +1,199 @@
+#se!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/globocom/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com [email protected]
+
+
+from pyvows import Vows, expect
+from hashlib import md5
+
+from thumbor.app import ThumborServiceApp
+from thumbor.importer import Importer
+from thumbor.context import Context, ServerParameters
+from thumbor.config import Config
+from fixtures.storage_fixture import IMAGE_URL, IMAGE_BYTES, get_server, AWS
+import time
+
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+
+from thumbor_aws.storages.s3_storage import Storage
+
+bucket = 'viadeo-images-test'
+
+@Vows.batch
+class S3StorageVows(Vows.Context):
+
+  def setup(self):
+    self.bucket = S3Connection().get_bucket(bucket)
+    #clean bucket
+
+    for x in self.bucket.list():
+      x.delete() 
+
+  class CanStoreImage(Vows.Context):
+    def topic(self):
+      thumborId = IMAGE_URL % '1'
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      store = storage.put(thumborId, IMAGE_BYTES )
+      k = Key(self.parent.bucket)
+      k.key = thumborId
+      result = k.get_contents_as_string()
+      return (store , result)
+
+    def should_be_in_catalog(self, topic):
+      expect(topic[0]).to_equal(IMAGE_URL % '1')
+      expect(topic[1]).not_to_be_null()
+      expect(topic[1]).not_to_be_an_error()
+      expect(topic[1]).to_equal(IMAGE_BYTES)
+
+  class CanGetImage(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      storage.put(IMAGE_URL % '2', IMAGE_BYTES)
+      return storage.get(IMAGE_URL % '2')
+
+    def should_not_be_null(self, topic):
+      expect(topic).not_to_be_null()
+      expect(topic).not_to_be_an_error()
+
+    def should_have_proper_bytes(self, topic):
+      expect(topic).to_equal(IMAGE_BYTES)
+
+  class CanGetImageExistance(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      storage.put(IMAGE_URL % '3', IMAGE_BYTES)
+      return storage.exists(IMAGE_URL % '3')
+
+    def should_exists(self, topic):
+      expect(topic).to_equal(True)
+
+  class CanGetImageInexistance(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      return storage.exists(IMAGE_URL % '9999')
+
+    def should_not_exists(self, topic):
+      expect(topic).to_equal(False)
+
+  class CanRemoveImage(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      storage.put(IMAGE_URL % '4', IMAGE_BYTES)
+      created = storage.exists(IMAGE_URL % '4')
+      time.sleep(1)
+      storage.remove(IMAGE_URL % '4')
+      time.sleep(1)
+      return storage.exists(IMAGE_URL % '4') != created
+
+    def should_be_put_and_removed(self, topic):
+      expect(topic).to_equal(True)
+
+  class CanRemovethenPutImage(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      storage.put(IMAGE_URL % '5', IMAGE_BYTES)
+      storage.remove(IMAGE_URL % '5')
+      time.sleep(1)  
+      created = storage.exists(IMAGE_URL % '5')
+      time.sleep(1)
+      storage.put(IMAGE_URL % '5', IMAGE_BYTES)
+      return storage.exists(IMAGE_URL % '5') != created
+
+    def should_be_put_and_removed(self, topic):
+      expect(topic).to_equal(True)
+
+  class CanReturnPath(Vows.Context):
+    def topic(self):
+      config=Config(STORAGE_BUCKET=bucket)
+      storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+      return storage.resolve_original_photo_path("toto")
+
+    def should_return_the_same(self, topic):
+      expect(topic).to_equal("toto")
+
+  class CryptoVows(Vows.Context):
+    class RaisesIfInvalidConfig(Vows.Context):
+      @Vows.capture_error
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True)
+        storage = Storage(Context(config=config, server=get_server('')))
+        storage.put(IMAGE_URL % '9999', IMAGE_BYTES)
+        storage.put_crypto(IMAGE_URL % '9999')
+
+      def should_be_an_error(self, topic):
+        expect(topic).to_be_an_error_like(RuntimeError)
+        expect(topic).to_have_an_error_message_of("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")
+
+    class GettingCryptoForANewImageReturnsNone(Vows.Context):
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True)
+        storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+        return storage.get_crypto(IMAGE_URL % '9999')
+
+      def should_be_null(self, topic):
+        expect(topic).to_be_null()
+
+    class DoesNotStoreIfConfigSaysNotTo(Vows.Context):
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket)
+        storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+        storage.put(IMAGE_URL % '9998', IMAGE_BYTES)
+        storage.put_crypto(IMAGE_URL % '9998')
+        return storage.get_crypto(IMAGE_URL % '9998')
+
+      def should_be_null(self, topic):
+        expect(topic).to_be_null()
+
+    class CanStoreCrypto(Vows.Context):
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket, STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True)
+        storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+        storage.put(IMAGE_URL % '6', IMAGE_BYTES)
+        storage.put_crypto(IMAGE_URL % '6')
+        return storage.get_crypto(IMAGE_URL % '6')
+
+      def should_not_be_null(self, topic):
+        expect(topic).not_to_be_null()
+        expect(topic).not_to_be_an_error()
+
+      def should_have_proper_key(self, topic):
+        expect(topic).to_equal('ACME-SEC')
+
+  class DetectorVows(Vows.Context):
+    class CanStoreDetectorData(Vows.Context):
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket)
+        storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+        storage.put(IMAGE_URL % '7', IMAGE_BYTES)
+        storage.put_detector_data(IMAGE_URL % '7', 'some-data')
+        return storage.get_detector_data(IMAGE_URL % '7')
+
+      def should_not_be_null(self, topic):
+        expect(topic).not_to_be_null()
+        expect(topic).not_to_be_an_error()
+
+      def should_equal_some_data(self, topic):
+        expect(topic).to_equal('some-data')
+
+    class ReturnsNoneIfNoDetectorData(Vows.Context):
+      def topic(self):
+        config=Config(STORAGE_BUCKET=bucket)
+        storage = Storage(Context(config=config, server=get_server('ACME-SEC')))
+        return storage.get_detector_data(IMAGE_URL % '9999')
+
+      def should_not_be_null(self, topic):
+        expect(topic).to_be_null()
+