Tests for googlestorage_container.AuthenticationClient.

Author: Itamar Turner-Trauring <itamar@futurefoundries.com>

src/allmydata/storage/backends/cloud/googlestorage/googlestorage_container.py

@@ -6,7 +6,7 @@ http://code.google.com/p/google-api-python-client/downloads/list
 
 import httplib2
 
-from twisted.internet.defer import DeferredLock
+from twisted.internet.defer import DeferredLock, maybeDeferred
 from twisted.internet.threads import deferToThread
 
 from oauth2client.client import SignedJwtAssertionCredentials
@@ -34,29 +34,48 @@ class AuthenticationClient(object):
     more details.
     """
 
-    def __init__(self, account_name, private_key, private_key_password='notasecret'):
+    def __init__(self, account_name, private_key, private_key_password='notasecret',
-        self.credentials = SignedJwtAssertionCredentials(
+                 _credentialsClass=SignedJwtAssertionCredentials,
+                 _deferToThread=deferToThread):
+        # Google ships pkcs12 private keys encrypted with "notasecret" as the
+        # password. In order for automated running to work we'd need to
+        # include the password in the config file, so it adds no extra
+        # security even if someone chooses a different password. So it's seems
+        # simplest to hardcode it for now and it'll work with unmodified
+        # private keys issued by Google.
+        self._credentials = _credentialsClass(
             account_name, private_key,
             "https://www.googleapis.com/auth/devstorage.read_write",
             private_key_password = private_key_password,
             )
+        self._deferToThread = _deferToThread
         self._need_first_auth = True
         self._lock = DeferredLock()
+        # Get initial token:
+        self._refresh_if_necessary(force=True)
+
+    def _refresh_if_necessary(self, force=False):
+        """
+        Get a new authorization token, if necessary.
+        """
+        def run():
+            if force or self._credentials.access_token_expired:
+                # Generally using a task-specific thread pool is better than using
+                # the reactor one. However, this particular call will only run
+                # once an hour, so it's not likely to tie up all the threads.
+                return self._deferToThread(self._credentials.refresh, httplib2.Http())
+        return self._lock.run(run)
 
     def get_authorization_header(self):
         """
         Return a Deferred that fires with the value to use for the
         Authorization header in HTTP requests.
         """
-        def refreshIfNecessary():
+        d = self._refresh_if_necessary()
-            if self._need_first_auth or self.credentials.access_token_expired:
-                self._need_first_auth = False
-                return deferToThread(self.credentials.refresh, httplib2.Http())
-        d = self._lock.run(refreshIfNecessary)
 
         def refreshed(ignore):
             headers = {}
-            self.credentials.apply(headers)
+            self._credentials.apply(headers)
             return headers['Authorization']
         d.addCallback(refreshed)
         return d

src/allmydata/test/test_storage.py

@@ -2,6 +2,7 @@
 import time, os.path, platform, re, simplejson, struct, itertools, urllib
 from collections import deque
 from cStringIO import StringIO
+import thread
 
 import mock
 from twisted.trial import unittest
@@ -33,6 +34,7 @@ from allmydata.storage.backends.cloud.cloud_common import CloudError, CloudServi
 from allmydata.storage.backends.cloud import mock_cloud, cloud_common
 from allmydata.storage.backends.cloud.mock_cloud import MockContainer
 from allmydata.storage.backends.cloud.openstack import openstack_container
+from allmydata.storage.backends.cloud.googlestorage import googlestorage_container
 from allmydata.storage.bucket import BucketWriter, BucketReader
 from allmydata.storage.common import DataTooLargeError, storage_index_to_dir
 from allmydata.storage.leasedb import SHARETYPE_IMMUTABLE, SHARETYPE_MUTABLE
@@ -629,6 +631,132 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u
         return d
 
 
+
+class GoogleStorageBackend(ShouldFailMixin, unittest.TestCase):
+    """
+    Tests for the Google Storage API backend.
+
+    All code references in docstrings/comments are to classes/functions in
+    allmydata.storage.backends.cloud.googlestorage.googlestorage_container
+    unless noted otherwise.
+    """
+
+    def test_authentication_credentials(self):
+        """
+        AuthenticationClient.get_authorization_header() initializes a
+        SignedJwtAssertionCredentials with the correct parameters.
+        """
+        # Somewhat fragile tests, but better than nothing.
+        auth = googlestorage_container.AuthenticationClient("u@example.com", "xxx123")
+        self.assertEqual(auth._credentials.service_account_name, "u@example.com")
+        self.assertEqual(auth._credentials.private_key, "xxx123".encode("base64").strip())
+
+    def test_authentication_initial(self):
+        """
+        When AuthenticationClient() is created, it refreshes its access token.
+        """
+        from oauth2client.client import SignedJwtAssertionCredentials
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=mock.create_autospec(SignedJwtAssertionCredentials),
+            _deferToThread=defer.maybeDeferred)
+        self.assertEqual(auth._credentials.refresh.call_count, 1)
+
+    def test_authentication_expired(self):
+        """
+        AuthenticationClient.get_authorization_header() refreshes its
+        credentials if the access token has expired.
+        """
+        from oauth2client.client import SignedJwtAssertionCredentials
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=mock.create_autospec(SignedJwtAssertionCredentials),
+            _deferToThread=defer.maybeDeferred)
+        auth._credentials.apply = lambda d: d.__setitem__('Authorization', 'xxx')
+        auth._credentials.access_token_expired = True
+        auth.get_authorization_header()
+        self.assertEqual(auth._credentials.refresh.call_count, 2)
+
+    def test_authentication_no_refresh(self):
+        """
+        AuthenticationClient.get_authorization_header() does not refresh its
+        credentials if the access token has not expired.
+        """
+        from oauth2client.client import SignedJwtAssertionCredentials
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=mock.create_autospec(SignedJwtAssertionCredentials),
+            _deferToThread=defer.maybeDeferred)
+        auth._credentials.apply = lambda d: d.__setitem__('Authorization', 'xxx')
+        auth._credentials.access_token_expired = False
+        auth.get_authorization_header()
+        self.assertEqual(auth._credentials.refresh.call_count, 1)
+
+    def test_authentication_header(self):
+        """
+        AuthenticationClient.get_authorization_header() returns a value to be
+        used for the Authorization header.
+        """
+        from oauth2client.client import SignedJwtAssertionCredentials
+        class NoNetworkCreds(SignedJwtAssertionCredentials):
+            def refresh(self, http):
+                self.access_token = "xxx"
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=NoNetworkCreds,
+            _deferToThread=defer.maybeDeferred)
+        result = []
+        auth.get_authorization_header().addCallback(result.append)
+        self.assertEqual(result, ["Bearer xxx"])
+
+    def test_authentication_one_refresh(self):
+        """
+        AuthenticationClient._refresh_if_necessary() only runs one refresh
+        request at a time.
+        """
+        # The second call shouldn't happen until the first Deferred fires!
+        results = [defer.Deferred(), defer.succeed(None)]
+        first = results[0]
+
+        def fakeDeferToThread(f, *args):
+            return results.pop(0)
+
+        from oauth2client.client import SignedJwtAssertionCredentials
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=mock.create_autospec(SignedJwtAssertionCredentials),
+            _deferToThread=fakeDeferToThread)
+        # Initial authorization call happens...
+        self.assertEqual(len(results), 1)
+        # ... and still isn't finished, so next one doesn't run yet:
+        auth._refresh_if_necessary(force=True)
+        self.assertEqual(len(results), 1)
+        # When first one finishes, second one can run:
+        first.callback(None)
+        self.assertEqual(len(results), 0)
+
+    def test_authentication_refresh_call(self):
+        """
+        AuthenticationClient._refresh_if_necessary() runs the
+        authentication refresh in a thread, since it blocks, with a
+        httplib2.Http instance.
+        """
+        from httplib2 import Http
+        from oauth2client.client import SignedJwtAssertionCredentials
+        class NoNetworkCreds(SignedJwtAssertionCredentials):
+            def refresh(cred_self, http):
+                cred_self.access_token = "xxx"
+                self.assertIsInstance(http, Http)
+                self.thread = thread.get_ident()
+        auth = googlestorage_container.AuthenticationClient(
+            "u@example.com", "xxx123",
+            _credentialsClass=NoNetworkCreds)
+
+        def gotResult(ignore):
+            self.assertNotEqual(thread.get_ident(), self.thread)
+        return auth.get_authorization_header().addCallback(gotResult)
+
+
 class ServerMixin:
     def allocate(self, account, storage_index, sharenums, size, canary=None):
         # These secrets are not used, but clients still provide them.