Project Sharing + AccountCreation plugin

Summary:
- Rebased to master as of 7/10/2017

- Create a method for creating accounts via 'mappers' like Grouper,
LDAP, and Xsede
- Feature flag the LDAP Group Mapper to only run when ENABLE_PROJECT_SHARING is True
- Remove 'created_by__username' calls for 'is user in list of shared'
for each resource (Instance, Volume)
- Modify ./configure to show 'unexpected errors' and the files that created them.
- Auto-Create a project that corresponds to the project name for the cloud provider.
- Update identity.key to be more verbose
- Include membership_query in 'shared_with_user' for Applications, Volumes, Instances, Identities.
- Include 'created_by' field in Project to make it easier to determine the original author
- Include the author as an implicit leader
- Add 'user' to summary-serializer for instances/volumes (as seen from projects), update leadership query and permissions to be less restrictive
- Validate ownership changes do not happen in projects with shared cloud resources, prior to update.
- Create a 'Feature Flag' -- ENABLE_PROJECT_SHARING. Disable permissions check when False.
- Bugfix: ProviderSummarySerializer was returning null for some results. 'owner' not a required API attribute for projects (When the feature-flag is disabled)
- Rename AccountCreation, Refactor scripts/add_new_accounts
- Move to AccountCreationPluginManager
- Problem: Instance and Volume should not be M2MField
  Solution: Make 'project' a ForeignKey of instance, volume to establish
  a many-to-one relationship with projects.
  - Refactor the project_instance and project_volume API endpoints to mask
  this for the time being.
  - Remove all instances of ProjectInstance, ProjectVolume
- Introduce ProjectMemberRequired, simplify ProjectLeaderRequired to subclass ProjectMemberRequired
- Include queryset filter for Identity and groups (required for Troposphere UI)
- Optimization: Verify identity prior to initializing account driver to save time
- Support for administration of Groups via API
- Add all identities available when creating/updating a group. (Simplification)
- Update APIs to have strict permissions on Create/Update.

Merged into upstream early:
- New monitoring methods for quick bootstrapping of databases.
- Bugfixes required to provide Bootable Volume support

Outside scope of PR but had to be done:
- Remove 'usage per identity' from the v2 serializer.
- Update 'usage per instance' (This is _WRONG_ and will be fixed in a separate PR)
- Remove references to 'old' allocation in the APIs.
- Re-add 'usage' to avoid failure
- Allow project to be updated on volume serializer
- Fix the 'add accounts' logic so that all public providers will be created, even if user already has an identity on some providers.
- Small bugfixes to avoid failure during the creation of new provider && the monitoring of cloud resources

Conflicts:
	api/v2/serializers/details/instance.py
	api/v2/views/instance.py
	api/v2/views/project.py
	atmosphere/settings/local.py.j2
	core/plugins.py
	scripts/add_new_accounts.py
	service/accounts/openstack_manager.py

api/exceptions.py

@@ -133,3 +133,10 @@ def over_allocation(allocation_exception):
     return failure_response(
         status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
         allocation_exception.message)
+
+
+def member_action_forbidden(username, object_type, object_id):
+    message = "User %s does not have permission to act on %s %s. Make sure that you are a Leader of the Project that %s %s resides in." % (username, object_type, object_id, object_type, object_id)
+    return failure_response(
+        status.HTTP_403_FORBIDDEN,
+        message)

api/permissions.py

@@ -8,7 +8,9 @@
 from threepio import logger
 
 from core.models.cloud_admin import CloudAdministrator, cloud_admin_list, get_cloud_admin_for_provider
-from core.models import Group, MaintenanceRecord, AtmosphereUser
+from core.models import (
+    Group, MaintenanceRecord, AtmosphereUser, ExternalLink,
+    Volume, Instance, Project, Identity)
 
 from api import ServiceUnavailable
 from django.conf import settings
@@ -41,8 +43,146 @@ def has_permission(self, request, view):
             logger.warn("Could not find kwarg:'project_uuid'")
             return False
         return any(
-            group for group in auth_user.group_set.all()
-            if group.projects.filter(uuid=project_uuid))
+            membership.group for membership in auth_user.memberships.all()
+            if membership.group.projects.filter(uuid=project_uuid))
+
+
+class ProjectMemberRequired(permissions.BasePermission):
+    message = "The requested user is not a member of the project."
+
+    def has_permission(self, request, view):
+        auth_user = request.user
+        if not auth_user.is_authenticated():
+            return False
+        if not settings.ENABLE_PROJECT_SHARING:
+            return True
+        request_path = request._request.path
+        if request_path.startswith("/api/v1"):
+            return self.v1_membership_test(request, view)
+        return self.v2_membership_test(request, view)
+
+    def v1_membership_test(self, request, view):
+        auth_user = request.user
+        identity_uuid = view.kwargs.get('identity_uuid')
+        instance_id = view.kwargs.get('instance_id')
+        volume_id = view.kwargs.get('volume_id')
+        request_method = request._request.META['REQUEST_METHOD']
+        if request_method == 'GET':
+            return True
+        if instance_id:
+            return self.test_instance_permissions(auth_user, instance_id)
+        elif volume_id:
+            return self.test_volume_permissions(auth_user, volume_id)
+        elif identity_uuid:
+            # Permissions specific to v1 Instance and Volume Creation
+            return self.test_identity_permissions(auth_user, identity_uuid)
+        else:
+            logger.debug("Cannot test membership without a kwarg:'instance_id', 'identity_id', or 'volume_id'")
+            return True
+
+    def v2_membership_test(self, request, view):
+        auth_user = request.user
+        key = view.kwargs.get('pk')
+        request_method = request._request.META['REQUEST_METHOD']
+        if request_method == 'GET':
+            return True  # Querying for the list -- Allow it.
+
+        SerializerCls = getattr(view, 'serializer_class', None)
+        serializer_classname = SerializerCls.__name__
+        # The V2 APIs don't use 'named kwargs' so everything comes in as 'pk'
+        # To overcome this hurdle and disambiguate views, we use the fact that every viewset defines a serializer class.
+        if serializer_classname == "VolumeSerializer":
+            volume_id = key
+            return self.test_volume_permissions(auth_user, volume_id)
+        elif serializer_classname == "InstanceSerializer":
+            instance_id = key
+            return self.test_instance_permissions(auth_user, instance_id)
+        #TODO: Re-evaluate logic below this line.
+        elif serializer_classname == "ProjectSerializer":
+            # Permissions specific to /v2/views/project.py
+            return self.test_project_permissions(auth_user, request_method, key)
+        elif serializer_classname == "ExternalLinkSerializer":
+            # Permissions specific to /v2/views/link.py
+            return self.test_link_permissions(auth_user, key)
+        elif serializer_classname in [
+                "ProjectApplicationSerializer", "ProjectExternalLinkSerializer",
+                "ProjectInstanceSerializer", "ProjectVolumeSerializer"]:
+            # Permissions specific to /v2/views/link.py
+            return self.test_project_resource_permissions(
+                SerializerCls.Meta.model, auth_user, request.data)
+        else:
+            return True
+
+    def test_link_permissions(user, link_id, is_leader=False):
+        link_kwargs = {}
+        if type(link_id) == int:
+            link_kwargs = {'id': link_id}
+        else:
+            link_kwargs = {'uuid': link_id}
+        return ExternalLink.shared_with_user(user, is_leader=is_leader).filter(**link_kwargs).exists()
+
+    def test_volume_permissions(self, user, volume_id, is_leader=False):
+        volume_kwargs = {}
+        if type(volume_id) == int:
+            volume_kwargs = {'id': volume_id}
+        else:
+            volume_kwargs = {'instance_source__identifier': volume_id}
+        return Volume.shared_with_user(user, is_leader=is_leader).filter(**volume_kwargs).exists()
+
+    def test_identity_permissions(self, user, identity_id, is_leader=False):
+        identity_kwargs = {}
+        if type(identity_id) == int:
+            identity_kwargs = {'id': identity_id}
+        else:
+            identity_kwargs = {'uuid': identity_id}
+        return Identity.shared_with_user(user, is_leader=is_leader).filter(**identity_kwargs).exists()
+
+    def test_instance_permissions(self, user, instance_id, is_leader=False):
+        instance_kwargs = {}
+        if type(instance_id) == int:
+            instance_kwargs = {'id': instance_id}
+        else:
+            instance_kwargs = {'provider_alias': instance_id}
+        return Instance.shared_with_user(user, is_leader=is_leader).filter(**instance_kwargs).exists()
+
+    def test_project_permissions(self, user, request_method, project_id, is_leader=False):
+        project_kwargs = {}
+        if not project_id and request_method == 'POST':
+            return True
+        elif type(project_id) == int or len(project_id) != 32:
+            project_kwargs = {'id': project_id}
+        else:
+            project_kwargs = {'uuid': str(project_id)}
+        return Project.shared_with_user(user, is_leader=is_leader).filter(**project_kwargs).exists()
+
+    def test_project_resource_permissions(self, SerializerCls, user, data):
+        if 'instance' in data:
+            return self.test_instance_permissions(user, data['instance'])
+        elif 'volume' in data:
+            return self.test_volume_permissions(user, data['volume'])
+        elif 'external_link' in data:
+            return self.test_link_permissions(user, data['external_link'])
+        else:
+            raise Exception("Unknown data - %s" % data)
+
+
+
+class ProjectLeaderRequired(ProjectMemberRequired):
+    message = "The requested user is not a leader of the project."
+    def test_project_permissions(self, user, request_method, project_id):
+        return self.test_project_permissions(user, request_method, project_id, is_leader=True)
+
+    def test_link_permissions(self, user, link_id):
+        return self.test_link_permissions(user, link_id, is_leader=True)
+
+    def test_volume_permissions(self, user, volume_id):
+        return self.test_volume_permissions(user, volume_id, is_leader=True)
+
+    def test_identity_permissions(self, user, identity_id):
+        return self.test_identity_permissions(user, identity_id, is_leader=True)
+
+    def test_instance_permissions(self, user, instance_id):
+        return self.test_instance_permissions(user, instance_id, is_leader=True)
 
 
 class ApiAuthRequired(permissions.BasePermission):

api/tests/factories/__init__.py

@@ -18,4 +18,3 @@
 from .platform_type_factory import PlatformTypeFactory
 from .size_factory import SizeFactory
 from .allocation_source_factory import AllocationSourceFactory, UserAllocationSourceFactory
-

api/v1/serializers/atmo_user_serializer.py

@@ -20,13 +20,15 @@ def validate_selected_identity(self, selected_identity):
         user = self.instance
         logger.info("Validating identity for %s" % user)
         logger.debug(selected_identity)
-        groups = user.group_set.all()
-        for g in groups:
-            for id_member in g.identity_memberships.all():
+        memberships = user.memberships.all()
+        for membership in memberships:
+            group = membership.group
+            for id_member in group.identity_memberships.all():
                 if id_member.identity == selected_identity:
                     return selected_identity
-        raise serializers.ValidationError( 
-                "User is not a member of selected_identity: %s" % selected_identity)
+        raise serializers.ValidationError(
+            "User is not a member of selected_identity: %s"
+            % selected_identity)
 
     class Meta:
         model = AtmosphereUser

api/v1/serializers/instance_serializer.py

@@ -4,7 +4,6 @@
 from rest_framework import serializers
 from .cleaned_identity_serializer import CleanedIdentitySerializer
 from .tag_related_field import TagRelatedField
-from .projects_field import ProjectsField
 from .boot_script_serializer import BootScriptSerializer
 from .get_context_user import get_context_user
 
@@ -43,7 +42,7 @@ class InstanceSerializer(serializers.ModelSerializer):
         required=False,
         many=True,
         queryset=Tag.objects.all())
-    projects = ProjectsField(required=False)
+    project = serializers.ReadOnlyField(source='project.id')
     scripts = BootScriptSerializer(many=True, required=False)
 
     def __init__(self, *args, **kwargs):

api/v1/serializers/projects_field.py

@@ -27,23 +27,12 @@ def to_internal_value(self, data, files, field_name, into):
         group = get_user_group(user.username)
         # Retrieve the New Project(s)
         if isinstance(value, list):
-            new_projects = value
+            project_id = value[0]
         else:
-            new_projects = [value, ]
+            project_id = value
 
-        # Remove related_obj from Old Project(s)
-        old_projects = related_obj.get_projects(user)
-        for old_proj in old_projects:
-            related_obj.projects.remove(old_proj)
-
-        # Add Project(s) to related_obj
-        for project_id in new_projects:
-            # Retrieve/Create the New Project
-            # TODO: When projects can be shared,
-            # change the qualifier here.
-            new_project = Project.objects.get(id=project_id, owner=group)
-            # Assign related_obj to New Project
-            if not related_obj.projects.filter(id=project_id):
-                related_obj.projects.add(new_project)
+        new_project = Project.objects.get(id=project_id, owner=group)
+        related_obj.project = new_project
+        related_obj.save()
         # Modifications to how 'project' should be displayed here:
-        into[field_name] = new_projects
+        into[field_name] = project_id

api/v1/serializers/volume_serializer.py

@@ -1,7 +1,6 @@
 from core.models.volume import Volume
 from rest_framework import serializers
 from .cleaned_identity_serializer import CleanedIdentitySerializer
-from .projects_field import ProjectsField
 from .get_context_user import get_context_user
 
 
@@ -18,7 +17,7 @@ class VolumeSerializer(serializers.ModelSerializer):
     alias = serializers.ReadOnlyField(source='instance_source.identifier')
     start_date = serializers.ReadOnlyField(source='instance_source.start_date')
     end_date = serializers.ReadOnlyField(source='instance_source.end_date')
-    projects = ProjectsField()
+    project = serializers.ReadOnlyField(source='project.id')  # FIXME: TEST THIS WORKS AS EXPECTED
 
     def __init__(self, *args, **kwargs):
         user = get_context_user(self, kwargs)

api/v1/urls.py

@@ -160,11 +160,6 @@
     url(identity_specific + r'$', views.Identity.as_view(),
         name='identity-detail'),  # OLD
 
-    url(r'^credential$', views.CredentialList.as_view(),
-        name='credential-list'),
-    url(r'^credential/(?P<identity_uuid>%s)$' % (uuid_match,),
-        views.CredentialDetail.as_view(), name='credential-detail'),
-
     url(r'^identity$', views.IdentityDetailList.as_view(),
         name='identity-detail-list'),
     url(r'^identity/(?P<identity_uuid>%s)$' % (uuid_match,),
@@ -269,6 +264,11 @@
     url(identity_specific + r'/meta/(?P<action>.*)$',
         views.MetaAction.as_view(), name='meta-action'),
 
+
+    url(r'^credential$', views.CredentialList.as_view(),
+        name='credential-list'),
+    url(r'^credential/(?P<identity_uuid>%s)$' % (uuid_match,),
+        views.CredentialDetail.as_view(), name='credential-detail'),
     url(identity_specific + r'/members$',
         views.IdentityMembershipList.as_view(),
         name='identity-membership-list'),

api/v1/views/__init__.py

@@ -5,9 +5,9 @@
     CloudAdminAccountList, CloudAdminAccount,\
     CloudAdminInstanceActionList, CloudAdminInstanceAction
 from api.v1.views.credential import CredentialList, CredentialDetail
+from api.v1.views.identity_membership import IdentityMembershipList, IdentityMembership
 from api.v1.views.email import Feedback, QuotaEmail, SupportEmail
 from api.v1.views.group import GroupList, Group
-from api.v1.views.identity_membership import IdentityMembershipList, IdentityMembership
 from api.v1.views.identity import IdentityList, Identity, IdentityDetail,\
     IdentityDetailList
 from api.v1.views.instance import InstanceList, Instance,\

api/v1/views/base.py

@@ -1,15 +1,17 @@
 from rest_framework.generics import ListAPIView
 from rest_framework.views import APIView
-from api.permissions import ApiAuthOptional, ApiAuthRequired, InMaintenance
+from api.permissions import ApiAuthOptional, ApiAuthRequired, InMaintenance, ProjectMemberRequired
 
 
 class MaintenanceAPIView(APIView):
     permission_classes = (InMaintenance,)
 
-
+# NOTE: There should be more strict requirements for 'owner of the Project' to protect "Members of a shared tenant" from interacting (in a bad way) with resources they do not own.
+# For now, all access/ACLs are handled in ProjectMemberRequired
 class AuthAPIView(MaintenanceAPIView):
     permission_classes = (ApiAuthRequired,
-                          InMaintenance,)
+                          InMaintenance,
+                          ProjectMemberRequired)
 
 
 class AuthOptionalAPIView(MaintenanceAPIView):

api/v1/views/group.py

@@ -45,7 +45,8 @@ def get(self, request):
         Authentication Required, A list of all the user's groups.
         """
         user = request.user
-        all_groups = user.group_set.order_by('name')
+        all_memberships = user.memberships.select_related('group').order_by('group__name')
+        all_groups = [member.group for member in all_memberships]
         serialized_data = GroupSerializer(all_groups).data
         response = Response(serialized_data)
         return response
@@ -66,7 +67,7 @@ def get(self, request, groupname):
         """
         logger.info(request.__dict__)
         user = request.user
-        group = user.group_set.get(name=groupname)
+        group = user.memberships.get(group__name=groupname).group
         serialized_data = GroupSerializer(group).data
         response = Response(serialized_data)
         return response

api/v1/views/identity.py

@@ -18,17 +18,13 @@ def get_provider(user, provider_uuid):
     return None or an Active provider
     """
     try:
-        group = Group.objects.get(name=user.username)
-    except Group.DoesNotExist:
-        logger.warn("Group %s DoesNotExist" % user.username)
-        return None
-
-    try:
-        provider = group.current_providers.get(uuid=provider_uuid)
+        provider = user.current_providers.get(uuid=provider_uuid)
         return provider
     except Provider.DoesNotExist:
-        logger.warn("Provider %s DoesNotExist for User:%s in Group:%s"
-                    % (provider_uuid, user, group))
+        logger.warn(
+            "Provider %s DoesNotExist and/or has not "
+            "been shared with any of the groups for User:%s"
+            % (provider_uuid, user))
         return None
 
 
@@ -37,20 +33,10 @@ def get_identity_list(user, provider=None):
     Given the (request) user
     return all identities on all active providers
     """
-    try:
-        group = Group.objects.get(name=user.username)
-        if provider:
-            identity_list = group.current_identities.filter(
-                provider=provider)
-        else:
-            identity_list = group.current_identities.all()
-        return identity_list
-    except Group.DoesNotExist:
-        logger.warn("Group %s DoesNotExist" % user.username)
-        return None
-    except CoreIdentity.DoesNotExist:
-        logger.warn("Identity DoesNotExist for user %s" % user.username)
-        return None
+    identity_list = CoreIdentity.shared_with_user(user)
+    if provider:
+        identity_list = identity_list.filter(provider=provider)
+    return identity_list
 
 
 def get_identity(user, identity_uuid):