Merge pull request #357 from isb-cgc/isb-cgc-prod-sp

Sprint 18

accounts/migrations/0013_authorizeddataset_duca_id.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2017-08-15 22:26
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0012_serviceaccount_authorized_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='authorizeddataset',
+            name='duca_id',
+            field=models.CharField(max_length=256, null=True),
+        ),
+    ]

accounts/models.py

@@ -38,6 +38,16 @@ class Meta:
     def get_google_email(self):
         return User.objects.get(pk=self.user_id).email
 
+    def get_auth_datasets(self):
+        result = None
+        try:
+            result = AuthorizedDataset.objects.filter(
+                id__in=self.userauthorizeddatasets_set.all().values_list('authorized_dataset', flat=True))
+        except Exception as e:
+            logger.error("[ERROR] While retrieving authorized datasets: ")
+            logger.exception(e)
+        return result
+
 
 class GoogleProject(models.Model):
     user = models.ManyToManyField(User)
@@ -67,6 +77,7 @@ class AuthorizedDataset(models.Model):
     whitelist_id = models.CharField(max_length=256, null=False)
     acl_google_group = models.CharField(max_length=256, null=False)
     public = models.BooleanField(default=False)
+    duca_id = models.CharField(max_length=256, null=True)
 
     def __str__(self):
         return self.name

accounts/views.py

@@ -43,7 +43,6 @@
 logger = logging.getLogger('main_logger')
 
 OPEN_ACL_GOOGLE_GROUP = settings.OPEN_ACL_GOOGLE_GROUP
-CONTROLLED_ACL_GOOGLE_GROUP = settings.ACL_GOOGLE_GROUP
 SERVICE_ACCOUNT_LOG_NAME = settings.SERVICE_ACCOUNT_LOG_NAME
 SERVICE_ACCOUNT_BLACKLIST_PATH = settings.SERVICE_ACCOUNT_BLACKLIST_PATH
 
@@ -73,7 +72,7 @@ def extended_logout_view(request):
                     for dataset in user_auth_datasets:
                         dataset.delete()
             else:
-                logger.error("[ERROR] No NIH user was found for user {} - no datasets revoked.".format(user.email))
+                logger.info("[STATUS] No NIH user was found for user {} - no datasets revoked.".format(user.email))
 
         directory_service, http_auth = get_directory_resource()
         user_email = user.email
@@ -185,6 +184,7 @@ def unlink_accounts_and_get_acl_tasks(user_id):
 
 @login_required
 def unlink_accounts(request):
+    logger.info("[STATUS] In unlink accounts")
     user_id = request.user.id
 
     try:
@@ -283,8 +283,16 @@ def user_gcp_list(request, user_id):
 
 @login_required
 def verify_gcp(request, user_id):
+    message = None
+    status = None
     try:
         gcp_id = request.GET.get('gcp-id', None)
+
+        gcp = GoogleProject.objects.filter(project_id=gcp_id)
+        # Can't register the same GCP twice - return immediately
+        if len(gcp) > 0:
+            return JsonResponse({'message': 'A Google Cloud Project with the project ID {} has already been registered.'.format(str(gcp_id))}, status='500')
+
         crm_service = get_special_crm_resource()
         iam_policy = crm_service.projects().getIamPolicy(
             resource=gcp_id, body={}).execute()
@@ -307,40 +315,65 @@ def verify_gcp(request, user_id):
                                        'registered_user': registered_user})
 
         if not user_found:
-            return JsonResponse({'message': 'You were not found on the project. You may not register a project you do not belong to.'}, status='403')
+            logger.error("[ERROR] While attempting to register GCP ID {}: ".format(str(gcp_id)))
+            logger.error("User {} was not found on GCP {}.".format(user.email,str(gcp_id)))
+            message = 'Your user email {} was not found in GCP {}. You may not register a project you do not belong to.'.format(user.email,str(gcp_id))
+            status='403'
         else:
             return JsonResponse({'roles': roles,
                                 'gcp_id': gcp_id}, status='200')
-    except HttpError:
-        return JsonResponse({'message': 'There was an error accessing your project. Please verify that you have entered the correct Google Cloud Project ID and set the permissions correctly.'}, status='403')
+    except Exception as e:
+        if type(e) is HttpError:
+            logger.error("[ERROR] While trying to access IAM policies for GCP ID {}:".format(str(gcp_id)))
+            message = 'There was an error accessing this project. Please verify that you have entered the correct Google Cloud Project ID and set the permissions correctly.'
+            status = '403'
+        else:
+            logger.error("[ERROR] While trying to verify GCP ID {}:".format(str(gcp_id)))
+            message = 'There was an error while attempting to verify this project. Please verify that you have entered the correct Google Cloud Project ID and set the permissions correctly.'
+            status = '500'
+        logger.exception(e)
+
+    return JsonResponse({'message': message}, status=status)
+
 
 @login_required
 def register_gcp(request, user_id):
-    if request.POST:
-        project_id = request.POST.get('gcp_id', None)
-        project_name = project_id
 
-        register_users = request.POST.getlist('register_users')
-        if not user_id or not project_id or not project_name:
-            pass
-        else:
-            try:
-                gcp = GoogleProject.objects.get(project_name=project_name,
-                                                project_id=project_id)
-            except ObjectDoesNotExist:
-                gcp = GoogleProject.objects.create(project_name=project_name,
-                                                   project_id=project_id,
-                                                   big_query_dataset='')
+    try:
+        if request.POST:
+            project_id = request.POST.get('gcp_id', None)
+            project_name = project_id
+
+            register_users = request.POST.getlist('register_users')
+            if not user_id or not project_id or not project_name:
+                pass
+            else:
+                try:
+                    gcp = GoogleProject.objects.get(project_name=project_name,
+                                                    project_id=project_id)
+                    messages.info(request, "A Google Cloud Project with the id {} already exists.".format(project_id))
+
+                except ObjectDoesNotExist:
+                    gcp = GoogleProject.objects.create(project_name=project_name,
+                                                       project_id=project_id,
+                                                       big_query_dataset='')
+                    gcp.save()
+
+            users = User.objects.filter(email__in=register_users)
+
+            for user in users:
+                gcp.user.add(user)
                 gcp.save()
+            return redirect('user_gcp_list', user_id=request.user.id)
 
-        users = User.objects.filter(email__in=register_users)
+    except Exception as e:
+        logger.error("[ERROR] While registering a Google Cloud Project:")
+        logger.exception(e)
+        messages.error(request, "There was an error while attempting to register this Google Cloud Project - please contact the administrator.")
 
-        for user in users:
-            gcp.user.add(user)
-            gcp.save()
-        return redirect('user_gcp_list', user_id=request.user.id)
     return render(request, 'GenespotRE/register_gcp.html', {})
 
+
 @login_required
 def gcp_detail(request, user_id, gcp_id):
     context = {}
@@ -366,7 +399,7 @@ def user_gcp_delete(request, user_id, gcp_id):
 
                     logger.info("Attempting to delete user {} from group {}. "
                                 "If an error message doesn't follow, they were successfully deleted"
-                                .format(saad.service_account.service_account, CONTROLLED_ACL_GOOGLE_GROUP))
+                                .format(saad.service_account.service_account, saad.authorized_dataset.acl_google_group))
                 except HttpError as e:
                     logger.info(e)
 
@@ -379,7 +412,7 @@ def user_gcp_delete(request, user_id, gcp_id):
     return redirect('user_gcp_list', user_id=request.user.id)
 
 
-def verify_service_account(gcp_id, service_account, datasets, user_email):
+def verify_service_account(gcp_id, service_account, datasets, user_email, is_refresh=False):
     # Only verify for protected datasets
     dataset_objs = AuthorizedDataset.objects.filter(id__in=datasets, public=False)
     dataset_obj_names = dataset_objs.values_list('name', flat=True)
@@ -408,6 +441,32 @@ def verify_service_account(gcp_id, service_account, datasets, user_email):
         st_logger.write_text_log_entry(log_name, "{0}: Service account is blacklisted.".format(service_account))
         return {'message': 'This service account cannot be registered.'}
 
+
+    # Refreshes require a service account to exist, and, you cannot register an account if it already exists with the same datasets
+    try:
+        sa = ServiceAccount.objects.get(service_account=service_account)
+        if not is_refresh:
+            reg_change = False
+            # If there are private datasets requested, it might not be a duplicate
+            if len(dataset_objs):
+                saads = AuthorizedDataset.objects.filter(id__in=ServiceAccountAuthorizedDatasets.objects.filter(service_account=sa).values_list('authorized_dataset', flat=True), public=False).values_list('whitelist_id',flat=True)
+                ads = dataset_objs.values_list('whitelist_id', flat=True)
+                # Only if the lengthes of the 2 dataset lists are the same do we need to check them against one another
+                if not reg_change:
+                    for ad in ads:
+                        if ad not in saads:
+                            reg_change = True
+            # but if there are not, it's only not a duplicate if the public dataset isn't yet registered
+            else:
+                reg_change = (len(AuthorizedDataset.objects.filter(id__in=ServiceAccountAuthorizedDatasets.objects.filter(service_account=sa).values_list('authorized_dataset', flat=True), public=True)) <= 0)
+            # If this isn't a refresh and the requested datasets aren't changing, we don't need to re-register
+            if not reg_change:
+                return {'message': 'Service account {} already exists with these datasets, and so does not need to be registered'.format(str(service_account))}
+    except ObjectDoesNotExist:
+        if is_refresh:
+            return {'message': 'Service account {} was not found so cannot be refreshed.'.format(str(service_account))}
+
+
     # 1. GET ALL USERS ON THE PROJECT.
     try:
         crm_service = get_special_crm_resource()
@@ -530,7 +589,8 @@ def verify_sa(request, user_id):
             gcp_id = request.POST.get('gcp_id')
             user_sa = request.POST.get('user_sa')
             datasets = request.POST.getlist('datasets')
-            result = verify_service_account(gcp_id, user_sa, datasets, user_email)
+            is_refresh = bool(request.POST.get('is_refresh') == 'true')
+            result = verify_service_account(gcp_id, user_sa, datasets, user_email, is_refresh)
             if 'message' in result.keys():
                 status = '400'
                 st_logger.write_struct_log_entry(SERVICE_ACCOUNT_LOG_NAME, {'message': '{0}: {1}'.format(user_sa, result['message'])})
@@ -571,6 +631,7 @@ def register_sa(request, user_id):
             gcp_id = request.POST.get('gcp_id')
             user_sa = request.POST.get('user_sa')
             datasets = request.POST.get('datasets').split(',')
+            is_refresh = bool(request.POST.get('is_refresh') == 'true')
             user_gcp = GoogleProject.objects.get(project_id=gcp_id)
 
             if len(datasets) == 1 and datasets[0] == '':
@@ -579,7 +640,7 @@ def register_sa(request, user_id):
                 datasets = map(int, datasets)
 
             # VERIFY AGAIN JUST IN CASE USER TRIED TO GAME THE SYSTEM
-            result = verify_service_account(gcp_id, user_sa, datasets, user_email)
+            result = verify_service_account(gcp_id, user_sa, datasets, user_email, is_refresh)
             logger.info("[STATUS] result of verification for {}: {}".format(user_sa,str(result)))
 
             # If the verification was successful, finalize access

cohorts/models.py

@@ -43,6 +43,7 @@ def get_all_tcga_cohort(self):
         all_isb_cohort_ids = Cohort_Perms.objects.filter(user=isb_user, perm=Cohort_Perms.OWNER).values_list('cohort_id', flat=True)
         return Cohort.objects.filter(name='All TCGA Data', id__in=all_isb_cohort_ids)[0]
 
+
 class Cohort(models.Model):
     id = models.AutoField(primary_key=True)
     name = models.TextField(null=True)