Allow project managers to request secure directories and conditionall…

…y manage users on them (#653)

* Allow project managers to request secure dirs; fix bug making inactive projects eligible; fix bug allowing 2nd set of dirs; fix bug allowing request under unrelated project

* Add SecureDirRequest.pi field

* Add dropdown to select PI in secure dir request flow

* Add untested runner class for handling a request to create a new secure directory; update notification email text

* Refactor secure dir request view to use runner; remove extraneous logic; clean up breadcrumbs

* Rename migration file; fix failing tests given new form step

* Remove unused logic for adding PIs to secure directory upon directory request approval

* Refactor AllocationDetailView

* Allow Project methods for getting PIs/managers to limit to 'Active'

* Add/integrate method for determining whether user may manage directory

* Notify PI, not requester, that RUA is ready to be signed

* Set MOU/RUA filename+contents based on PI or requester based on request type

* Remove redundant original copies of moved functions

* Refactor/add exception handling to secure dir utils

* Update email template context variables

* Include new PI field when creating SecureDirRequest in MOU test

* Remove unused Project model method

* Break up secure_dir_views into four modules

* Break up secure_dir_utils

* Also email all active PIs upon new directory decisions; adjust tests

* Refactor email logic in user management views

* Update tests

* * Add wrapper object around Allocation representing secure directory
* Refactor user management + permissions logic into it

* Add runners for secure-dir user management requests

* Break up add/remove emails into two files for simplicity

* Clean up view for managing secure-dir users

* * Avoid repeat DB calls to get directory path
* Add in-progress logic to auto-add dir requester to directory

* Auto-add requester to dir if has active cluster access

* Clean up email handling in secure dir views

* Correct import, syntax bugs

* Fix failing tests + bugs

* Display PI in secure dir request list, detail views

* Add migration to fill in SecureDirRequest.pi for existing objs

* Add note about manager permissions to 'Add users' template

* Correct count of users added/removed to/from dir

* * Restore method for fetching PI ProjectUsers to email
* Incorporate it so that only those PIs are notified re: secure dir events

* Add buttons to navigate back in 'data description' step

* Correct phrasing in secure dir request checklist step

* Allow secure dir request requester/PI to upload RUA

coldfront/core/allocation/forms_/secure_dir_forms.py

@@ -1,8 +1,11 @@
 from django import forms
 from django.core.validators import MinLengthValidator
 
-from coldfront.core.allocation.utils_.secure_dir_utils import is_secure_directory_name_suffix_available
-from coldfront.core.allocation.utils_.secure_dir_utils import SECURE_DIRECTORY_NAME_PREFIX
+from coldfront.core.allocation.utils_.secure_dir_utils.new_directory import is_secure_directory_name_suffix_available
+from coldfront.core.allocation.utils_.secure_dir_utils.new_directory import SECURE_DIRECTORY_NAME_PREFIX
+from coldfront.core.project.models import ProjectUser
+from coldfront.core.project.models import ProjectUserRoleChoice
+from coldfront.core.project.models import ProjectUserStatusChoice
 
 
 class SecureDirNameField(forms.CharField):
@@ -72,6 +75,31 @@ class SecureDirManageUsersRequestCompletionForm(forms.Form):
         widget=forms.Select())
 
 
+class SecureDirPISelectionForm(forms.Form):
+
+    pi = forms.ModelChoiceField(
+        label='Principal Investigator',
+        queryset=ProjectUser.objects.none(),
+        required=True,
+        widget=forms.Select())
+
+    def __init__(self, *args, **kwargs):
+        self._project_pk = kwargs.pop('project_pk', None)
+        super().__init__(*args, **kwargs)
+
+        if not self._project_pk:
+            return
+        self._set_pi_queryset()
+
+    def _set_pi_queryset(self):
+        """Set the 'pi' choices to active PIs on the project."""
+        pi_role = ProjectUserRoleChoice.objects.get(
+            name='Principal Investigator')
+        active_status = ProjectUserStatusChoice.objects.get(name='Active')
+        self.fields['pi'].queryset = ProjectUser.objects.filter(
+            project__pk=self._project_pk, role=pi_role, status=active_status)
+
+
 class SecureDirDataDescriptionForm(forms.Form):
     department = forms.CharField(
         label=('Specify the full name of the department that this directory ' 
@@ -96,10 +124,6 @@ class SecureDirDataDescriptionForm(forms.Form):
               'the Information Security and Policy team) about your data?',
         required=False)
 
-    def __init__(self, *args, **kwargs):
-        kwargs.pop('breadcrumb_project', None)
-        super().__init__(*args, **kwargs)
-
 
 class SecureDirRDMConsultationForm(forms.Form):
     rdm_consultants = forms.CharField(
@@ -110,10 +134,6 @@ class SecureDirRDMConsultationForm(forms.Form):
         required=True,
         widget=forms.Textarea(attrs={'rows': 3}))
 
-    def __init__(self, *args, **kwargs):
-        kwargs.pop('breadcrumb_project', None)
-        super().__init__(*args, **kwargs)
-
 
 class SecureDirDirectoryNamesForm(forms.Form):
 
@@ -125,11 +145,6 @@ class SecureDirDirectoryNamesForm(forms.Form):
         required=True,
         widget=forms.Textarea(attrs={'rows': 1}))
 
-    def __init__(self, *args, **kwargs):
-        kwargs.pop('breadcrumb_rdm_consultation', None)
-        kwargs.pop('breadcrumb_project', None)
-        super().__init__(*args, **kwargs)
-
 
 class SecureDirSetupForm(forms.Form):
 
@@ -217,6 +232,7 @@ class SecureDirRDMConsultationReviewForm(forms.Form):
         required=False,
         widget=forms.Textarea(attrs={'rows': 3}))
 
+
 class SecureDirRequestEditDepartmentForm(forms.Form):
 
     department = forms.CharField(

coldfront/core/allocation/migrations/0016_securedirrequest_pi.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.5 on 2024-04-23 15:43
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('allocation', '0015_allocationrenewalrequest_renewal_survey_answers'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='securedirrequest',
+            name='pi',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='secure_dir_request_pi', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='securedirrequest',
+            name='requester',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='secure_dir_request_requester', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

coldfront/core/allocation/migrations/0017_add_pis_to_secure_dir_requests.py

@@ -0,0 +1,31 @@
+from django.db import migrations
+
+
+def set_request_pis_to_requesters(apps, schema_editor):
+    """Prior to this migration, only PIs could request new secure
+    directories. Set the PI of each existing request to be equal to the
+    requester."""
+    SecureDirRequest = apps.get_model('allocation', 'SecureDirRequest')
+    for secure_dir_request in SecureDirRequest.objects.all():
+        secure_dir_request.pi = secure_dir_request.requester
+        secure_dir_request.save()
+
+
+def unset_request_pis(apps, schema_editor):
+    """Unset the PI for all requests."""
+    SecureDirRequest = apps.get_model('allocation', 'SecureDirRequest')
+    for secure_dir_request in SecureDirRequest.objects.all():
+        secure_dir_request.pi = None
+        secure_dir_request.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('allocation', '0016_securedirrequest_pi'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            set_request_pis_to_requesters, unset_request_pis)
+    ]

coldfront/core/allocation/models.py

@@ -729,7 +729,12 @@ def secure_dir_request_state_schema():
 
 
 class SecureDirRequest(TimeStampedModel):
-    requester = models.ForeignKey(User, on_delete=models.CASCADE)
+    requester = models.ForeignKey(
+        User, on_delete=models.CASCADE,
+        related_name='secure_dir_request_requester')
+    pi = models.ForeignKey(
+        User, null=True, on_delete=models.CASCADE,
+        related_name='secure_dir_request_pi')
     directory_name = models.TextField()
     department = models.TextField(null=True)
     data_description = models.TextField()

coldfront/core/allocation/templates/allocation/allocation_detail.html

@@ -203,15 +203,15 @@ <h2 class="d-inline">
       Users in Allocation
     </h2>
     <span class="badge badge-secondary">{{allocation_users.count}}</span>
+    {% if add_remove_users_buttons_visible %}
     <div class="float-right">
-      {% if secure_dir and can_edit_users %}
         {% comment %}
           <a class="btn btn-primary" href="{{mailto}}" role="button"><i class="far fa-envelope"></i> Email Project Users</a>
         {% endcomment %}
         <a class="btn btn-success" href="{% url 'secure-dir-manage-users' allocation.pk 'add' %}" role="button"><i class="fas fa-user-plus" aria-hidden="true"></i> Add Users</a>
         <a class="btn btn-danger" href="{% url 'secure-dir-manage-users' allocation.pk 'remove' %}" role="button"><i class="fas fa-user-times" aria-hidden="true"></i> Remove Users</a>
-      {% endif %}
     </div>
+    {% endif %}
   </div>
   <div class="card-body">
     <div class="table-responsive">
@@ -222,7 +222,9 @@ <h2 class="d-inline">
             <th scope="col">Email</th>
             <th scope="col">Cluster Username</th>
 <!--            <th scope="col">Last Modified</th>-->
-            <th scope="col">Usage</th>
+            {% if allocation_user_usages_visible %}
+              <th scope="col">Usage</th>
+            {% endif %}
             {% flag_enabled 'LRC_ONLY' as lrc_only %}
             {% if lrc_only %}
               <th scope="col">Billing ID</th>
@@ -244,7 +246,9 @@ <h2 class="d-inline">
               </td>
             {% endif %}
 <!--            <td>{{ user.modified|date:"M. d, Y" }}</td>-->
-            <td>{{ allocation_user_su_usages|get_value_from_dict:user.user.username }}</td>
+            {% if allocation_user_usages_visible %}
+              <td>{{ allocation_user_su_usages|get_value_from_dict:user.user.username }}</td>
+            {% endif %}
             {% flag_enabled 'LRC_ONLY' as lrc_only %}
             {% if lrc_only %}
               <td>{{ allocation_user_billing_ids|get_value_from_dict:user.user.username}}</td>
@@ -259,7 +263,7 @@ <h2 class="d-inline">
 </div>
 
 <!-- Start Removed Project Users -->
-{% if not secure_dir %}
+{% if removed_users_visible %}
   <div class="card mb-3">
       <div class="card-header">
           <h2 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Users in Allocation and Removed from Project </h2> <span

coldfront/core/allocation/templates/secure_dir/secure_dir_manage_users.html

@@ -10,17 +10,20 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
   {% if formset %}
     <div class="card border-light">
       <div class="card-body">
-        <form action="{% url url allocation.pk action %}" method="post">
+        {% if action == 'add' %}
+        <p>
+          <b>Note:</b> Any project managers who have been added to the directory will also be able to add and remove users to and from the directory.
+        </p>
+        {% endif %}
+        <form action="{{ manage_users_url }}" method="post">
           {% csrf_token %}
           <div class="table-responsive">
             <table class="table table-sm table-hover">
               <thead>
               <tr>
-                {% if can_manage_users %}
-                  <th>
-                    <input type="checkbox" class="check" id="selectAll">
-                  </th>
-                {% endif %}
+                <th>
+                  <input type="checkbox" class="check" id="selectAll">
+                </th>
                 <th scope="col">#</th>
                 <th scope="col">Username</th>
                 <th scope="col">First Name</th>
@@ -31,9 +34,7 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
               <tbody>
               {% for form in formset %}
                 <tr>
-                  {% if can_manage_users %}
-                    <td>{{ form.selected }}</td>
-                  {% endif %}
+                  <td>{{ form.selected }}</td>
 
                   {% if form.selected %}
                     <td>{{ forloop.counter }}</td>
@@ -55,13 +56,11 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
           </div>
           {{ formset.management_form }}
           <div>
-            {% if can_manage_users %}
-              <button type="submit" class="btn {{ button }}">
-                <i class="fas fa-user-check" aria-hidden="true"></i>
-                {{ action|title }} Selected Users
-              </button>
-            {% endif %}
-            <a class="btn btn-secondary" href="{% url 'allocation-detail' allocation.pk %}" role="button">
+            <button type="submit" class="btn {{ button_class }}">
+              <i class="fas fa-user-check" aria-hidden="true"></i>
+              {{ action|title }} Selected Users
+            </button>
+            <a class="btn btn-secondary" href="{{ allocation_url }}" role="button">
               <i class="fas fa-long-arrow-left" aria-hidden="true"></i>
               Back to Allocation
             </a>
@@ -71,7 +70,7 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
       </div>
     </div>
   {% else %}
-    <a class="btn btn-secondary mb-3" href="{% url 'allocation-detail' allocation.pk %}" role="button">
+    <a class="btn btn-secondary mb-3" href="{{ allocation_url }}" role="button">
       <i class="fas fa-long-arrow-left" aria-hidden="true"></i>
       Back to Allocation
     </a>

coldfront/core/allocation/templates/secure_dir/secure_dir_request/data_description.html

@@ -20,11 +20,11 @@
 
   <h1>Secure Directory: Data Description</h1><hr>
 
-  {% if breadcrumb_project %}
-    <ol class="breadcrumb">
-        <li>{{ breadcrumb_project }}</li>
-    </ol>
-  {% endif %}
+  <ol class="breadcrumb">
+    <li>{{ breadcrumb_project }}</li>
+    <li>&nbsp;<i class="fas fa-angle-double-right"></i>&nbsp;</li>
+    <li>{{ breadcrumb_pi }}</li>
+  </ol>
 
   <p>Please respond to the following questions to provide us with more information about your data.</p>
 
@@ -41,6 +41,24 @@ <h1>Secure Directory: Data Description</h1><hr>
         {{ wizard.form|crispy }}
       {% endif %}
     </table>
+    {% if wizard.steps.prev %}
+      <button
+              class="btn btn-secondary"
+              formnovalidate="formnovalidate"
+              name="wizard_goto_step"
+              type="submit"
+              value="{{ wizard.steps.first }}">
+        First Step
+      </button>
+      <button
+              class="btn btn-secondary"
+              formnovalidate="formnovalidate"
+              name="wizard_goto_step"
+              type="submit"
+              value="{{ wizard.steps.prev }}">
+        Previous Step
+      </button>
+    {% endif %}
     <input class="btn btn-primary" type="submit" value="Next Step"/>
   </form>
   <br>

coldfront/core/allocation/templates/secure_dir/secure_dir_request/directory_name.html

@@ -19,13 +19,11 @@
   <h1>Secure Directory: Directory Name</h1><hr>
 
   <ol class="breadcrumb">
-    {% if breadcrumb_project %}
-      <li>{{ breadcrumb_project }}</li>
-      <li>&nbsp;<i class="fas fa-angle-double-right"></i>&nbsp;</li>
-    {% endif %}
-    <li>Data Description: Submitted</li>
+    <li>{{ breadcrumb_project }}</li>
+    <li>&nbsp;<i class="fas fa-angle-double-right"></i>&nbsp;</li>
+    <li>{{ breadcrumb_pi }}</li>
     <li>&nbsp;<i class="fas fa-angle-double-right"></i>&nbsp;</li>
-    <li>RDM Consult: {{ breadcrumb_rdm_consultation }}</li>
+    <li>{{ breadcrumb_rdm_consultation }}</li>
   </ol>
 
   <p>Please provide the name of the secure directory you are requesting.</p>

coldfront/core/allocation/templates/secure_dir/secure_dir_request/pi_selection.html

@@ -0,0 +1,50 @@
+{% extends "common/base.html" %}
+{% load crispy_forms_tags %}
+{% load static %}
+
+
+{% block title %}
+  Secure Directory Request: PI Selection
+{% endblock %}
+
+
+{% block head %}
+  {{ wizard.form.media }}
+{% endblock %}
+
+
+{% block content %}
+  <script type="text/javascript" src="{% static 'common/js/leave_form_alert.js' %}"></script>
+  <script type='text/javascript' src="{% static 'selectize/selectize.min.js' %}"></script>
+  <link rel='stylesheet' type='text/css' href="{% static 'selectize/selectize.bootstrap3.css' %}"/>
+
+  <h1>Secure Directory: PI Selection</h1><hr>
+
+  {% if breadcrumb_project %}
+    <ol class="breadcrumb">
+        <li>{{ breadcrumb_project }}</li>
+    </ol>
+  {% endif %}
+
+  <p>Select a PI of the project. This PI will be asked to sign a Researcher Use Agreement during the approval process.</p>
+
+  <form action="" method="post">
+    {% csrf_token %}
+    <table>
+      {{ wizard.management_form }}
+      {% if wizard.form.forms %}
+        {{ wizard.form.management_form }}
+        {% for form in wizard.form.forms %}
+          {{ form|crispy }}
+        {% endfor %}
+      {% else %}
+        {{ wizard.form|crispy }}
+      {% endif %}
+    </table>
+    <input class="btn btn-primary" type="submit" value="Next Step"/>
+  </form>
+  <br>
+
+  <p>Step {{ wizard.steps.step1 }} of {{ wizard.steps.count }}</p>
+
+{% endblock %}