From df905ff698765e13519d10a60020811c1eba093f Mon Sep 17 00:00:00 2001 From: Leszek Pietrzak Date: Wed, 13 Nov 2024 14:02:02 +0100 Subject: [PATCH 1/8] style(universalTable) use Array for columns prop (#5260) --- .../universalTable/paginatedQueryUniversalTable.component.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsapp/js/universalTable/paginatedQueryUniversalTable.component.tsx b/jsapp/js/universalTable/paginatedQueryUniversalTable.component.tsx index b5775af594..e53245b3d5 100644 --- a/jsapp/js/universalTable/paginatedQueryUniversalTable.component.tsx +++ b/jsapp/js/universalTable/paginatedQueryUniversalTable.component.tsx @@ -18,7 +18,7 @@ interface PaginatedQueryUniversalTableProps { // Below are props from `UniversalTable` that should come from the parent // component (these are kind of "configuration" props). The other // `UniversalTable` props are being handled here internally. - columns: UniversalTableColumn[]; + columns: Array>; } const PAGE_SIZES = [10, 30, 50, 100]; From 9c51f17cf942e1932f6068c634167825da2fd700 Mon Sep 17 00:00:00 2001 From: James Kiger <68701146+jamesrkiger@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:27:42 -0500 Subject: [PATCH 2/8] refactor(organizations): add configurable redirect route to org validator TASK-975 (#5262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### đŸ“Ŗ Summary Makes the `ValidateOrgPermissions` component require a redirect route so it can be used outside of the account settings area of the app. Moves the component file to the root router folder. ### 💭 Notes I made the redirect route a required prop. Previously redirect was a boolean that defaulted to true. We don't currently have any use case for not doing a redirect with this component and requiring it reduces boolean checks. --------- Co-authored-by: Leszek Pietrzak --- jsapp/js/account/routes.tsx | 20 +++++++++++++++---- .../validateOrgPermissions.component.tsx | 15 ++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) rename jsapp/js/{account/organizations => router}/validateOrgPermissions.component.tsx (74%) diff --git a/jsapp/js/account/routes.tsx b/jsapp/js/account/routes.tsx index 1046d5ed85..ba5f3a313e 100644 --- a/jsapp/js/account/routes.tsx +++ b/jsapp/js/account/routes.tsx @@ -1,7 +1,7 @@ import React from 'react'; import {Navigate, Route} from 'react-router-dom'; import RequireAuth from 'js/router/requireAuth'; -import {ValidateOrgPermissions} from 'js/account/organizations/validateOrgPermissions.component'; +import {ValidateOrgPermissions} from 'js/router/validateOrgPermissions.component'; import {OrganizationUserRole} from './stripe.types'; import { ACCOUNT_ROUTES, @@ -36,7 +36,10 @@ export default function routes() { index element={ - + @@ -47,7 +50,10 @@ export default function routes() { index element={ - + @@ -63,6 +69,7 @@ export default function routes() { OrganizationUserRole.owner, OrganizationUserRole.admin, ]} + redirectRoute={ACCOUNT_ROUTES.ACCOUNT_SETTINGS} > @@ -78,6 +85,7 @@ export default function routes() { OrganizationUserRole.owner, OrganizationUserRole.admin, ]} + redirectRoute={ACCOUNT_ROUTES.ACCOUNT_SETTINGS} > - +
Organization members view to be implemented
 @@ -124,6 +135,7 @@ export default function routes() { OrganizationUserRole.admin, ]} mmoOnly + redirectRoute={ACCOUNT_ROUTES.ACCOUNT_SETTINGS} >
Organization settings view to be implemented
 diff --git a/jsapp/js/account/organizations/validateOrgPermissions.component.tsx b/jsapp/js/router/validateOrgPermissions.component.tsx similarity index 74% rename from jsapp/js/account/organizations/validateOrgPermissions.component.tsx rename to jsapp/js/router/validateOrgPermissions.component.tsx index 8d2e83792e..a35ba98f1b 100644 --- a/jsapp/js/account/organizations/validateOrgPermissions.component.tsx +++ b/jsapp/js/router/validateOrgPermissions.component.tsx @@ -1,15 +1,14 @@ import React, {Suspense, useEffect} from 'react'; import {useNavigate} from 'react-router-dom'; import LoadingSpinner from 'js/components/common/loadingSpinner'; -import {ACCOUNT_ROUTES} from 'js/account/routes.constants'; import {useOrganizationQuery} from 'js/account/stripe.api'; -import {OrganizationUserRole} from '../stripe.types'; +import {OrganizationUserRole} from '../account/stripe.types'; interface Props { children: React.ReactNode; + redirectRoute: string; validRoles?: OrganizationUserRole[]; mmoOnly?: boolean; - redirect?: boolean; } /** @@ -19,9 +18,9 @@ interface Props { */ export const ValidateOrgPermissions = ({ children, + redirectRoute, validRoles = undefined, mmoOnly = false, - redirect = true, }: Props) => { const navigate = useNavigate(); const orgQuery = useOrganizationQuery(); @@ -30,18 +29,16 @@ export const ValidateOrgPermissions = ({ ) : true; const hasValidOrg = mmoOnly ? orgQuery.data?.is_mmo : true; - // Redirect to Account Settings if conditions not met useEffect(() => { if ( - redirect && orgQuery.data && (!hasValidRole || !hasValidOrg) ) { - navigate(ACCOUNT_ROUTES.ACCOUNT_SETTINGS); + navigate(redirectRoute); } - }, [redirect, orgQuery.data, navigate]); + }, [redirectRoute, orgQuery.data, navigate]); - return redirect && hasValidRole && hasValidOrg ? ( + return hasValidRole && hasValidOrg ? ( {children} ) : ( From 48df83ac7ee4eb50665f55a36f9942e4017f4f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20L=C3=A9ger?= Date: Wed, 13 Nov 2024 12:04:24 -0500 Subject: [PATCH 3/8] refactor(organization): use autocomplete fields in Admin UI for organizations TASK-965 (#5254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### đŸ“Ŗ Summary Switch to autocomplete fields for usernames instead of input fields for user IDs. ### 📖 Description Replaced input fields requiring user IDs with autocomplete fields for usernames. This update simplifies the process of adding members to an organization by allowing users to search and select members by their usernames, rather than manually finding and entering user IDs or OrganizationUser relationship IDs. --- hub/admin/extend_user.py | 40 ++++- hub/static/admin/css/inline_as_fieldset.css | 3 + kobo/apps/kobo_auth/models.py | 2 +- .../apps/viewer/models/parsed_instance.py | 2 +- kobo/apps/organizations/admin.py | 64 ------- kobo/apps/organizations/admin/__init__.py | 7 + kobo/apps/organizations/admin/organization.py | 93 +++++++++++ .../admin/organization_invite.py | 7 + .../organizations/admin/organization_owner.py | 24 +++ .../organizations/admin/organization_user.py | 153 +++++++++++++++++ kobo/apps/organizations/forms.py | 23 +++ kobo/apps/organizations/models.py | 4 + kobo/apps/organizations/tasks.py | 30 ++++ kobo/apps/organizations/utils.py | 24 ++- .../0003_create_proxy_and_add_invite_type.py | 53 ++++++ kobo/apps/project_ownership/models/invite.py | 156 +++++++++++++++++- .../apps/project_ownership/models/transfer.py | 29 +++- .../project_ownership/serializers/invite.py | 148 +---------------- kobo/apps/project_ownership/utils.py | 77 ++++++++- kobo/settings/base.py | 3 + 20 files changed, 719 insertions(+), 223 deletions(-) create mode 100644 hub/static/admin/css/inline_as_fieldset.css delete mode 100644 kobo/apps/organizations/admin.py create mode 100644 kobo/apps/organizations/admin/__init__.py create mode 100644 kobo/apps/organizations/admin/organization.py create mode 100644 kobo/apps/organizations/admin/organization_invite.py create mode 100644 kobo/apps/organizations/admin/organization_owner.py create mode 100644 kobo/apps/organizations/admin/organization_user.py create mode 100644 kobo/apps/organizations/forms.py create mode 100644 kobo/apps/organizations/tasks.py create mode 100644 kobo/apps/project_ownership/migrations/0003_create_proxy_and_add_invite_type.py diff --git a/hub/admin/extend_user.py b/hub/admin/extend_user.py index 47116f5ede..e18e80493f 100644 --- a/hub/admin/extend_user.py +++ b/hub/admin/extend_user.py @@ -87,8 +87,10 @@ class OrgInline(admin.StackedInline): 'organization', 'is_admin', ] + can_delete = False + # Override H2 style to make inline section like other fieldsets + classes = ('no-upper',) raw_id_fields = ('user', 'organization') - readonly_fields = settings.STRIPE_ENABLED and ('active_subscription_status',) or [] def active_subscription_status(self, obj): if settings.STRIPE_ENABLED: @@ -98,6 +100,12 @@ def active_subscription_status(self, obj): else 'None' ) + def get_readonly_fields(self, request, obj=None): + readonly_fields = ['organization', 'is_admin'] + if settings.STRIPE_ENABLED: + readonly_fields.append('active_subscription_status') + return readonly_fields + def has_add_permission(self, request, obj=OrganizationUser): return False @@ -158,6 +166,11 @@ class ExtendedUserAdmin(AdvancedSearchMixin, UserAdmin): ) actions = ['remove', 'delete'] + class Media: + css = { + 'all': ('admin/css/inline_as_fieldset.css',) + } + @admin.action(description='Remove selected users (delete everything but their username)') def remove(self, request, queryset, **kwargs): """ @@ -235,8 +248,9 @@ def get_queryset(self, request): ) def get_search_results(self, request, queryset, search_term): - if request.path != '/admin/auth/user/': + queryset = self._filter_queryset(request, queryset) + # If search comes from autocomplete field, use parent class method return super(UserAdmin, self).get_search_results( request, queryset, search_term @@ -261,6 +275,28 @@ def monthly_submission_count(self, obj): ).aggregate(counter=Sum('counter')) return instances.get('counter') + def _filter_queryset(self, request, queryset): + auto_complete = request.path == '/admin/autocomplete/' + app_label = request.GET.get('app_label') + model_name = request.GET.get('model_name') + + if ( + auto_complete + and app_label == 'organizations' + and model_name == 'organizationuser' + ): + return self._filter_queryset_for_organization_user(queryset) + + return queryset + + def _filter_queryset_for_organization_user(self, queryset): + """ + Displays only users whose organization has a single member. + """ + return queryset.annotate( + user_count=Count('organizations_organization__organization_users') + ).filter(user_count__lte=1).order_by('username') + def _remove_or_delete( self, request, diff --git a/hub/static/admin/css/inline_as_fieldset.css b/hub/static/admin/css/inline_as_fieldset.css new file mode 100644 index 0000000000..fc35ae7805 --- /dev/null +++ b/hub/static/admin/css/inline_as_fieldset.css @@ -0,0 +1,3 @@ +.no-upper h2 { + text-transform: unset; +} diff --git a/kobo/apps/kobo_auth/models.py b/kobo/apps/kobo_auth/models.py index e9798b4dde..f81891682d 100644 --- a/kobo/apps/kobo_auth/models.py +++ b/kobo/apps/kobo_auth/models.py @@ -59,7 +59,7 @@ def organization(self): # Database allows multiple organizations per user, but we restrict it to one. if organization := Organization.objects.filter( organization_users__user=self - ).first(): + ).order_by('-organization_users__created').first(): return organization try: diff --git a/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py b/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py index 21d3acc45a..dd230db9e7 100644 --- a/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py +++ b/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py @@ -8,7 +8,6 @@ from pymongo.errors import PyMongoError from kobo.apps.hook.utils.services import call_services -from kobo.celery import celery_app from kobo.apps.openrosa.apps.logger.models import Instance, Note from kobo.apps.openrosa.libs.utils.common_tags import ( ATTACHMENTS, @@ -24,6 +23,7 @@ ) from kobo.apps.openrosa.libs.utils.decorators import apply_form_field_names from kobo.apps.openrosa.libs.utils.model_tools import queryset_iterator +from kobo.celery import celery_app from kpi.utils.log import logging from kpi.utils.mongo_helper import MongoHelper diff --git a/kobo/apps/organizations/admin.py b/kobo/apps/organizations/admin.py deleted file mode 100644 index b288ad4e6a..0000000000 --- a/kobo/apps/organizations/admin.py +++ /dev/null @@ -1,64 +0,0 @@ -from django.contrib import admin -from django.contrib.auth import get_user_model -from import_export import resources -from import_export.admin import ImportExportModelAdmin -from import_export.fields import Field -from import_export.widgets import ForeignKeyWidget -from organizations.base_admin import ( - BaseOrganizationAdmin, - BaseOrganizationOwnerAdmin, - BaseOrganizationUserAdmin, - BaseOwnerInline, -) - -from .models import ( - Organization, - OrganizationInvitation, - OrganizationOwner, - OrganizationUser, -) - -User = get_user_model() - - -class OwnerInline(BaseOwnerInline): - model = OrganizationOwner - - -class OrgUserInline(admin.StackedInline): - model = OrganizationUser - raw_id_fields = ('user',) - view_on_site = False - extra = 0 - - -@admin.register(Organization) -class OrgAdmin(BaseOrganizationAdmin): - inlines = [OwnerInline, OrgUserInline] - readonly_fields = ['id'] - - -class OrgUserResource(resources.ModelResource): - user = Field( - attribute='user', - column_name='user', - widget=ForeignKeyWidget(User, field='username'), - ) - - class Meta: - model = OrganizationUser - - -@admin.register(OrganizationUser) -class OrgUserAdmin(ImportExportModelAdmin, BaseOrganizationUserAdmin): - resource_classes = [OrgUserResource] - - -@admin.register(OrganizationOwner) -class OrgOwnerAdmin(BaseOrganizationOwnerAdmin): - pass - - -@admin.register(OrganizationInvitation) -class OrgInvitationAdmin(admin.ModelAdmin): - pass diff --git a/kobo/apps/organizations/admin/__init__.py b/kobo/apps/organizations/admin/__init__.py new file mode 100644 index 0000000000..debd1e39d5 --- /dev/null +++ b/kobo/apps/organizations/admin/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa: F401 +from .organization import OrgAdmin +from .organization_owner import OrgOwnerAdmin +from .organization_invite import OrgInvitationAdmin +from .organization_user import OrgUserAdmin + +__all__ = ['OrgAdmin', 'OrgOwnerAdmin', 'OrgInvitationAdmin', 'OrgUserAdmin'] diff --git a/kobo/apps/organizations/admin/organization.py b/kobo/apps/organizations/admin/organization.py new file mode 100644 index 0000000000..f538d3e628 --- /dev/null +++ b/kobo/apps/organizations/admin/organization.py @@ -0,0 +1,93 @@ +from django.contrib import admin, messages +from django.db.models import Count +from django.utils.safestring import mark_safe +from organizations.base_admin import BaseOrganizationAdmin + +from kobo.apps.kobo_auth.shortcuts import User +from .organization_owner import OwnerInline +from .organization_user import OrgUserInline +from ..models import ( + Organization, + OrganizationUser, +) +from ..tasks import transfer_user_ownership_to_org +from ..utils import revoke_org_asset_perms + + +@admin.register(Organization) +class OrgAdmin(BaseOrganizationAdmin): + inlines = [OwnerInline, OrgUserInline] + view_on_site = False + readonly_fields = ['id'] + fields = ['id', 'name', 'slug', 'is_active', 'mmo_override'] + + def save_related(self, request, form, formsets, change): + super().save_related(request, form, formsets, change) + + for formset in formsets: + if formset.prefix == 'organization_users': + # retrieve all users + member_ids = formset.queryset.values('user_id') + organization_id = form.instance.id + new_members = self._get_new_members_queryset( + member_ids, organization_id + ) + self._transfer_user_ownership(request, new_members) + self._delete_previous_organizations(new_members, organization_id) + + deleted_user_ids = [] + for obj in formset.deleted_objects: + deleted_user_ids.append(obj.user_id) + + if deleted_user_ids: + revoke_org_asset_perms(form.instance, deleted_user_ids) + + def _delete_previous_organizations( + self, new_members: 'QuerySet', organization_id: int + ): + new_member_ids = (new_member['pk'] for new_member in new_members) + Organization.objects.filter( + organization_users__user_id__in=new_member_ids + ).exclude(pk=organization_id).delete() + + def _get_new_members_queryset( + self, member_ids: 'QuerySet', organization_id: int + ) -> 'QuerySet': + + users_in_multiple_orgs = ( + OrganizationUser.objects.values('user_id') + .annotate(org_count=Count('organization_id', distinct=True)) + .filter(org_count__gt=1, user_id__in=member_ids) + .values_list('user_id', flat=True) + ) + + queryset = ( + User.objects + .filter( + organizations_organizationuser__organization_id=organization_id + ) + .filter(id__in=users_in_multiple_orgs) + .values('pk', 'username') + ) + + return queryset + + def _transfer_user_ownership(self, request: 'HttpRequest', new_members: 'QuerySet'): + + if new_members.exists(): + + html_username_list = [] + for user in new_members: + html_username_list.append(f"{user['username']}") + transfer_user_ownership_to_org.delay(user['pk']) + + message = ( + 'The following new members have been added, and their project ' + 'transfers have started: ' + ) + ', '.join(html_username_list) + + self.message_user( + request, + mark_safe(message), + messages.INFO, + ) diff --git a/kobo/apps/organizations/admin/organization_invite.py b/kobo/apps/organizations/admin/organization_invite.py new file mode 100644 index 0000000000..c63f521576 --- /dev/null +++ b/kobo/apps/organizations/admin/organization_invite.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from ..models import OrganizationInvitation + + +@admin.register(OrganizationInvitation) +class OrgInvitationAdmin(admin.ModelAdmin): + pass diff --git a/kobo/apps/organizations/admin/organization_owner.py b/kobo/apps/organizations/admin/organization_owner.py new file mode 100644 index 0000000000..317c7e11df --- /dev/null +++ b/kobo/apps/organizations/admin/organization_owner.py @@ -0,0 +1,24 @@ +from django.contrib import admin +from organizations.base_admin import BaseOrganizationOwnerAdmin, BaseOwnerInline + +from ..models import OrganizationOwner + + +class OwnerInline(BaseOwnerInline): + model = OrganizationOwner + autocomplete_fields = ['organization_user'] + + can_delete = False + + def get_readonly_fields(self, request, obj=None): + """ + Hook for specifying custom readonly fields. + """ + if obj is not None and obj.pk: + return ['organization_user'] + return [] + + +@admin.register(OrganizationOwner) +class OrgOwnerAdmin(BaseOrganizationOwnerAdmin): + autocomplete_fields = ['organization_user', 'organization'] diff --git a/kobo/apps/organizations/admin/organization_user.py b/kobo/apps/organizations/admin/organization_user.py new file mode 100644 index 0000000000..b11832478c --- /dev/null +++ b/kobo/apps/organizations/admin/organization_user.py @@ -0,0 +1,153 @@ +from django import forms +from django.conf import settings +from django.contrib import admin, messages +from django.db.models import Count +from django.utils.safestring import mark_safe +from import_export import resources +from import_export.admin import ImportExportModelAdmin +from import_export.fields import Field +from import_export.widgets import ForeignKeyWidget +from organizations.base_admin import BaseOrganizationUserAdmin + +from kobo.apps.kobo_auth.shortcuts import User +from ..forms import OrgUserAdminForm +from ..models import OrganizationUser +from ..tasks import transfer_user_ownership_to_org +from ..utils import revoke_org_asset_perms + + +def _max_users_for_edit_mode(): + """ + This function represents an arbitrary limit + to prevent the form's POST request from exceeding + `settings.DATA_UPLOAD_MAX_NUMBER_FIELDS`. + """ + return settings.DATA_UPLOAD_MAX_NUMBER_FIELDS // 3 + + +class OrgUserInlineFormSet(forms.models.BaseInlineFormSet): + def clean(self): + if self.is_valid(): + members = 0 + users = [] + if len(self.forms) >= _max_users_for_edit_mode(): + return + + for form in self.forms: + if form.cleaned_data: + members += 1 + users.append(form.cleaned_data['user'].pk) + + if not self.instance.is_mmo and members > 0: + raise forms.ValidationError( + 'Users cannot be added to an organization that is not multi-member' + ) + + if members > 0: + queryset = OrganizationUser.objects.filter(user_id__in=users) + if self.instance.pk: + queryset = queryset.exclude(organization_id=self.instance.pk) + + queryset = ( + queryset.values('user_id') + .annotate(org_count=Count('organization_id', distinct=True)) + .filter(org_count__gt=1) + ) + + if queryset.exists(): + raise forms.ValidationError( + 'You cannot add users who are already members of another ' + 'multi-member organization.' + ) + + +class OrgUserInline(admin.StackedInline): + model = OrganizationUser + formset = OrgUserInlineFormSet + raw_id_fields = ('user',) + view_on_site = False + extra = 0 + fields = ['user', 'is_admin'] + autocomplete_fields = ['user'] + + def get_queryset(self, request): + queryset = super().get_queryset(request) + if queryset: + queryset = queryset.filter(organizationowner__isnull=True) + return queryset + + def get_readonly_fields(self, request, obj=None): + """ + Hook for specifying custom readonly fields. + """ + if not obj: + return [] + + if obj.organization_users.count() >= _max_users_for_edit_mode(): + return ['user', 'is_admin'] + + return [] + + def has_add_permission(self, request, obj=None): + if not obj: + return True + + return obj.organization_users.count() < _max_users_for_edit_mode() + + def has_delete_permission(self, request, obj=None): + if not obj: + return True + + return obj.organization_users.count() < _max_users_for_edit_mode() + + +class OrgUserResource(resources.ModelResource): + user = Field( + attribute='user', + column_name='user', + widget=ForeignKeyWidget(User, field='username'), + ) + + class Meta: + model = OrganizationUser + + +@admin.register(OrganizationUser) +class OrgUserAdmin(ImportExportModelAdmin, BaseOrganizationUserAdmin): + resource_classes = [OrgUserResource] + search_fields = ('user__username',) + autocomplete_fields = ['user', 'organization'] + form = OrgUserAdminForm + + def get_search_results(self, request, queryset, search_term): + auto_complete = request.path == '/admin/autocomplete/' + app_label = request.GET.get('app_label') + model_name = request.GET.get('model_name') + if ( + auto_complete + and app_label == 'organizations' + and model_name == 'organizationowner' + ): + queryset = queryset.annotate( + user_count=Count('organization__organization_users') + ).filter(user_count__lte=1).order_by('user__username') + + return super().get_search_results(request, queryset, search_term) + + def save_model(self, request, obj, form, change): + previous_organization = form.cleaned_data.get('previous_organization') + super().save_model(request, obj, form, change) + if previous_organization: + transfer_user_ownership_to_org.delay(obj.user.pk) + message = ( + f'User {obj.user.username} has been added to ' + f'{obj.organization.name}, and their project transfers have ' + f'started' + ) + + self.message_user( + request, + mark_safe(message), + messages.INFO, + ) + revoke_org_asset_perms(previous_organization, [obj.user.pk]) diff --git a/kobo/apps/organizations/forms.py b/kobo/apps/organizations/forms.py new file mode 100644 index 0000000000..bf45473c91 --- /dev/null +++ b/kobo/apps/organizations/forms.py @@ -0,0 +1,23 @@ +from django import forms +from django.core.exceptions import ValidationError +from .models import OrganizationUser + + +class OrgUserAdminForm(forms.ModelForm): + class Meta: + model = OrganizationUser + fields = '__all__' + + def clean(self): + cleaned_data = super().clean() + organization = cleaned_data.get('organization') + user = cleaned_data.get('user') + if not organization.is_owner(user) and not organization.is_mmo: + raise ValidationError( + 'Users cannot be added to an organization that is not multi-member' + ) + + cleaned_data['previous_organization'] = ( + user.organization if user.organization != organization else None + ) + return cleaned_data diff --git a/kobo/apps/organizations/models.py b/kobo/apps/organizations/models.py index d03c01bb3b..9aac437b01 100644 --- a/kobo/apps/organizations/models.py +++ b/kobo/apps/organizations/models.py @@ -165,6 +165,10 @@ def owner_user_object(self) -> 'User': class OrganizationUser(AbstractOrganizationUser): + + def __str__(self): + return f'' + @property def active_subscription_statuses(self): """ diff --git a/kobo/apps/organizations/tasks.py b/kobo/apps/organizations/tasks.py new file mode 100644 index 0000000000..c3effa3b33 --- /dev/null +++ b/kobo/apps/organizations/tasks.py @@ -0,0 +1,30 @@ +from django.conf import settings +from more_itertools import chunked + +from kobo.apps.kobo_auth.shortcuts import User +from kobo.celery import celery_app +from kobo.apps.project_ownership.utils import create_invite + + +@celery_app.task( + queue='kpi_low_priority_queue', + soft_time_limit=settings.CELERY_LONG_RUNNING_TASK_SOFT_TIME_LIMIT, + time_limit=settings.CELERY_LONG_RUNNING_TASK_TIME_LIMIT, +) +def transfer_user_ownership_to_org(user_id: int): + sender = User.objects.get(pk=user_id) + recipient = sender.organization.owner_user_object + user_assets = sender.assets.only('pk', 'uid').iterator() + + # Splitting assets into batches to avoid creating a long-running + # Celery task that could exceed Celery's soft time limit or consume + # too much RAM, potentially leading to termination by Kubernetes. + for asset_batch in chunked( + user_assets, settings.USER_ASSET_ORG_TRANSFER_BATCH_SIZE + ): + create_invite( + sender=sender, + recipient=recipient, + assets=asset_batch, + invite_class_name='OrgMembershipAutoInvite', + ) diff --git a/kobo/apps/organizations/utils.py b/kobo/apps/organizations/utils.py index 42fa00f56d..4b5cc18503 100644 --- a/kobo/apps/organizations/utils.py +++ b/kobo/apps/organizations/utils.py @@ -1,16 +1,13 @@ -from typing import Union - -try: - from zoneinfo import ZoneInfo -except ImportError: - from backports.zoneinfo import ZoneInfo - from datetime import datetime +from typing import Union +from zoneinfo import ZoneInfo from dateutil.relativedelta import relativedelta from django.utils import timezone from kobo.apps.organizations.models import Organization +from kpi.models.asset import Asset +from kpi.models.object_permission import ObjectPermission def get_monthly_billing_dates(organization: Union[Organization, None]): @@ -101,3 +98,16 @@ def get_yearly_billing_dates(organization: Union[Organization, None]): anchor_date += relativedelta(years=1) period_end = period_start + relativedelta(years=1) return period_start, period_end + + +def revoke_org_asset_perms(organization: Organization, user_ids: list[int]): + """ + Revokes permissions assigned to removed members on all assets belonging to + the organization. + """ + subquery = Asset.objects.values_list('pk', flat=True).filter( + owner=organization.owner_user_object + ) + ObjectPermission.objects.filter( + asset_id__in=subquery, user_id__in=user_ids + ).delete() diff --git a/kobo/apps/project_ownership/migrations/0003_create_proxy_and_add_invite_type.py b/kobo/apps/project_ownership/migrations/0003_create_proxy_and_add_invite_type.py new file mode 100644 index 0000000000..b6a44db49e --- /dev/null +++ b/kobo/apps/project_ownership/migrations/0003_create_proxy_and_add_invite_type.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.15 on 2024-11-08 16:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + 'project_ownership', + '0002_alter_invite_date_created_alter_invite_date_modified_and_more', + ), + ] + + operations = [ + migrations.CreateModel( + name='OrgMembershipAutoInvite', + fields=[], + options={ + 'verbose_name': 'auto-invite for user project transfer to organization', + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('project_ownership.invite',), + ), + migrations.AddField( + model_name='invite', + name='invite_type', + field=models.CharField( + choices=[ + ('org-membership', 'Org Membership'), + ('org-ownership-transfer', 'Org Ownership Transfer'), + ('user-ownership-transfer', 'User Ownership Transfer'), + ], + default='user-ownership-transfer', + max_length=30, + ), + ), + migrations.AddField( + model_name='transfer', + name='invite_type', + field=models.CharField( + choices=[ + ('org-membership', 'Org Membership'), + ('org-ownership-transfer', 'Org Ownership Transfer'), + ('user-ownership-transfer', 'User Ownership Transfer'), + ], + default='user-ownership-transfer', + max_length=30, + ), + ), + ] diff --git a/kobo/apps/project_ownership/models/invite.py b/kobo/apps/project_ownership/models/invite.py index 593eb7955d..fdf0a91a4a 100644 --- a/kobo/apps/project_ownership/models/invite.py +++ b/kobo/apps/project_ownership/models/invite.py @@ -1,14 +1,39 @@ from __future__ import annotations +from constance import config from django.conf import settings from django.db import models -from django.utils import timezone +from django.utils.translation import gettext as t from kpi.fields import KpiUidField from kpi.models.abstract_models import AbstractTimeStampedModel +from kpi.utils.mailer import EmailMessage, Mailer from .choices import InviteStatusChoices +class InviteType(models.TextChoices): + ORG_MEMBERSHIP = 'org-membership' + ORG_OWNERSHIP_TRANSFER = 'org-ownership-transfer' + USER_OWNERSHIP_TRANSFER = 'user-ownership-transfer' + + +class InviteAllManager(models.Manager): + pass + + +class InviteManager(models.Manager): + + def create(self, **kwargs): + return super().create(invite_type=InviteType.USER_OWNERSHIP_TRANSFER, **kwargs) + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter(invite_type=InviteType.USER_OWNERSHIP_TRANSFER) + ) + + class Invite(AbstractTimeStampedModel): uid = KpiUidField(uid_prefix='poi') @@ -28,6 +53,13 @@ class Invite(AbstractTimeStampedModel): default=InviteStatusChoices.PENDING, db_index=True ) + invite_type = models.CharField( + choices=InviteType.choices, + default=InviteType.USER_OWNERSHIP_TRANSFER, + max_length=30, + ) + objects = InviteManager() + all_objects = InviteAllManager() class Meta: verbose_name = 'project ownership transfer invite' @@ -38,3 +70,125 @@ def __str__(self): f'{self.recipient.username} ' f'({InviteStatusChoices(self.status)})' ) + + @property + def auto_accept_invites(self): + return config.PROJECT_OWNERSHIP_AUTO_ACCEPT_INVITES + + def send_acceptance_email(self): + + template_variables = { + 'username': self.sender.username, + 'recipient': self.recipient.username, + 'transfers': [ + { + 'asset_uid': transfer.asset.uid, + 'asset_name': transfer.asset.name, + } + for transfer in self.transfers.all() + ], + 'base_url': settings.KOBOFORM_URL, + } + + email_message = EmailMessage( + to=self.sender.email, + subject=t('KoboToolbox project ownership transfer accepted'), + plain_text_content_or_template='emails/accepted_invite.txt', + template_variables=template_variables, + html_content_or_template='emails/accepted_invite.html', + language=self.recipient.extra_details.data.get('last_ui_language') + ) + + Mailer.send(email_message) + + def send_invite_email(self): + + template_variables = { + 'username': self.recipient.username, + 'sender_username': self.sender.username, + 'sender_email': self.sender.email, + 'transfers': [ + { + 'asset_uid': transfer.asset.uid, + 'asset_name': transfer.asset.name, + } + for transfer in self.transfers.all() + ], + 'base_url': settings.KOBOFORM_URL, + 'invite_expiry': config.PROJECT_OWNERSHIP_INVITE_EXPIRY, + 'invite_uid': self.uid, + } + + email_message = EmailMessage( + to=self.recipient.email, + subject=t( + 'Action required: KoboToolbox project ownership transfer request' + ), + plain_text_content_or_template='emails/new_invite.txt', + template_variables=template_variables, + html_content_or_template='emails/new_invite.html', + language=self.recipient.extra_details.data.get('last_ui_language') + ) + + Mailer.send(email_message) + + def send_refusal_email(self): + + template_variables = { + 'username': self.sender.username, + 'recipient': self.recipient.username, + 'transfers': [ + { + 'asset_uid': transfer.asset.uid, + 'asset_name': transfer.asset.name, + } + for transfer in self.transfers.all() + ], + 'base_url': settings.KOBOFORM_URL, + } + + email_message = EmailMessage( + to=self.sender.email, + subject=t('KoboToolbox project ownership transfer incomplete'), + plain_text_content_or_template='emails/declined_invite.txt', + template_variables=template_variables, + html_content_or_template='emails/declined_invite.html', + language=self.recipient.extra_details.data.get('last_ui_language') + ) + + Mailer.send(email_message) + + +class OrgMembershipAutoInviteManager(models.Manager): + + def create(self, **kwargs): + return super().create(invite_type=InviteType.ORG_MEMBERSHIP, **kwargs) + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter(invite_type=InviteType.ORG_MEMBERSHIP) + ) + + +class OrgMembershipAutoInvite(Invite): + + class Meta: + proxy = True + verbose_name = 'auto-invite for user project transfer to organization' + + objects = OrgMembershipAutoInviteManager() + + @property + def auto_accept_invites(self): + return True + + def send_acceptance_email(self): + pass + + def send_invite_email(self): + pass + + def send_refusal_email(self): + pass diff --git a/kobo/apps/project_ownership/models/transfer.py b/kobo/apps/project_ownership/models/transfer.py index fd3505734c..4ab95dfa95 100644 --- a/kobo/apps/project_ownership/models/transfer.py +++ b/kobo/apps/project_ownership/models/transfer.py @@ -27,7 +27,7 @@ TransferStatusChoices, TransferStatusTypeChoices, ) -from .invite import Invite +from .invite import Invite, InviteType, OrgMembershipAutoInvite class Transfer(AbstractTimeStampedModel): @@ -43,6 +43,16 @@ class Transfer(AbstractTimeStampedModel): related_name='transfers', on_delete=models.CASCADE, ) + # The `invite_type` field is **always** copied from the `Invite` model during the + # save process to eliminate the need to load the related `Invite` model solely to + # determine its type. + # This optimization allows directly resolving the appropriate model with + # `get_invite_model()`. + invite_type = models.CharField( + choices=InviteType.choices, + default=InviteType.USER_OWNERSHIP_TRANSFER, + max_length=30, + ) class Meta: verbose_name = 'project ownership transfer' @@ -54,9 +64,18 @@ def __str__(self) -> str: f'{self.invite.recipient.username}' ) + def get_invite_model(self) -> Invite | OrgMembershipAutoInvite: + if self.invite_type == InviteType.USER_OWNERSHIP_TRANSFER: + return Invite + if self.invite_type == InviteType.ORG_MEMBERSHIP: + return OrgMembershipAutoInvite + + raise NotImplementedError + def save(self, *args, **kwargs): is_new = self.pk is None + self.invite_type = self.invite.invite_type super().save(*args, **kwargs) if is_new: @@ -269,8 +288,8 @@ def _update_invite_status(self): This method must be called within a transaction because of the lock acquired the object row (with `select_for_update`) """ - invite = self.invite.__class__.objects.select_for_update().get( - pk=self.invite.pk + invite = self.get_invite_model().objects.select_for_update().get( + pk=self.invite_id ) previous_status = invite.status is_complete = True @@ -296,8 +315,8 @@ def _update_invite_status(self): if invite.status == InviteStatusChoices.FAILED: send_email_to_admins.delay(invite.uid) - if previous_status != invite.status: - self.invite.refresh_from_db() + self.invite.status = invite.status + self.invite.date_modified = invite.date_modified class TransferStatus(AbstractTimeStampedModel): diff --git a/kobo/apps/project_ownership/serializers/invite.py b/kobo/apps/project_ownership/serializers/invite.py index 785f68c0f8..769f964250 100644 --- a/kobo/apps/project_ownership/serializers/invite.py +++ b/kobo/apps/project_ownership/serializers/invite.py @@ -1,9 +1,6 @@ from __future__ import annotations -from constance import config -from django.conf import settings from django.contrib.auth import get_user_model -from django.db import transaction from django.db.models import Max, Prefetch from django.utils.translation import gettext as t from rest_framework import exceptions, serializers @@ -11,16 +8,15 @@ from kpi.fields import RelativePrefixHyperlinkedRelatedField from kpi.models import Asset -from kpi.utils.mailer import EmailMessage, Mailer from .transfer import TransferListSerializer from ..models import ( Invite, InviteStatusChoices, Transfer, TransferStatus, - TransferStatusChoices, TransferStatusTypeChoices, ) +from ..utils import create_invite, update_invite class InviteSerializer(serializers.ModelSerializer): @@ -59,36 +55,12 @@ class Meta: def create(self, validated_data: dict) -> Invite: request = self.context['request'] - with transaction.atomic(): - instance = Invite.objects.create( - sender=request.user, - recipient=validated_data['recipient'] - ) - transfers = Transfer.objects.bulk_create( - [ - Transfer(invite=instance, asset=asset) - for asset in validated_data['assets'] - ] - ) - statuses = [] - for transfer in transfers: - for status_type in TransferStatusTypeChoices.values: - statuses.append( - TransferStatus( - transfer=transfer, - status_type=status_type, - ) - ) - TransferStatus.objects.bulk_create(statuses) - - if config.PROJECT_OWNERSHIP_AUTO_ACCEPT_INVITES: - instance = self.update( - instance, {'status': InviteStatusChoices.ACCEPTED} - ) - else: - self._send_invite_email(instance) - - return instance + return create_invite( + sender=request.user, + recipient=validated_data['recipient'], + assets=validated_data['assets'], + invite_class_name='Invite', + ) def get_transfers(self, invite: Invite) -> list: transfers = ( @@ -220,108 +192,4 @@ def validate_status(self, status: str) -> str: def update(self, instance: Invite, validated_data: dict) -> Invite: status = validated_data['status'] - - # Keep `status` value to email condition below - instance.status = ( - InviteStatusChoices.IN_PROGRESS - if status == InviteStatusChoices.ACCEPTED - else status - ) - instance.save(update_fields=['status', 'date_modified']) - - for transfer in instance.transfers.all(): - if instance.status != InviteStatusChoices.IN_PROGRESS: - transfer.statuses.update( - status=TransferStatusChoices.CANCELLED - ) - else: - transfer.transfer_project() - - if not config.PROJECT_OWNERSHIP_AUTO_ACCEPT_INVITES: - if status == InviteStatusChoices.DECLINED: - self._send_refusal_email(instance) - elif status == InviteStatusChoices.ACCEPTED: - self._send_acceptance_email(instance) - - return instance - - def _send_acceptance_email(self, invite: Invite): - - template_variables = { - 'username': invite.sender.username, - 'recipient': invite.recipient.username, - 'transfers': [ - { - 'asset_uid': transfer.asset.uid, - 'asset_name': transfer.asset.name, - } - for transfer in invite.transfers.all() - ], - 'base_url': settings.KOBOFORM_URL, - } - - email_message = EmailMessage( - to=invite.sender.email, - subject=t('KoboToolbox project ownership transfer accepted'), - plain_text_content_or_template='emails/accepted_invite.txt', - template_variables=template_variables, - html_content_or_template='emails/accepted_invite.html', - language=invite.recipient.extra_details.data.get('last_ui_language') - ) - - Mailer.send(email_message) - - def _send_invite_email(self, invite: Invite): - - template_variables = { - 'username': invite.recipient.username, - 'sender_username': invite.sender.username, - 'sender_email': invite.sender.email, - 'transfers': [ - { - 'asset_uid': transfer.asset.uid, - 'asset_name': transfer.asset.name, - } - for transfer in invite.transfers.all() - ], - 'base_url': settings.KOBOFORM_URL, - 'invite_expiry': config.PROJECT_OWNERSHIP_INVITE_EXPIRY, - 'invite_uid': invite.uid, - } - - email_message = EmailMessage( - to=invite.recipient.email, - subject=t('Action required: KoboToolbox project ownership transfer request'), - plain_text_content_or_template='emails/new_invite.txt', - template_variables=template_variables, - html_content_or_template='emails/new_invite.html', - language=invite.recipient.extra_details.data.get('last_ui_language') - ) - - Mailer.send(email_message) - - def _send_refusal_email(self, invite: Invite): - - template_variables = { - 'username': invite.sender.username, - 'recipient': invite.recipient.username, - 'transfers': [ - { - 'asset_uid': transfer.asset.uid, - 'asset_name': transfer.asset.name, - } - for transfer in invite.transfers.all() - ], - 'base_url': settings.KOBOFORM_URL, - } - - email_message = EmailMessage( - to=invite.sender.email, - subject=t('KoboToolbox project ownership transfer incomplete'), - plain_text_content_or_template='emails/declined_invite.txt', - template_variables=template_variables, - html_content_or_template='emails/declined_invite.html', - language=invite.recipient.extra_details.data.get('last_ui_language') - ) - - Mailer.send(email_message) + return update_invite(invite=instance, status=status) diff --git a/kobo/apps/project_ownership/utils.py b/kobo/apps/project_ownership/utils.py index 253dd97fd0..dd4f1bcd7d 100644 --- a/kobo/apps/project_ownership/utils.py +++ b/kobo/apps/project_ownership/utils.py @@ -1,19 +1,62 @@ import os import time -from typing import Optional +from typing import Literal, Optional, Union +from django.db import transaction from django.apps import apps from django.utils import timezone from kobo.apps.openrosa.apps.main.models import MetaData from kobo.apps.openrosa.apps.logger.models.attachment import Attachment -from kpi.models.asset import AssetFile +from kobo.apps.project_ownership.models import InviteStatusChoices +from kpi.models.asset import Asset, AssetFile from .exceptions import AsyncTaskException from .constants import ASYNC_TASK_HEARTBEAT, FILE_MOVE_CHUNK_SIZE from .models.choices import TransferStatusChoices, TransferStatusTypeChoices +def create_invite( + sender: 'User', + recipient: 'User', + assets: list[Asset], + invite_class_name: str = Literal['Invite', 'OrgMembershipAutoInvite'], +) -> Union['Invite', 'OrgMembershipAutoInvite']: + + InviteModel = apps.get_model('project_ownership', invite_class_name) + Transfer = apps.get_model('project_ownership', 'Transfer') + TransferStatus = apps.get_model('project_ownership', 'TransferStatus') + + with transaction.atomic(): + invite = InviteModel.objects.create( + sender=sender, + recipient=recipient + ) + transfers = Transfer.objects.bulk_create( + [ + Transfer(invite=invite, asset=asset) + for asset in assets + ] + ) + statuses = [] + for transfer in transfers: + for status_type in TransferStatusTypeChoices.values: + statuses.append( + TransferStatus( + transfer=transfer, + status_type=status_type, + ) + ) + TransferStatus.objects.bulk_create(statuses) + + if invite.auto_accept_invites: + update_invite(invite, status=InviteStatusChoices.ACCEPTED) + else: + invite.send_invite_email() + + return invite + + def get_target_folder( previous_owner_username: str, new_owner_username: str, filename: str ) -> Optional[str]: @@ -186,6 +229,36 @@ def rewrite_mongo_userform_id(transfer: 'project_ownership.Transfer'): ) +def update_invite( + invite: Union['Invite', 'OrgMembershipAutoInvite'], + status: str, +) -> Union['Invite', 'OrgMembershipAutoInvite']: + + # Keep `status` value to email condition below + invite.status = ( + InviteStatusChoices.IN_PROGRESS + if status == InviteStatusChoices.ACCEPTED + else status + ) + invite.save(update_fields=['status', 'date_modified']) + + for transfer in invite.transfers.all(): + if invite.status != InviteStatusChoices.IN_PROGRESS: + transfer.statuses.update( + status=TransferStatusChoices.CANCELLED + ) + else: + transfer.transfer_project() + + if not invite.auto_accept_invites: + if status == InviteStatusChoices.DECLINED: + invite.send_refusal_email() + elif status == InviteStatusChoices.ACCEPTED: + invite.send_acceptance_email() + + return invite + + def _mark_task_as_successful( transfer: 'project_ownership.Transfer', async_task_type: str ): diff --git a/kobo/settings/base.py b/kobo/settings/base.py index 0e632527bc..648c6e1b09 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -1583,6 +1583,7 @@ def dj_stripe_request_callback_method(): MONGO_QUERY_TIMEOUT = SYNCHRONOUS_REQUEST_TIME_LIMIT + 5 # seconds MONGO_CELERY_QUERY_TIMEOUT = CELERY_TASK_TIME_LIMIT + 10 # seconds + SESSION_ENGINE = 'redis_sessions.session' # django-redis-session expects a dictionary with `url` redis_session_url = env.cache_url( @@ -1799,3 +1800,5 @@ def dj_stripe_request_callback_method(): # in the ObjectPermission table), but the code will still conduct the permission # checks as if they were. ADMIN_ORG_INHERITED_PERMS = [PERM_DELETE_ASSET, PERM_MANAGE_ASSET] + +USER_ASSET_ORG_TRANSFER_BATCH_SIZE = 20 From c47f297c1c1b86d2d691e294da15733e5d331205 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 13 Nov 2024 15:19:41 -0500 Subject: [PATCH 4/8] feat(projectHistoryLog): log exports TASK-944 (#5241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### đŸ“Ŗ Summary Record project history logs for when users export an asset. ### 👀 Preview steps Bug template: 1. ℹī¸ Log in as a super user. Create and deploy a project 2. Add at least 1 submission to the project 3. Go to Data > Downloads and hit Export. Note: the export may get stuck in processing. This is due to an unrelated bug (https://www.notion.so/kobotoolbox/Exports-fail-with-ExportTask-matching-query-does-not-exist-1377e515f65480328422e0c8fcfed1c0) but doesn't affect this work since we want to log every attempt to export, not just successful ones 5. đŸŸĸ Go to `api/v2/audit-logs/?q=metadata__asset_uid: AND log_type='project-history'` 6. There should be a new PH log with action=`export` (usual metadata) ### 💭 Notes Uses the AuditLoggedViewSet and `create_from_related_request` flows (similar to asset files, paired data, etc.) --- kobo/apps/audit_log/audit_actions.py | 1 + kobo/apps/audit_log/base_views.py | 2 +- kobo/apps/audit_log/models.py | 35 ++++++++++-- .../tests/test_project_history_logs.py | 55 +++++++++++++++++++ kpi/models/import_export_task.py | 18 +++--- kpi/views/v1/export_task.py | 6 +- kpi/views/v2/export_task.py | 6 +- 7 files changed, 107 insertions(+), 16 deletions(-) diff --git a/kobo/apps/audit_log/audit_actions.py b/kobo/apps/audit_log/audit_actions.py index a85b3ef563..d8a29c4c33 100644 --- a/kobo/apps/audit_log/audit_actions.py +++ b/kobo/apps/audit_log/audit_actions.py @@ -14,6 +14,7 @@ class AuditAction(models.TextChoices): DISABLE_SHARING = 'disable-sharing' DISCONNECT_PROJECT = 'disconnect-project' ENABLE_SHARING = 'enable-sharing' + EXPORT = 'export' IN_TRASH = 'in-trash' MODIFY_IMPORTED_FIELDS = 'modify-imported-fields' MODIFY_SERVICE = 'modify-service' diff --git a/kobo/apps/audit_log/base_views.py b/kobo/apps/audit_log/base_views.py index 1632772768..f697f267ca 100644 --- a/kobo/apps/audit_log/base_views.py +++ b/kobo/apps/audit_log/base_views.py @@ -83,8 +83,8 @@ def perform_destroy(self, instance): field_label = field[0] if isinstance(field, tuple) else field value = get_nested_field(instance, field_path) audit_log_data[field_label] = value - self.request._request.initial_data = audit_log_data self.perform_destroy_override(instance) + self.request._request.initial_data = audit_log_data def perform_destroy_override(self, instance): super().perform_destroy(instance) diff --git a/kobo/apps/audit_log/models.py b/kobo/apps/audit_log/models.py index 2bc70292cb..1778aaaa59 100644 --- a/kobo/apps/audit_log/models.py +++ b/kobo/apps/audit_log/models.py @@ -333,6 +333,8 @@ def create_from_request(cls, request): 'paired-data-list': cls.create_from_paired_data_request, 'asset-file-detail': cls.create_from_file_request, 'asset-file-list': cls.create_from_file_request, + 'asset-export-list': cls.create_from_export_request, + 'exporttask-list': cls.create_from_v1_export, } url_name = request.resolver_match.url_name method = url_name_to_action.get(url_name, None) @@ -491,6 +493,10 @@ def create_from_paired_data_request(cls, request): AuditAction.MODIFY_IMPORTED_FIELDS, ) + @classmethod + def create_from_export_request(cls, request): + cls.create_from_related_request(request, None, AuditAction.EXPORT, None, None) + @staticmethod def sharing_change(old_fields, new_fields): old_enabled = old_fields.get('enabled', False) @@ -550,17 +556,21 @@ def create_from_related_request( 'log_subtype': PROJECT_HISTORY_LOG_PROJECT_SUBTYPE, 'ip_address': get_client_ip(request), 'source': get_human_readable_client_user_agent(request), - label: source_data, } + if label: + metadata.update({label: source_data}) if updated_data is None: action = delete_action elif initial_data is None: action = add_action else: action = modify_action - ProjectHistoryLog.objects.create( - user=request.user, object_id=object_id, action=action, metadata=metadata - ) + if action: + # some actions on related objects do not need to be logged, + # eg deleting an ExportTask + ProjectHistoryLog.objects.create( + user=request.user, object_id=object_id, action=action, metadata=metadata + ) @classmethod def create_from_import_task(cls, task: ImportTask): @@ -598,3 +608,20 @@ def create_from_import_task(cls, task: ImportTask): action=AuditAction.UPDATE_NAME, metadata=metadata, ) + + @classmethod + def create_from_v1_export(cls, request): + updated_data = getattr(request, 'updated_data', None) + if not updated_data: + return + ProjectHistoryLog.objects.create( + user=request.user, + object_id=updated_data['asset_id'], + action=AuditAction.EXPORT, + metadata={ + 'asset_uid': updated_data['asset_uid'], + 'log_subtype': PROJECT_HISTORY_LOG_PROJECT_SUBTYPE, + 'ip_address': get_client_ip(request), + 'source': get_human_readable_client_user_agent(request), + }, + ) diff --git a/kobo/apps/audit_log/tests/test_project_history_logs.py b/kobo/apps/audit_log/tests/test_project_history_logs.py index e386bc4733..693a072eef 100644 --- a/kobo/apps/audit_log/tests/test_project_history_logs.py +++ b/kobo/apps/audit_log/tests/test_project_history_logs.py @@ -807,3 +807,58 @@ def test_create_from_import_task(self, file_or_url, change_name, use_v2): NEW: new_asset.name, }, ) + + def test_export_creates_log(self): + self.asset.deploy(backend='mock', active=True) + request_data = { + 'fields_from_all_versions': True, + 'fields': [], + 'group_sep': '/', + 'hierarchy_in_labels': False, + 'lang': '_default', + 'multiple_select': 'both', + 'type': 'xls', + 'xls_types_as_text': False, + 'include_media_url': True, + } + self._base_project_history_log_test( + method=self.client.post, + url=reverse( + 'api_v2:asset-export-list', + kwargs={ + 'parent_lookup_asset': self.asset.uid, + }, + ), + expected_action=AuditAction.EXPORT, + request_data=request_data, + expected_subtype=PROJECT_HISTORY_LOG_PROJECT_SUBTYPE, + ) + + def test_export_v1_creates_log(self): + self.asset.deploy(backend='mock', active=True) + request_data = { + 'fields_from_all_versions': True, + 'fields': [], + 'group_sep': '/', + 'hierarchy_in_labels': False, + 'lang': '_default', + 'multiple_select': 'both', + 'type': 'xls', + 'xls_types_as_text': False, + 'include_media_url': True, + 'source': reverse('api_v2:asset-detail', kwargs={'uid': self.asset.uid}), + } + # can't use _base_project_history_log_test because + # the old endpoint doesn't like format=json + self.client.post( + path=reverse('exporttask-list'), + data=request_data, + ) + + log_query = ProjectHistoryLog.objects.filter( + metadata__asset_uid=self.asset.uid, action=AuditAction.EXPORT + ) + self.assertEqual(log_query.count(), 1) + log = log_query.first() + self._check_common_metadata(log.metadata, PROJECT_HISTORY_LOG_PROJECT_SUBTYPE) + self.assertEqual(log.object_id, self.asset.id) diff --git a/kpi/models/import_export_task.py b/kpi/models/import_export_task.py index 532586b5dd..6c901fa1b4 100644 --- a/kpi/models/import_export_task.py +++ b/kpi/models/import_export_task.py @@ -816,6 +816,16 @@ def _run_task(self, messages): else: self.save(update_fields=['result', 'last_submission_time']) + @property + def asset(self): + source_url = self.data.get('source', False) + if not source_url: + raise Exception('no source specified for the export') + try: + return resolve_url_to_asset(source_url) + except Asset.DoesNotExist: + raise self.InaccessibleData + def delete(self, *args, **kwargs): # removing exported file from storage self.result.delete(save=False) @@ -833,13 +843,7 @@ def get_export_object( submission_ids = self.data.get('submission_ids', []) if source is None: - source_url = self.data.get('source', False) - if not source_url: - raise Exception('no source specified for the export') - try: - source = resolve_url_to_asset(source_url) - except Asset.DoesNotExist: - raise self.InaccessibleData + source = self.asset source_perms = source.get_perms(self.user) if ( diff --git a/kpi/views/v1/export_task.py b/kpi/views/v1/export_task.py index 95c2d261bc..42ed479b81 100644 --- a/kpi/views/v1/export_task.py +++ b/kpi/views/v1/export_task.py @@ -5,14 +5,14 @@ from rest_framework.response import Response from rest_framework.reverse import reverse +from kobo.apps.audit_log.base_views import AuditLoggedNoUpdateModelViewSet from kpi.models import Asset, ExportTask from kpi.serializers import ExportTaskSerializer from kpi.tasks import export_in_background from kpi.utils.models import remove_string_prefix, resolve_url_to_asset -from kpi.views.no_update_model import NoUpdateModelViewSet -class ExportTaskViewSet(NoUpdateModelViewSet): +class ExportTaskViewSet(AuditLoggedNoUpdateModelViewSet): """ ## This document is for a deprecated version of kpi's API. @@ -133,6 +133,7 @@ class ExportTaskViewSet(NoUpdateModelViewSet): queryset = ExportTask.objects.all() serializer_class = ExportTaskSerializer lookup_field = 'uid' + log_type = 'project-history' def get_queryset(self, *args, **kwargs): if self.request.user.is_anonymous: @@ -198,6 +199,7 @@ def create(self, request, *args, **kwargs): except Asset.DoesNotExist: raise serializers.ValidationError( {'source': 'The specified asset does not exist.'}) + request._request.updated_data = {'asset_id': source.id, 'asset_uid': source.uid} # Complain if it's not deployed if not source.has_deployment: raise serializers.ValidationError( diff --git a/kpi/views/v2/export_task.py b/kpi/views/v2/export_task.py index 281ab24a2e..32f9ca8623 100644 --- a/kpi/views/v2/export_task.py +++ b/kpi/views/v2/export_task.py @@ -5,17 +5,17 @@ ) from rest_framework_extensions.mixins import NestedViewSetMixin +from kobo.apps.audit_log.base_views import AuditLoggedNoUpdateModelViewSet from kpi.filters import SearchFilter from kpi.models import ExportTask from kpi.permissions import ExportTaskPermission from kpi.serializers.v2.export_task import ExportTaskSerializer from kpi.utils.object_permission import get_database_user from kpi.utils.viewset_mixins import AssetNestedObjectViewsetMixin -from kpi.views.no_update_model import NoUpdateModelViewSet class ExportTaskViewSet( - AssetNestedObjectViewsetMixin, NestedViewSetMixin, NoUpdateModelViewSet + AssetNestedObjectViewsetMixin, NestedViewSetMixin, AuditLoggedNoUpdateModelViewSet ): """ ## List of export tasks endpoints @@ -159,6 +159,8 @@ class ExportTaskViewSet( search_default_field_lookups = [ 'uid__icontains', ] + log_type = 'project-history' + logged_fields = [('object_id', 'asset.id')] def get_queryset(self): user = get_database_user(self.request.user) From cd3d2000df10bf8a6b81351987ed111028519f72 Mon Sep 17 00:00:00 2001 From: James Kiger <68701146+jamesrkiger@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:16:49 -0500 Subject: [PATCH 5/8] feat(billing): merge NLP addons feature into main TASK-1255 (#5265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### đŸ“Ŗ Summary Adds billing and tracking support for one-time addons. ### 💭 Notes This PR is for merging the `billing-addons-backend` feature branch into `main`. All substantive changes have already been reviewed and tested, though further testing will be conducted on the beta server once the merge is completed. The frontend feature has been placed behind a feature flag, and so will require the param `?ff_oneTimeAddonsEnabled=true`. --------- Co-authored-by: Jessica Thomas Co-authored-by: RuthShryock Co-authored-by: Guillermo Co-authored-by: RuthShryock <81720958+RuthShryock@users.noreply.github.com> Co-authored-by: Olivier LÊger --- hub/admin/extend_user.py | 15 +- jsapp/js/account/accountSidebar.tsx | 18 +- .../account/add-ons/addOnList.component.tsx | 323 ++++++++++-------- .../js/account/add-ons/addOnList.module.scss | 117 ++++++- .../add-ons/oneTimeAddOnRow.component.tsx | 205 +++++++++++ .../add-ons/updateBadge.component..tsx | 62 ++++ .../billingContextProvider.component.tsx | 7 +- .../account/plans/billingButton.module.scss | 1 + .../account/plans/planContainer.component.tsx | 8 +- .../js/account/plans/useDisplayPrice.hook.tsx | 6 + jsapp/js/account/stripe.api.ts | 73 +++- jsapp/js/account/stripe.types.ts | 34 +- jsapp/js/account/stripe.utils.ts | 52 ++- .../oneTimeAddOnList.component.tsx | 69 ++++ .../oneTimeAddOnList.module.scss | 21 ++ .../oneTimeAddOnUsageModal.component.tsx | 105 ++++++ .../oneTimeAddOnUsageModal.module.scss | 57 ++++ jsapp/js/account/usage/usage.component.tsx | 151 ++++++-- jsapp/js/account/usage/usageContainer.tsx | 178 +++++----- jsapp/js/account/useOneTimeAddonList.hook.ts | 37 ++ jsapp/js/api.endpoints.ts | 1 + jsapp/js/components/modals/koboModal.scss | 1 - .../special/koboAccessibleSelect.module.scss | 16 +- .../special/koboAccessibleSelect.tsx | 105 ++++-- .../usageLimits/useExceedingLimits.hook.ts | 30 +- jsapp/js/featureFlags.ts | 3 +- kobo/apps/audit_log/signals.py | 1 + kobo/apps/hook/tests/hook_test_case.py | 1 + kobo/apps/hook/utils/tests/mixins.py | 25 +- kobo/apps/kobo_auth/models.py | 10 +- .../viewsets/test_xform_submission_api.py | 6 +- .../apps/api/viewsets/xform_submission_api.py | 1 + .../commands/populate_submission_counters.py | 21 +- .../update_attachment_storage_bytes.py | 19 +- ..._populate_daily_xform_counters_for_year.py | 34 +- .../0030_backfill_lost_monthly_counters.py | 68 ---- .../0031_remove_null_user_daily_counters.py | 10 - ...032_alter_daily_submission_counter_user.py | 2 +- .../migrations/0039_populate_counters.py | 120 +++++++ .../openrosa/apps/logger/models/instance.py | 7 +- .../apps/openrosa/apps/logger/models/xform.py | 12 +- .../0017_userprofile_submissions_suspended.py | 18 + .../openrosa/apps/main/models/user_profile.py | 1 + kobo/apps/openrosa/apps/main/urls.py | 257 ++++++++------ kobo/apps/openrosa/libs/filters.py | 3 +- kobo/apps/openrosa/libs/utils/logger_tools.py | 2 +- kobo/apps/organizations/admin/__init__.py | 2 +- kobo/apps/organizations/admin/organization.py | 13 +- .../admin/organization_invite.py | 1 + .../organizations/admin/organization_user.py | 9 +- kobo/apps/organizations/forms.py | 1 + .../0006_update_organization_name.py | 2 +- kobo/apps/organizations/models.py | 97 ++++-- kobo/apps/organizations/permissions.py | 5 +- kobo/apps/organizations/serializers.py | 3 +- kobo/apps/organizations/tasks.py | 2 +- .../organizations/tests/test_organizations.py | 2 +- .../tests/test_organizations_api.py | 59 ++-- .../tests/test_organizations_model.py | 22 ++ kobo/apps/organizations/types.py | 3 + kobo/apps/organizations/utils.py | 11 +- kobo/apps/organizations/views.py | 28 +- kobo/apps/project_ownership/models/invite.py | 13 +- .../apps/project_ownership/models/transfer.py | 5 +- .../project_ownership/serializers/invite.py | 3 +- .../tests/api/v2/test_api.py | 4 + kobo/apps/project_ownership/utils.py | 22 +- kobo/apps/stripe/admin.py | 59 ++++ kobo/apps/stripe/constants.py | 11 + kobo/apps/stripe/migrations/0001_initial.py | 102 ++++++ kobo/apps/stripe/migrations/__init__.py | 0 kobo/apps/stripe/models.py | 254 ++++++++++++++ kobo/apps/stripe/serializers.py | 21 +- .../templates/admin/add-ons/change_list.html | 8 + .../stripe/tests/test_customer_portal_api.py | 11 +- .../stripe/tests/test_one_time_addons_api.py | 188 ++++++++-- .../stripe/tests/test_organization_usage.py | 51 +++ kobo/apps/stripe/utils.py | 90 ++++- kobo/apps/stripe/views.py | 62 ++-- .../integrations/google/google_transcribe.py | 2 + .../tests/test_submission_extras_api_post.py | 15 +- kobo/apps/trackers/tests/test_utils.py | 121 +++++++ kobo/apps/trackers/utils.py | 69 +++- kobo/settings/base.py | 3 +- kpi/backends.py | 2 +- kpi/db_routers.py | 5 +- kpi/deployment_backends/openrosa_backend.py | 20 +- kpi/filters.py | 19 +- kpi/mixins/object_permission.py | 12 +- kpi/permissions.py | 8 +- kpi/serializers/v2/asset.py | 1 + kpi/serializers/v2/service_usage.py | 4 + .../test_api_asset_permission_assignment.py | 4 +- kpi/tests/api/v2/test_api_assets.py | 2 +- kpi/tests/api/v2/test_api_collections.py | 60 ++-- kpi/tests/api/v2/test_api_permissions.py | 17 +- kpi/tests/api/v2/test_api_service_usage.py | 8 +- kpi/tests/api/v2/test_api_submissions.py | 3 +- kpi/tests/base_test_case.py | 5 +- kpi/tests/kpi_test_case.py | 4 +- kpi/tests/test_cache_utils.py | 55 +++ kpi/tests/test_permissions.py | 6 +- kpi/tests/test_usage_calculator.py | 40 ++- kpi/tests/utils/mixins.py | 32 +- kpi/utils/cache.py | 106 +++++- kpi/utils/django_orm_helper.py | 32 +- kpi/utils/usage_calculator.py | 66 +++- kpi/views/v2/asset.py | 4 +- kpi/views/v2/asset_snapshot.py | 15 +- 109 files changed, 3235 insertions(+), 982 deletions(-) create mode 100644 jsapp/js/account/add-ons/oneTimeAddOnRow.component.tsx create mode 100644 jsapp/js/account/add-ons/updateBadge.component..tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/one-time-add-on-list/oneTimeAddOnList.component.tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/one-time-add-on-list/oneTimeAddOnList.module.scss create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/oneTimeAddOnUsageModal.component.tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/oneTimeAddOnUsageModal.module.scss create mode 100644 jsapp/js/account/useOneTimeAddonList.hook.ts create mode 100644 kobo/apps/openrosa/apps/logger/migrations/0039_populate_counters.py create mode 100644 kobo/apps/openrosa/apps/main/migrations/0017_userprofile_submissions_suspended.py create mode 100644 kobo/apps/organizations/tests/test_organizations_model.py create mode 100644 kobo/apps/organizations/types.py create mode 100644 kobo/apps/stripe/admin.py create mode 100644 kobo/apps/stripe/migrations/0001_initial.py create mode 100644 kobo/apps/stripe/migrations/__init__.py create mode 100644 kobo/apps/stripe/models.py create mode 100644 kobo/apps/stripe/templates/admin/add-ons/change_list.html create mode 100644 kobo/apps/trackers/tests/test_utils.py create mode 100644 kpi/tests/test_cache_utils.py diff --git a/hub/admin/extend_user.py b/hub/admin/extend_user.py index e18e80493f..b962f2a1ad 100644 --- a/hub/admin/extend_user.py +++ b/hub/admin/extend_user.py @@ -25,6 +25,7 @@ from kobo.apps.trash_bin.models.account import AccountTrash from kobo.apps.trash_bin.utils import move_to_trash from kpi.models.asset import AssetDeploymentStatus + from .filters import UserAdvancedSearchFilter from .mixins import AdvancedSearchMixin @@ -167,9 +168,7 @@ class ExtendedUserAdmin(AdvancedSearchMixin, UserAdmin): actions = ['remove', 'delete'] class Media: - css = { - 'all': ('admin/css/inline_as_fieldset.css',) - } + css = {'all': ('admin/css/inline_as_fieldset.css',)} @admin.action(description='Remove selected users (delete everything but their username)') def remove(self, request, queryset, **kwargs): @@ -293,9 +292,13 @@ def _filter_queryset_for_organization_user(self, queryset): """ Displays only users whose organization has a single member. """ - return queryset.annotate( - user_count=Count('organizations_organization__organization_users') - ).filter(user_count__lte=1).order_by('username') + return ( + queryset.annotate( + user_count=Count('organizations_organization__organization_users') + ) + .filter(user_count__lte=1) + .order_by('username') + ) def _remove_or_delete( self, diff --git a/jsapp/js/account/accountSidebar.tsx b/jsapp/js/account/accountSidebar.tsx index a8ef94ca45..41aa3a54ba 100644 --- a/jsapp/js/account/accountSidebar.tsx +++ b/jsapp/js/account/accountSidebar.tsx @@ -48,10 +48,6 @@ function AccountSidebar() { setShowPlans(true); }, [subscriptionStore.isInitialised]); - const showAddOnsLink = useMemo(() => { - return !subscriptionStore.planResponse.length; - }, [subscriptionStore.isInitialised]); - return (