From 88374a0f9a4a96f67d963c666d7288a63db51acf Mon Sep 17 00:00:00 2001 From: Muhammad Sameer Amin <35958006+sameeramin@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:34:54 +0500 Subject: [PATCH 1/2] feat: add `integrated_channel` subapp with utils --- channel_integrations/README.md | 74 ++ channel_integrations/catalog_service_utils.py | 27 + channel_integrations/exceptions.py | 14 + .../integrated_channel/__init__.py | 5 + .../integrated_channel/admin/__init__.py | 142 +++ .../integrated_channel/apps.py | 13 + .../integrated_channel/channel_settings.py | 19 + .../integrated_channel/client.py | 101 ++ .../integrated_channel/constants.py | 6 + .../integrated_channel/exporters/__init__.py | 32 + .../exporters/content_metadata.py | 691 ++++++++++++ .../exporters/learner_data.py | 842 ++++++++++++++ .../integrated_channel/management/__init__.py | 3 + .../management/commands/__init__.py | 159 +++ .../assign_skills_to_degreed_courses.py | 128 +++ .../commands/backfill_course_end_dates.py | 41 + .../backfill_missing_csod_foreign_keys.py | 76 ++ .../commands/backfill_missing_foreign_keys.py | 134 +++ .../backfill_remote_action_timestamps.py | 60 + .../cleanup_duplicate_assignment_records.py | 60 + ..._learner_transmissions_transmitted_true.py | 48 + .../mark_orphaned_content_metadata_audits.py | 28 + ...cate_learner_transmission_audit_records.py | 67 ++ ...remove_null_catalog_transmission_audits.py | 30 + ...emove_stale_integrated_channel_api_logs.py | 41 + .../commands/reset_csod_remote_deleted_at.py | 62 + .../reset_sapsf_learner_transmissions.py | 69 ++ .../commands/transmit_content_metadata.py | 54 + .../commands/transmit_learner_data.py | 58 + .../transmit_subsection_learner_data.py | 61 + .../commands/unlink_inactive_sap_learners.py | 31 + .../update_content_transmission_catalogs.py | 50 + .../integrated_channel/migrations/__init__.py | 0 .../integrated_channel/models.py | 1002 +++++++++++++++++ .../integrated_channel/tasks.py | 481 ++++++++ .../transmitters/__init__.py | 31 + .../transmitters/content_metadata.py | 264 +++++ .../transmitters/learner_data.py | 488 ++++++++ channel_integrations/lms_utils.py | 131 +++ channel_integrations/utils.py | 565 ++++++++++ 40 files changed, 6188 insertions(+) create mode 100644 channel_integrations/README.md create mode 100644 channel_integrations/catalog_service_utils.py create mode 100644 channel_integrations/exceptions.py create mode 100644 channel_integrations/integrated_channel/__init__.py create mode 100644 channel_integrations/integrated_channel/admin/__init__.py create mode 100644 channel_integrations/integrated_channel/apps.py create mode 100644 channel_integrations/integrated_channel/channel_settings.py create mode 100644 channel_integrations/integrated_channel/client.py create mode 100644 channel_integrations/integrated_channel/constants.py create mode 100644 channel_integrations/integrated_channel/exporters/__init__.py create mode 100644 channel_integrations/integrated_channel/exporters/content_metadata.py create mode 100644 channel_integrations/integrated_channel/exporters/learner_data.py create mode 100644 channel_integrations/integrated_channel/management/__init__.py create mode 100644 channel_integrations/integrated_channel/management/commands/__init__.py create mode 100644 channel_integrations/integrated_channel/management/commands/assign_skills_to_degreed_courses.py create mode 100644 channel_integrations/integrated_channel/management/commands/backfill_course_end_dates.py create mode 100644 channel_integrations/integrated_channel/management/commands/backfill_missing_csod_foreign_keys.py create mode 100644 channel_integrations/integrated_channel/management/commands/backfill_missing_foreign_keys.py create mode 100644 channel_integrations/integrated_channel/management/commands/backfill_remote_action_timestamps.py create mode 100644 channel_integrations/integrated_channel/management/commands/cleanup_duplicate_assignment_records.py create mode 100644 channel_integrations/integrated_channel/management/commands/mark_learner_transmissions_transmitted_true.py create mode 100644 channel_integrations/integrated_channel/management/commands/mark_orphaned_content_metadata_audits.py create mode 100644 channel_integrations/integrated_channel/management/commands/remove_duplicate_learner_transmission_audit_records.py create mode 100644 channel_integrations/integrated_channel/management/commands/remove_null_catalog_transmission_audits.py create mode 100644 channel_integrations/integrated_channel/management/commands/remove_stale_integrated_channel_api_logs.py create mode 100644 channel_integrations/integrated_channel/management/commands/reset_csod_remote_deleted_at.py create mode 100644 channel_integrations/integrated_channel/management/commands/reset_sapsf_learner_transmissions.py create mode 100644 channel_integrations/integrated_channel/management/commands/transmit_content_metadata.py create mode 100644 channel_integrations/integrated_channel/management/commands/transmit_learner_data.py create mode 100644 channel_integrations/integrated_channel/management/commands/transmit_subsection_learner_data.py create mode 100644 channel_integrations/integrated_channel/management/commands/unlink_inactive_sap_learners.py create mode 100644 channel_integrations/integrated_channel/management/commands/update_content_transmission_catalogs.py create mode 100644 channel_integrations/integrated_channel/migrations/__init__.py create mode 100644 channel_integrations/integrated_channel/models.py create mode 100644 channel_integrations/integrated_channel/tasks.py create mode 100644 channel_integrations/integrated_channel/transmitters/__init__.py create mode 100644 channel_integrations/integrated_channel/transmitters/content_metadata.py create mode 100644 channel_integrations/integrated_channel/transmitters/learner_data.py create mode 100644 channel_integrations/lms_utils.py create mode 100644 channel_integrations/utils.py diff --git a/channel_integrations/README.md b/channel_integrations/README.md new file mode 100644 index 0000000..6cd237e --- /dev/null +++ b/channel_integrations/README.md @@ -0,0 +1,74 @@ +# Integrated Channels + +## Overview + +An integrated channel is an abstraction meant to represent a third-party system +which provides an API that can be used to transmit EdX data to the third-party +system. The most common example of such a third-party system is an enterprise-level +learning management system (LMS). LMS users are able to discover content made available +by many different content providers and manage the learning outcomes that are produced +by interaction with the content providers. In such a scenario, EdX would be the content +provider while a system like SAP SuccessFactors would be the integrated channel. + +The integrated channel subsystem was developed as a means for consolidating common code +needed for integrating the EdX ecosystem with any integrated channel and minimizing the +amount of code that would need to be written in order to add a new integrated channel. + +## Integration Phases + +The subsystem organizes the integration into two separate phases: + +1. The *export* phase is where data is collected from the EdX ecosystem + and transformed into the schema expected by the integrated channel. +2. The *transmission* phase is where the exported data is transmitted to + the integrated channel by whatever means is provided by the third-party + system, usually an HTTP-based API. + +There are [base implementations](https://github.com/openedx/edx-enterprise/tree/master/integrated_channels/integrated_channel) +for each of these phases which can be extended for +channel-specific customizations needed for integrating with a given integrated channel. +Channel-specific implementation code should be placed in a new directory adjacent to +the base implementation code. + +For example: + +* [degreed](https://github.com/openedx/edx-enterprise/tree/master/integrated_channels/degreed) +* [sap_success_factors](https://github.com/openedx/edx-enterprise/tree/master/integrated_channels/sap_success_factors) + +## Integration Points + +There are currently two integration points supported for integrated channels: + +1. *Content metadata* - Metadata (e.g. titles, descriptions, etc.) related to EdX content (e.g. courses, programs) can be exported and transmitted to an integrated channel to assist content discovery in the third-party system. +2. *Learner data* - Specifically, learner outcome data for each content enrollment can be + exported and transmitted to the integrated channel. + +Additional integration points may be added in the future. + +## Integrated Channel Configuration + +There is a many-to-many relationship between integrated channels and enterprise customers. +Configuration information related to an enterprise-specific integrated channel is stored in +the database by creating a concrete implementation of the abstract +[EnterpriseCustomerPluginConfiguration](https://github.com/openedx/edx-enterprise/blob/master/integrated_channels/integrated_channel/models.py) model. Fields can be added to the concrete +implementation to store values such as the API credentials for a given enterprise customer's +instance of an integrated channel. + +Configuration that is common for all instances of integrated channel regardless of enterprise +customer should be persisted by implementing a model within the channel-specific implementation +directory, (e.g. [SAPSuccessFactorsGlobalConfiguration](https://github.com/openedx/edx-enterprise/blob/master/integrated_channels/sap_success_factors/models.py)). + +## Content Metadata Synchronization + +The set of content metadata transmitted for a given integrated channel instance is defined by the +EnterpriseCustomerCatalogs configured for the associated EnterpriseCustomer. In order to ensure that the content metadata transmitted to an integrated channel is synchronized with the content made available by the EnterpriseCustomer's catalogs, each content metadata item transmission is persisted using the [ContentMetadataItemTransmission](https://github.com/openedx/edx-enterprise/blob/master/integrated_channels/integrated_channel/models.py) model. ContentMetadataItemTransmission records are created, updated, and deleted as EnterpriseCustomerCatalogs are modified and modified sets of content metadata are exported and transmitted to the integrated channel. + +## Implementing a new Integrated Channel + +Here is the general implementation plan for creating a new integrated channel. + +1. Obtain API documentation for the third-party service which the integrated + channel will represent. +2. Implement a concrete subclass of [IntegratedChannelApiClient](https://github.com/openedx/edx-enterprise/blob/master/integrated_channels/integrated_channel/client.py). +3. Copy/paste one of the existing integrated channel directories and modify the implementation + as needed for the new integrated channel. diff --git a/channel_integrations/catalog_service_utils.py b/channel_integrations/catalog_service_utils.py new file mode 100644 index 0000000..383dba3 --- /dev/null +++ b/channel_integrations/catalog_service_utils.py @@ -0,0 +1,27 @@ +""" +A utility collection for calls from integrated_channels to Catalog service +""" +from enterprise.api_client.discovery import get_course_catalog_api_service_client + + +def get_course_id_for_enrollment(enterprise_enrollment): + """ + Fetch course_id for a given enterprise enrollment + Returns None if no course id found for course_id associated with input enterprise_enrollment + """ + course_catalog_client = get_course_catalog_api_service_client( + site=enterprise_enrollment.enterprise_customer_user.enterprise_customer.site + ) + return course_catalog_client.get_course_id(enterprise_enrollment.course_id) + + +def get_course_run_for_enrollment(enterprise_enrollment): + """ + Fetch course_run associated with the enrollment + Returns empty dict {} if no course_run found for enterprise_enrollment.course_id + """ + course_catalog_client = get_course_catalog_api_service_client( + site=enterprise_enrollment.enterprise_customer_user.enterprise_customer.site + ) + course_run = course_catalog_client.get_course_run(enterprise_enrollment.course_id) + return course_run diff --git a/channel_integrations/exceptions.py b/channel_integrations/exceptions.py new file mode 100644 index 0000000..18d1d41 --- /dev/null +++ b/channel_integrations/exceptions.py @@ -0,0 +1,14 @@ +""" +Integrated channel custom exceptions. +""" + + +class ClientError(Exception): + """ + Indicate a problem when interacting with an integrated channel. + """ + def __init__(self, message, status_code=500): + """Save the status code and message raised from the client.""" + self.status_code = status_code + self.message = message + super().__init__(message) diff --git a/channel_integrations/integrated_channel/__init__.py b/channel_integrations/integrated_channel/__init__.py new file mode 100644 index 0000000..1d910dd --- /dev/null +++ b/channel_integrations/integrated_channel/__init__.py @@ -0,0 +1,5 @@ +""" +Base Integrated Channel application for specific integrated channels to use as a starting point. +""" + +__version__ = "0.1.0" diff --git a/channel_integrations/integrated_channel/admin/__init__.py b/channel_integrations/integrated_channel/admin/__init__.py new file mode 100644 index 0000000..ca649fb --- /dev/null +++ b/channel_integrations/integrated_channel/admin/__init__.py @@ -0,0 +1,142 @@ +""" +Admin site configurations for integrated channel's Content Metadata Transmission table. +""" + +from django.contrib import admin + +from channel_integrations.integrated_channel.models import ( + ApiResponseRecord, + ContentMetadataItemTransmission, + IntegratedChannelAPIRequestLogs, +) +from channel_integrations.utils import get_enterprise_customer_from_enterprise_enrollment + + +class BaseLearnerDataTransmissionAuditAdmin(admin.ModelAdmin): + """ + Base admin class to hold commonly used methods across integrated channel admin views + """ + def enterprise_customer_name(self, obj): + """ + Returns: the name for the attached EnterpriseCustomer or None if customer object does not exist. + Args: + obj: The instance of Django model being rendered with this admin form. + """ + ent_customer = get_enterprise_customer_from_enterprise_enrollment(obj.enterprise_course_enrollment_id) + return ent_customer.name if ent_customer else None + + +@admin.action(description='Clear remote_deleted_at on ContentMetadataItemTransmission item(s)') +def clear_remote_deleted_at(modeladmin, request, queryset): # pylint: disable=unused-argument + queryset.update(remote_deleted_at=None) + + +@admin.register(ContentMetadataItemTransmission) +class ContentMetadataItemTransmissionAdmin(admin.ModelAdmin): + """ + Admin for the ContentMetadataItemTransmission audit table + """ + list_display = ( + 'enterprise_customer', + 'integrated_channel_code', + 'content_id', + 'remote_deleted_at', + 'modified' + ) + + search_fields = ( + 'enterprise_customer__name', + 'enterprise_customer__uuid', + 'integrated_channel_code', + 'content_id' + ) + + raw_id_fields = ( + 'enterprise_customer', + ) + + readonly_fields = [ + 'api_record', + 'api_response_status_code', + 'friendly_status_message', + ] + + actions = [ + clear_remote_deleted_at, + ] + + list_per_page = 1000 + + +@admin.register(ApiResponseRecord) +class ApiResponseRecordAdmin(admin.ModelAdmin): + """ + Admin for the ApiResponseRecord table + """ + list_display = ( + 'id', + 'status_code' + ) + + search_fields = ( + 'id', + 'status_code' + ) + + readonly_fields = ( + 'status_code', + 'body' + ) + + list_per_page = 1000 + + +@admin.register(IntegratedChannelAPIRequestLogs) +class IntegratedChannelAPIRequestLogAdmin(admin.ModelAdmin): + """ + Django admin model for IntegratedChannelAPIRequestLogs. + """ + + list_display = [ + "endpoint", + "enterprise_customer_id", + "time_taken", + "status_code", + ] + search_fields = [ + "enterprise_customer__name__icontains", + "enterprise_customer__uuid__iexact", + "enterprise_customer_configuration_id__iexact", + "endpoint__icontains", + ] + readonly_fields = [ + "status_code", + "enterprise_customer", + "enterprise_customer_configuration_id", + "endpoint", + "time_taken", + "response_body", + "payload", + ] + list_filter = ('status_code',) + + list_per_page = 20 + + def get_queryset(self, request): + """ + Optimize queryset by selecting related 'enterprise_customer' and limiting fields. + """ + queryset = super().get_queryset(request) + return queryset.select_related('enterprise_customer').only( + 'id', + 'endpoint', + 'enterprise_customer_id', + 'time_taken', + 'status_code', + 'enterprise_customer__name', + 'enterprise_customer__uuid', + 'enterprise_customer_configuration_id' + ) + + class Meta: + model = IntegratedChannelAPIRequestLogs diff --git a/channel_integrations/integrated_channel/apps.py b/channel_integrations/integrated_channel/apps.py new file mode 100644 index 0000000..b3b4646 --- /dev/null +++ b/channel_integrations/integrated_channel/apps.py @@ -0,0 +1,13 @@ +""" +Enterprise Integrated Channel Django application initialization. +""" + +from django.apps import AppConfig + + +class IntegratedChannelConfig(AppConfig): + """ + Configuration for the Enterprise Integrated Channel Django application. + """ + name = 'channel_integrations.integrated_channel' + verbose_name = "Enterprise Integrated Channels" diff --git a/channel_integrations/integrated_channel/channel_settings.py b/channel_integrations/integrated_channel/channel_settings.py new file mode 100644 index 0000000..116bf38 --- /dev/null +++ b/channel_integrations/integrated_channel/channel_settings.py @@ -0,0 +1,19 @@ +""" +Channel level settings (global for all channels). +""" + + +class ChannelSettingsMixin: + """ + Mixin for channel settings that apply to all channels. + Provides common settings for all channels. Each channels is free to override settings at their + Exporter or Transmitter or Client level, as needed + + Important: If you add a setting here, please add a test to cover this default, as well as + any overrides you add on a per channel basis. See this test for an example: + tests/test_integrated_channels/test_sap_success_factors/test_transmitters/test_learner_data.py + """ + + # a channel should override this to False if they don't want grade changes to + # cause retransmission of completion records + INCLUDE_GRADE_FOR_COMPLETION_AUDIT_CHECK = True diff --git a/channel_integrations/integrated_channel/client.py b/channel_integrations/integrated_channel/client.py new file mode 100644 index 0000000..32ee456 --- /dev/null +++ b/channel_integrations/integrated_channel/client.py @@ -0,0 +1,101 @@ +""" +Base API client for integrated channels. +""" + +from enum import Enum + + +class IntegratedChannelHealthStatus(Enum): + """ + Health status list for Integrated Channels + """ + HEALTHY = 'HEALTHY' + INVALID_CONFIG = 'INVALID_CONFIG' + CONNECTION_FAILURE = 'CONNECTION_FAILURE' + + +class IntegratedChannelApiClient: + """ + This is the interface to be implemented by API clients for integrated channels. + """ + + def __init__(self, enterprise_configuration): + """ + Instantiate a new base client. + + Args: + enterprise_configuration: An enterprise customers's configuration model for connecting with the channel + + Raises: + ValueError: If an enterprise configuration is not provided. + """ + if not enterprise_configuration: + raise ValueError( + 'An Enterprise Customer Configuration is required to instantiate an Integrated Channel API client.' + ) + self.enterprise_configuration = enterprise_configuration + + def create_course_completion(self, user_id, payload): + """ + Make a POST request to the integrated channel's completion API to update completion status for a user. + + :param user_id: The ID of the user for whom completion status must be updated. + :param payload: The JSON encoded payload containing the completion data. + """ + raise NotImplementedError('Implement in concrete subclass.') + + def delete_course_completion(self, user_id, payload): + """ + Make a DELETE request to the integrated channel's completion API to update completion status for a user. + + :param user_id: The ID of the user for whom completion status must be updated. + :param payload: The JSON encoded payload containing the completion data. + """ + raise NotImplementedError('Implement in concrete subclass.') + + def create_content_metadata(self, serialized_data): + """ + Create content metadata using the integrated channel's API. + """ + raise NotImplementedError() + + def update_content_metadata(self, serialized_data): + """ + Update content metadata using the integrated channel's API. + """ + raise NotImplementedError() + + def delete_content_metadata(self, serialized_data): + """ + Delete content metadata using the integrated channel's API. + """ + raise NotImplementedError() + + def create_assessment_reporting(self, user_id, payload): + """ + Send a request to the integrated channel's grade API to update the assessment level reporting status for a user. + """ + raise NotImplementedError() + + def cleanup_duplicate_assignment_records(self, courses): + """ + Delete duplicate assignments transmitted through the integrated channel's API. + """ + raise NotImplementedError() + + def health_check(self): + """Check integrated channel's config health + + Returns: IntegratedChannelHealthStatus + HEALTHY if configuration is valid + INVALID_CONFIG if configuration is incomplete/invalid + """ + is_valid = self.enterprise_configuration.is_valid + missing_fields = is_valid[0] + missing_ct = len(missing_fields['missing']) if 'missing' in missing_fields else 0 + incorrect_fields = is_valid[1] + incorrect_ct = len(incorrect_fields['incorrect']) if 'incorrect' in incorrect_fields else 0 + if missing_ct > 0 or incorrect_ct > 0: + return IntegratedChannelHealthStatus.INVALID_CONFIG + else: + return IntegratedChannelHealthStatus.HEALTHY diff --git a/channel_integrations/integrated_channel/constants.py b/channel_integrations/integrated_channel/constants.py new file mode 100644 index 0000000..8636e65 --- /dev/null +++ b/channel_integrations/integrated_channel/constants.py @@ -0,0 +1,6 @@ +""" +Constants used by the integrated channels +""" + +ISO_8601_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' +TASK_LOCK_EXPIRY_SECONDS = 60 * 60 * 12 diff --git a/channel_integrations/integrated_channel/exporters/__init__.py b/channel_integrations/integrated_channel/exporters/__init__.py new file mode 100644 index 0000000..7763c84 --- /dev/null +++ b/channel_integrations/integrated_channel/exporters/__init__.py @@ -0,0 +1,32 @@ +""" +Package for generic data exporters which serialize data to be transmitted to integrated channels. +""" + + +class Exporter: + """ + Interface for exporting data to be transmitted to an integrated channel. + + The interface contains the following method(s): + + export() + Yields a serialized piece of data plus the HTTP method to be used by the transmitter. + """ + + def __init__(self, user, enterprise_configuration): + """ + Store the data needed to export the learner data to the integrated channel. + + Arguments: + * user: User instance with access to the Grades API for the Enterprise Customer's courses. + * enterprise_configuration - The configuration connecting an enterprise to an integrated channel. + """ + self.user = user + self.enterprise_configuration = enterprise_configuration + self.enterprise_customer = enterprise_configuration.enterprise_customer + + def export(self, **kwargs): + """ + Export (read: serialize) data to be used by a transmitter to transmit to an integrated channel. + """ + raise NotImplementedError('Implement in concrete subclass transmitter.') diff --git a/channel_integrations/integrated_channel/exporters/content_metadata.py b/channel_integrations/integrated_channel/exporters/content_metadata.py new file mode 100644 index 0000000..dbe4155 --- /dev/null +++ b/channel_integrations/integrated_channel/exporters/content_metadata.py @@ -0,0 +1,691 @@ +""" +Assist integrated channels with retrieving content metadata. + +Module contains resources for integrated channels to retrieve all the +metadata for content contained in the catalogs associated with a particular +enterprise customer. +""" + +import sys +from logging import getLogger + +from django.apps import apps +from django.conf import settings +from django.db.models import Q +from django.utils import timezone + +from enterprise.api_client.enterprise_catalog import EnterpriseCatalogApiClient +from enterprise.constants import ( + EXEC_ED_CONTENT_DESCRIPTION_TAG, + EXEC_ED_COURSE_TYPE, + IC_CREATE_ACTION, + IC_UPDATE_ACTION, + TRANSMISSION_MARK_CREATE, +) +from enterprise.utils import get_content_metadata_item_id +from channel_integrations.integrated_channel.exporters import Exporter +from channel_integrations.utils import generate_formatted_log, truncate_item_dicts + +LOGGER = getLogger(__name__) + + +class ContentMetadataExporter(Exporter): + """ + Base class for content metadata exporters. + """ + + # DATA_TRANSFORM_MAPPING is used to map the content metadata field names expected by the integrated channel + # to the edX content metadata schema. The values contained in the dict will be used as keys to access values + # in each content metadata item dict which is being exported. + # + # Example: + # { + # 'contentID': 'key', + # 'courseTitle': 'title' + # } + # + # Defines a transformation of the content metadata item to: + # + # { + # 'contentID': content_metadata_item['key'], + # 'courseTitle': content_metadata_item['title'] + # } + # + # Subclasses should override this class variable. By default, the edX content metadata schema is returned in + # its entirety. + # + # In addition, subclasses can implement transform functions which receive a content metadata item for more + # complicated field transformations. These functions can be content type-specific or generic for all content + # types. + # + # Example: + # DATA_TRANSFORM_MAPPING = { + # 'coursePrice': 'price' + # } + # # Content type-specific transformer + # def transform_course_price(self, course): + # return course['course_runs'][0]['seats']['verified]['price'] + # # Generic transformer + # def transform_provider_id(self, course): + # return self.enterprise_configuration.provider_id + # + # TODO: Move this to the EnterpriseCustomerPluginConfiguration model as a JSONField. + DATA_TRANSFORM_MAPPING = {} + SKIP_KEY_IF_NONE = False + LAST_24_HRS = timezone.now() - timezone.timedelta(hours=24) + + def __init__(self, user, enterprise_configuration): + """ + Initialize the exporter. + """ + super().__init__(user, enterprise_configuration) + self.enterprise_catalog_api = EnterpriseCatalogApiClient(self.user) + + def _log_info(self, msg, course_or_course_run_key=None): + LOGGER.info( + generate_formatted_log( + channel_name=self.enterprise_configuration.channel_code(), + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=course_or_course_run_key, + plugin_configuration_id=self.enterprise_configuration.id, + message=msg + ) + ) + + def _log_exception(self, msg, course_or_course_run_key=None): + LOGGER.exception( + generate_formatted_log( + channel_name=self.enterprise_configuration.channel_code(), + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=course_or_course_run_key, + plugin_configuration_id=self.enterprise_configuration.id, + message=msg + ) + ) + + def _get_catalog_content_keys(self, enterprise_customer_catalog=None): + """ + Retrieve all non-deleted content transmissions under a given customer's catalog + """ + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + + # created, not deleted, no error code + base_content_query = Q( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + remote_deleted_at__isnull=True, + remote_created_at__isnull=False, + ) + # enterprise_customer_catalog filter is optional + if enterprise_customer_catalog is not None: + base_content_query.add(Q(enterprise_customer_catalog_uuid=enterprise_customer_catalog.uuid), Q.AND) + # api_response_status_code can be null, treat that as successful, otherwise look for less than http 400 + base_content_query.add(Q(api_response_status_code__isnull=True) | Q(api_response_status_code__lt=400), Q.AND) + + # query for records that have a created at date and a failure status code + failed_query = Q( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + api_response_status_code__gte=400, + remote_created_at__isnull=False, + ) + # filter only records who have failed to delete or update, meaning we know they exist on the customer's + # instance and require some kind of action + failed_query.add(Q(remote_deleted_at__isnull=False) | Q(remote_updated_at__isnull=False), Q.AND) + # enterprise_customer_catalog filter is optional + if enterprise_customer_catalog is not None: + failed_query.add(Q(enterprise_customer_catalog_uuid=enterprise_customer_catalog.uuid), Q.AND) + + # base query OR failed delete query + final_content_query = Q(base_content_query | failed_query) + + past_transmissions = ContentMetadataItemTransmission.objects.filter( + final_content_query + ).values('content_id') + if not past_transmissions: + return [] + return [key.get('content_id') for key in past_transmissions] + + def _check_matched_content_updated_at( + self, + enterprise_customer_catalog, + matched_items, + force_retrieve_all_catalogs + ): + """ + Take a list of content keys and their respective last updated time and build a mapping between content keys and + past content metadata transmission record when the last updated time comes after the last updated time of the + record. + + Args: + enterprise_customer_catalog (EnterpriseCustomerCatalog): The enterprise catalog object + + matched_items (list): A list of dicts containing content keys and the last datetime that the respective + content was updated + + force_retrieve_all_catalogs (Bool): If set to True, all content under the catalog will be retrieved, + regardless of the last updated at time + """ + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + items_to_update = {} + for matched_item in matched_items: + content_id = matched_item.get('content_key') + content_last_changed = matched_item.get('date_updated') + incomplete_transmission = ContentMetadataItemTransmission.incomplete_update_transmissions( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + plugin_configuration_id=self.enterprise_configuration.id, + integrated_channel_code=self.enterprise_configuration.channel_code(), + content_id=content_id, + ).first() + if incomplete_transmission: + self._log_info( + 'Found an unsent content update record while creating record. ' + 'Including record.', + course_or_course_run_key=content_id + ) + incomplete_transmission.mark_for_update() + items_to_update[content_id] = incomplete_transmission + else: + content_query = Q( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + enterprise_customer_catalog_uuid=enterprise_customer_catalog.uuid, + plugin_configuration_id=self.enterprise_configuration.id, + content_id=content_id, + remote_deleted_at__isnull=True, + remote_created_at__isnull=False, + ) + + content_query.add( + Q(remote_errored_at__lt=self.LAST_24_HRS) | Q(remote_errored_at__isnull=True) | + Q(remote_errored_at__lt=self.enterprise_customer.modified), + Q.AND + ) + # If not force_retrieve_all_catalogs, filter content records where `content last changed` is less than + # the matched item's `date_updated`, otherwise select the row regardless of what the updated at time is. + if not force_retrieve_all_catalogs: + last_changed_query = Q(content_last_changed__lt=content_last_changed) + last_changed_query.add(Q(content_last_changed__isnull=True), Q.OR) + get_marked_for = Q(marked_for='update') + get_marked_for.add(last_changed_query, Q.OR) + content_query.add(get_marked_for, Q.AND) + items_to_update_query = ContentMetadataItemTransmission.objects.filter(content_query) + item = items_to_update_query.first() + if item: + item.mark_for_update() + items_to_update[content_id] = item + return items_to_update + + def _check_matched_content_to_create( + self, + enterprise_customer_catalog, + matched_items + ): + """ + Take a list of content keys and create ContentMetadataItemTransmission records. When existed soft-deleted + records exist, resurrect them. When created but not-yet-transmitted records exist, include them. + + Args: + enterprise_customer_catalog (EnterpriseCustomerCatalog): The enterprise catalog object + + matched_items (list): A list of dicts containing content keys and the last datetime that the respective + content was updated + + force_retrieve_all_catalogs (Bool): If set to True, all content under the catalog will be retrieved, + regardless of the last updated at time + """ + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + items_to_create = {} + for matched_item in matched_items: + content_id = matched_item.get('content_key') + content_last_changed = matched_item.get('date_updated') + past_deleted_transmission = ContentMetadataItemTransmission.deleted_transmissions( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + plugin_configuration_id=self.enterprise_configuration.id, + integrated_channel_code=self.enterprise_configuration.channel_code(), + content_id=content_id, + ).first() + if past_deleted_transmission: + self._log_info( + 'Found previously deleted content record while creating record. ' + 'Marking record as active.', + course_or_course_run_key=content_id + ) + past_deleted_transmission.prepare_to_recreate(content_last_changed, enterprise_customer_catalog.uuid) + items_to_create[content_id] = past_deleted_transmission + else: + incomplete_transmission = ContentMetadataItemTransmission.incomplete_create_transmissions( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + plugin_configuration_id=self.enterprise_configuration.id, + integrated_channel_code=self.enterprise_configuration.channel_code(), + content_id=content_id, + ).first() + if incomplete_transmission: + incomplete_transmission.mark_for_create() + items_to_create[content_id] = incomplete_transmission + else: + new_transmission = ContentMetadataItemTransmission( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + content_id=content_id, + channel_metadata=None, + content_last_changed=content_last_changed, + enterprise_customer_catalog_uuid=enterprise_customer_catalog.uuid, + plugin_configuration_id=self.enterprise_configuration.id, + marked_for=TRANSMISSION_MARK_CREATE + ) + new_transmission.save() + items_to_create[content_id] = new_transmission + return items_to_create + + def _get_catalog_diff( + self, + enterprise_catalog, + content_keys, + force_retrieve_all_catalogs, + max_item_count + ): + """ + From the enterprise catalog API, request a catalog diff based off of a list of content keys. Using the diff, + retrieve past content metadata transmission records for update and delete payloads. + """ + items_to_create, items_to_delete, matched_items = self.enterprise_catalog_api.get_catalog_diff( + enterprise_catalog, + content_keys + ) + + # Fetch all existing, non-deleted transmission audit content keys for the customer/configuration + existing_content_keys = set(self._get_catalog_content_keys()) + unique_new_items_to_create = [] + + # We need to remove any potential create transmissions if the content already exists on the customer's instance + # under a different catalog + for item in items_to_create: + # If the catalog system has indicated that the content is new and needs creating, we need to check if the + # content already exists on the customer's instance under a different catalog. If it does, we need to + # check if the content key exists as an orphaned transmission record for this customer and config, + # indicating that the content was previously created but then the config under which it was created was + # deleted. + content_key = item.get('content_key') + orphaned_content = self._get_customer_config_orphaned_content( + max_set_count=1, + content_key=content_key + ).first() + + # if it does exist as an orphaned content record: 1) don't add the item to the list of items to create, + # 2) swap the catalog uuid of the transmission audit associated with the orphaned record, and 3) mark the + # orphaned record resolved + if orphaned_content: + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission', + ) + ContentMetadataItemTransmission.objects.filter( + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + content_id=content_key + ).update( + enterprise_customer_catalog_uuid=enterprise_catalog.uuid + ) + + self._log_info( + 'Found an orphaned content record while creating. ' + 'Swapping catalog uuid and marking record as resolved.', + course_or_course_run_key=content_key + ) + orphaned_content.resolved = True + orphaned_content.save() + + # if the item to create doesn't exist as an orphaned piece of content, do all the normal checks + elif content_key not in existing_content_keys: + unique_new_items_to_create.append(item) + + content_to_create = self._check_matched_content_to_create( + enterprise_catalog, + unique_new_items_to_create + ) + content_to_update = self._check_matched_content_updated_at( + enterprise_catalog, + matched_items, + force_retrieve_all_catalogs + ) + content_to_delete = self._check_matched_content_to_delete( + enterprise_catalog, + items_to_delete + ) + + truncated_create, truncated_update, truncated_delete = truncate_item_dicts( + content_to_create, + content_to_update, + content_to_delete, + max_item_count + ) + + return truncated_create, truncated_update, truncated_delete + + def _check_matched_content_to_delete(self, enterprise_customer_catalog, items): + """ + Retrieve all past content metadata transmission records that have a `content_id` contained within a provided + list. + renamed from _retrieve_past_transmission_content + """ + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + + items_to_delete = {} + for item in items: + content_id = item.get('content_key') + + incomplete_transmission = ContentMetadataItemTransmission.incomplete_delete_transmissions( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + plugin_configuration_id=self.enterprise_configuration.id, + integrated_channel_code=self.enterprise_configuration.channel_code(), + content_id=content_id, + ).first() + + if incomplete_transmission: + self._log_info( + 'Found an unsent content delete record while deleting record. ' + 'Including record.', + course_or_course_run_key=content_id + ) + incomplete_transmission.mark_for_delete() + items_to_delete[content_id] = incomplete_transmission + else: + past_content_query = Q( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + enterprise_customer_catalog_uuid=enterprise_customer_catalog.uuid, + plugin_configuration_id=self.enterprise_configuration.id, + content_id=content_id + ) + + past_content_query.add( + Q(remote_errored_at__lt=self.LAST_24_HRS) | Q(remote_errored_at__isnull=True) | + Q(remote_errored_at__lt=self.enterprise_customer.modified), Q.AND) + past_content = ContentMetadataItemTransmission.objects.filter( + past_content_query).first() + if past_content: + past_content.mark_for_delete() + items_to_delete[content_id] = past_content + else: + self._log_info( + 'Could not find a content record while deleting record. ' + 'Skipping record.', + course_or_course_run_key=content_id + ) + return items_to_delete + + def _get_customer_config_orphaned_content(self, max_set_count, content_key=None): + """ + Helper method to retrieve the customer's orphaned content metadata items. + """ + OrphanedContentTransmissions = apps.get_model( + 'integrated_channel', + 'OrphanedContentTransmissions' + ) + content_query = Q(content_id=content_key) if content_key else Q() + base_query = Q( + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + resolved=False, + ) & content_query + + # Grab orphaned content metadata items for the customer, ordered by oldest to newest + orphaned_content = OrphanedContentTransmissions.objects.filter(base_query) + ordered_and_chunked_orphaned_content = orphaned_content.order_by('created')[:max_set_count] + return ordered_and_chunked_orphaned_content + + def _sanitize_and_set_item_metadata(self, item, metadata, action): + """ + Helper method to sanitize and set the metadata of an audit record according to + the provided action being performed on the item. + """ + metadata_transformed_for_exec_ed = self._transform_exec_ed_content(metadata) + transformed_item = self._transform_item(metadata_transformed_for_exec_ed, action=action) + + item.channel_metadata = transformed_item + item.content_title = metadata.get('title') + item.content_last_changed = metadata.get('content_last_modified') + item.save() + + def export(self, **kwargs): + """ + Export transformed content metadata if there has been an update to the consumer's catalogs + """ + enterprise_customer_catalogs = self.enterprise_configuration.customer_catalogs_to_transmit or \ + self.enterprise_customer.enterprise_customer_catalogs.all() + + # a maximum number of changes/payloads to export at once + # default to something huge to simplify logic, the max system int size + max_payload_count = kwargs.get('max_payload_count', sys.maxsize) + + self._log_info( + f'Beginning export for customer: {self.enterprise_customer.uuid}, with ' + f'max_payload_count of {max_payload_count}, and catalogs: ' + f'{enterprise_customer_catalogs}' + ) + + create_payload = {} + update_payload = {} + delete_payload = {} + key_to_content_metadata_mapping = {} + for enterprise_customer_catalog in enterprise_customer_catalogs: + + # if we're already at the max in a multi-catalog situation, break out + if len(create_payload) + len(update_payload) + len(delete_payload) >= max_payload_count: + self._log_info(f'Reached max_payload_count of {max_payload_count} breaking.') + break + + content_keys = self._get_catalog_content_keys(enterprise_customer_catalog) + + self._log_info( + f'Retrieved {len(content_keys)} content keys for past transmissions to customer: ' + f'{self.enterprise_customer.uuid} under catalog: {enterprise_customer_catalog.uuid}.' + ) + + # From the saved content records, use the enterprise catalog API to determine what needs sending + items_to_create, items_to_update, items_to_delete = self._get_catalog_diff( + enterprise_customer_catalog, + content_keys, + kwargs.get('force_retrieve_all_catalogs', False), + max_payload_count + ) + + content_keys_filter = list(items_to_create.keys()) + list(items_to_update.keys()) + if content_keys_filter: + content_metadata_items = self.enterprise_catalog_api.get_content_metadata( + self.enterprise_customer, + [enterprise_customer_catalog], + content_keys_filter, + ) + key_to_content_metadata_mapping = { + get_content_metadata_item_id(item): item for item in content_metadata_items + } + for key, item in items_to_create.items(): + try: + self._sanitize_and_set_item_metadata(item, key_to_content_metadata_mapping[key], IC_CREATE_ACTION) + except Exception as exc: + self._log_exception( + f'Failed to sanitize and set item metadata for item: {item}, with content key: ' + f'{key}, and content metadata: {key_to_content_metadata_mapping[key]}, action: ' + f'{IC_CREATE_ACTION}. Exception: {exc}' + ) + raise exc + + # Sanity check + item.enterprise_customer_catalog_uuid = enterprise_customer_catalog.uuid + item.save() + + create_payload[key] = item + for key, item in items_to_update.items(): + try: + self._sanitize_and_set_item_metadata(item, key_to_content_metadata_mapping[key], IC_UPDATE_ACTION) + except Exception as exc: + self._log_exception( + f'Failed to sanitize and set item metadata for item: {item}, with content key: ' + f'{key}, and content metadata: {key_to_content_metadata_mapping[key]}, action: ' + f'{IC_UPDATE_ACTION}. Exception: {exc}' + ) + raise exc + + # Sanity check + item.enterprise_customer_catalog_uuid = enterprise_customer_catalog.uuid + item.save() + + update_payload[key] = item + for key, item in items_to_delete.items(): + metadata = self._apply_delete_transformation(item.channel_metadata) + item.channel_metadata = metadata + item.save() + delete_payload[key] = item + + # If we're not at the max payload count, we can check for orphaned content and shove it in the delete payload + current_payload_count = len(create_payload) + len(update_payload) + len(delete_payload) + + self._log_info( + f'Exporter finished iterating over catalogs for customer: {self.enterprise_customer.uuid},' + f'with a current payload count of: {current_payload_count}. Is there room for orphaned content' + f'in the exporter payload?: {current_payload_count < max_payload_count}' + ) + + if current_payload_count < max_payload_count: + self._log_info( + f'Exporter has {max_payload_count - current_payload_count} slots left in the payload for customer: ' + f'{self.enterprise_customer.uuid}, searching for orphaned content to append' + ) + + space_left_in_payload = max_payload_count - current_payload_count + orphaned_content_to_delete = self._get_customer_config_orphaned_content( + max_set_count=space_left_in_payload, + ) + + for orphaned_item in orphaned_content_to_delete: + # log the content that would have been deleted because it's orphaned + self._log_info( + f'Exporter intends to delete orphaned content for customer: {self.enterprise_customer.uuid}, ' + f'config {self.enterprise_configuration.channel_code}-{self.enterprise_configuration} with ' + f'content_id: {orphaned_item.content_id}' + ) + if getattr(settings, "ALLOW_ORPHANED_CONTENT_REMOVAL", False): + delete_payload[orphaned_item.content_id] = orphaned_item.transmission + + self._log_info( + f'Exporter finished for customer: {self.enterprise_customer.uuid} with payloads- create_payload: ' + f'{create_payload}, update_payload: {update_payload}, delete_payload: {delete_payload}' + ) + + # collections of ContentMetadataItemTransmission objects + return create_payload, update_payload, delete_payload + + def _apply_delete_transformation(self, metadata): + """ + Base implementation of a delete transformation method. This method is designed to be a NOOP as it is up to the + individual channel to define specific transformations for the channel metadata payload of transmissions + intended for deletion. + """ + return metadata + + def _transform_exec_ed_content(self, content): + """ + Transform only executive education course type content to add executive education identifying tags to both the + title and description of the content + """ + if content.get('course_type') == EXEC_ED_COURSE_TYPE: + if title := content.get('title'): + content['title'] = "ExecEd: " + title + if description := content.get('full_description'): + content['full_description'] = EXEC_ED_CONTENT_DESCRIPTION_TAG + description + return content + + def _transform_item(self, content_metadata_item, action): + """ + Transform the provided content metadata item to the schema expected by the integrated channel. + """ + content_metadata_type = content_metadata_item['content_type'] + transformed_item = {} + for integrated_channel_schema_key, edx_data_schema_key in self.DATA_TRANSFORM_MAPPING.items(): + # Look for transformer functions defined on subclasses. + # Favor content type-specific functions. + transformer = ( + getattr(self, f'transform_{content_metadata_type}_{edx_data_schema_key}', None) + or + getattr(self, f'transform_{edx_data_schema_key}', None) + ) + + transformer_for_action = ( + getattr(self, f'transform_for_action_{content_metadata_type}_{edx_data_schema_key}', None) + or + getattr(self, f'transform_for_action_{edx_data_schema_key}', None) + ) + + # pylint: disable=not-callable + if transformer: + transformed_value = transformer(content_metadata_item) + elif transformer_for_action: + transformed_value = transformer_for_action(content_metadata_item, action) + else: + # The concrete subclass does not define an override for the given field, + # so just use the data key to index the content metadata item dictionary. + try: + transformed_value = content_metadata_item[edx_data_schema_key] + except KeyError: + # There may be a problem with the DATA_TRANSFORM_MAPPING on + # the concrete subclass or the concrete subclass does not implement + # the appropriate field transformer function. + self._log_exception( + f'Failed to transform content metadata item field {edx_data_schema_key} ' + f'for {self.enterprise_customer.name}: {content_metadata_item}' + ) + continue + + if transformed_value is None and self.SKIP_KEY_IF_NONE: + continue + transformed_item[integrated_channel_schema_key] = transformed_value + + return transformed_item + + def update_content_transmissions_catalog_uuids(self): + """ + Retrieve all content under the enterprise customer's catalog(s) and update all past transmission audits to have + it's associated catalog uuid. + """ + enterprise_customer_catalogs = self.enterprise_configuration.customer_catalogs_to_transmit or \ + self.enterprise_customer.enterprise_customer_catalogs.all() + + for enterprise_customer_catalog in enterprise_customer_catalogs: + content_metadata_items = self.enterprise_catalog_api.get_content_metadata( + self.enterprise_customer, + [enterprise_customer_catalog] + ) + content_ids = [get_content_metadata_item_id(item) for item in content_metadata_items] + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + transmission_items = ContentMetadataItemTransmission.objects.filter( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + content_id__in=content_ids + ) + self._log_info( + f'Found {len(transmission_items)} past content transmissions that need to be updated with their ' + f'respective catalog (catalog: {enterprise_customer_catalog.uuid}) UUIDs' + ) + for item in transmission_items: + item.enterprise_customer_catalog_uuid = enterprise_customer_catalog.uuid + item.save() diff --git a/channel_integrations/integrated_channel/exporters/learner_data.py b/channel_integrations/integrated_channel/exporters/learner_data.py new file mode 100644 index 0000000..4bcc82a --- /dev/null +++ b/channel_integrations/integrated_channel/exporters/learner_data.py @@ -0,0 +1,842 @@ +""" +Assist integrated channels with retrieving learner completion data. + +Module contains resources for integrated pipelines to retrieve all the +grade and completion data for enrollments belonging to a particular +enterprise customer. +""" + +from datetime import datetime +from logging import getLogger + +from opaque_keys import InvalidKeyError +from requests.exceptions import HTTPError + +from django.apps import apps +from django.contrib import auth +from django.utils import timezone +from django.utils.dateparse import parse_datetime + +from channel_integrations.integrated_channel.channel_settings import ChannelSettingsMixin + +try: + from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +except ImportError: + CourseOverview = None +from consent.models import DataSharingConsent +from enterprise.api_client.lms import GradesApiClient +from enterprise.models import EnterpriseCourseEnrollment +from channel_integrations.catalog_service_utils import get_course_id_for_enrollment +from channel_integrations.integrated_channel.exporters import Exporter +from channel_integrations.lms_utils import ( + get_completion_summary, + get_course_certificate, + get_course_details, + get_persistent_grade, + get_single_user_grade, +) +from channel_integrations.utils import ( + generate_formatted_log, + is_already_transmitted, + is_course_completed, + parse_datetime_to_epoch_millis, +) + +LOGGER = getLogger(__name__) +User = auth.get_user_model() + + +class LearnerExporter(ChannelSettingsMixin, Exporter): + """ + Base class for exporting learner completion data to integrated channels. + """ + + GRADE_AUDIT = 'Audit' + GRADE_PASSING = 'Pass' + GRADE_FAILING = 'Fail' + GRADE_INCOMPLETE = 'In Progress' + + def __init__(self, user, enterprise_configuration): + """ + Store the data needed to export the learner data to the integrated channel. + + Arguments: + + * ``user``: User instance with access to the Grades API for the Enterprise Customer's courses. + * ``enterprise_configuration``: EnterpriseCustomerPluginConfiguration instance for the current channel. + """ + # The Grades API and Certificates API clients require an OAuth2 access token, + # so cache the client to allow the token to be reused. Cache other clients for + # general reuse. + self.grades_api = None + self.certificates_api = None + self.course_api = None + self.course_enrollment_api = None + + super().__init__(user, enterprise_configuration) + + @property + def grade_passing(self): + """ + Returns the string used for a passing grade. + """ + return self.GRADE_PASSING + + @property + def grade_failing(self): + """ + Returns the string used for a failing grade. + """ + return self.GRADE_FAILING + + @property + def grade_incomplete(self): + """ + Returns the string used for an incomplete course grade. + """ + return self.GRADE_INCOMPLETE + + @property + def grade_audit(self): + """ + Returns the string used for an audit course grade. + """ + return self.GRADE_AUDIT + + def bulk_assessment_level_export(self): + """ + Collect assessment level learner data for the ``EnterpriseCustomer`` where data sharing consent is granted. + + Yields a ``LearnerDataTransmissionAudit`` for each subsection in a course under an enrollment, containing: + + * ``enterprise_course_enrollment_id``: The id reference to the ``EnterpriseCourseEnrollment`` object. + * ``course_id``: The string ID of the course under the enterprise enrollment. + * ``subsection_id``: The string ID of the subsection within the course. + * ``grade``: string grade recorded for the learner in the course. + """ + enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related( + 'enterprise_customer_user' + ).filter( + enterprise_customer_user__enterprise_customer=self.enterprise_customer, + enterprise_customer_user__active=True, + ).order_by('course_id') + + # Create a record of each subsection from every enterprise enrollment + for enterprise_enrollment in enrollment_queryset: + if not LearnerExporter.has_data_sharing_consent(enterprise_enrollment): + continue + + assessment_grade_data = self._collect_assessment_grades_data(enterprise_enrollment) + + records = self.get_learner_assessment_data_records( + enterprise_enrollment=enterprise_enrollment, + assessment_grade_data=assessment_grade_data, + ) + if records: + # There are some cases where we won't receive a record from the above + # method; right now, that should only happen if we have an Enterprise-linked + # user for the integrated channel, and transmission of that user's + # data requires an upstream user identifier that we don't have (due to a + # failure of SSO or similar). In such a case, `get_learner_data_record` + # would return None, and we'd simply skip yielding it here. + yield from records + + def single_assessment_level_export(self, **kwargs): + """ + Collect an assessment level learner data for the ``EnterpriseCustomer`` where data sharing consent is + granted. + + Yields a ``LearnerDataTransmissionAudit`` for each subsection of the course that the learner is enrolled in, + containing: + + * ``enterprise_course_enrollment_id``: The id reference to the ``EnterpriseCourseEnrollment`` object. + * ``course_id``: The string ID of the course under the enterprise enrollment. + * ``subsection_id``: The string ID of the subsection within the course. + * ``grade``: string grade recorded for the learner in the course. + * ``learner_to_transmit``: REQUIRED User object, representing the learner whose data is being exported. + + """ + lms_user_for_filter = kwargs.get('learner_to_transmit') + TransmissionAudit = kwargs.get('TransmissionAudit', None) + course_run_id = kwargs.get('course_run_id', None) + grade = kwargs.get('grade', None) + subsection_id = kwargs.get('subsection_id') + enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related( + 'enterprise_customer_user' + ).filter( + enterprise_customer_user__active=True, + enterprise_customer_user__user_id=lms_user_for_filter.id, + course_id=course_run_id, + ).order_by('course_id') + + # We are transmitting for an enrollment, so grab just the one. + enterprise_enrollment = enrollment_queryset.first() + + if not enterprise_enrollment: + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + self.enterprise_configuration.enterprise_customer.uuid, + lms_user_for_filter, + course_run_id, + 'Either qualifying enrollments not found for learner, or, ' + 'enterprise_customer_user record is inactive. Skipping transmit assessment grades.' + )) + return + + already_transmitted = is_already_transmitted( + TransmissionAudit, + enterprise_enrollment.id, + self.enterprise_configuration.id, + grade, + subsection_id, + detect_grade_updated=self.INCLUDE_GRADE_FOR_COMPLETION_AUDIT_CHECK, + ) + + if not (TransmissionAudit and already_transmitted) and LearnerExporter.has_data_sharing_consent( + enterprise_enrollment): + + assessment_grade_data = self._collect_assessment_grades_data(enterprise_enrollment) + + records = self.get_learner_assessment_data_records( + enterprise_enrollment=enterprise_enrollment, + assessment_grade_data=assessment_grade_data, + ) + if records: + # There are some cases where we won't receive a record from the above + # method; right now, that should only happen if we have an Enterprise-linked + # user for the integrated channel, and transmission of that user's + # data requires an upstream user identifier that we don't have (due to a + # failure of SSO or similar). In such a case, `get_learner_data_record` + # would return None, and we'd simply skip yielding it here. + yield from records + + @staticmethod + def has_data_sharing_consent(enterprise_enrollment): + """ + Helper method to determine whether an enrollment has data sharing consent or not. + """ + consent = DataSharingConsent.objects.proxied_get( + username=enterprise_enrollment.enterprise_customer_user.username, + course_id=enterprise_enrollment.course_id, + enterprise_customer=enterprise_enrollment.enterprise_customer_user.enterprise_customer + ) + if consent.granted and not enterprise_enrollment.audit_reporting_disabled: + return True + + return False + + def _determine_enrollments_permitted( + self, + lms_user_for_filter, + course_run_id, + channel_name, + skip_transmitted, + TransmissionAudit, + grade, + ): + """ + Determines which enrollments can be safely transmitted after checking + * enrollments that are already transmitted + * enrollments that are permitted to be transmitted by selecting enrollments for which: + * - data sharing consent is granted + * - audit_reporting is enabled (via enterprise level switch) + """ + enrollments_to_process = self.get_enrollments_to_process( + lms_user_for_filter, + course_run_id, + channel_name, + ) + + if TransmissionAudit and skip_transmitted: + untransmitted_enrollments = self._filter_out_pre_transmitted_enrollments( + enrollments_to_process, + channel_name, + grade, + TransmissionAudit + ) + else: + untransmitted_enrollments = enrollments_to_process + + # filter out enrollments which don't allow channel_integrations grade transmit + enrollments_permitted = set() + for enrollment in untransmitted_enrollments: + if (not LearnerExporter.has_data_sharing_consent(enrollment) or + enrollment.audit_reporting_disabled): + continue + enrollments_permitted.add(enrollment) + return enrollments_permitted + + def export_unique_courses(self): + """ + Retrieve and export all unique course ID's from an enterprise customer's learner enrollments. + """ + enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related( + 'enterprise_customer_user' + ).filter( + enterprise_customer_user__enterprise_customer=self.enterprise_customer, + enterprise_customer_user__active=True, + ).order_by('course_id') + return {get_course_id_for_enrollment(enrollment) for enrollment in enrollment_queryset} + + def get_grades_summary( + self, + course_details, + enterprise_enrollment, + channel_name, + incomplete_count=None, + ): + ''' + Fetch grades info using either certificate api, or grades api. + Note: This logic is going to be refactored, so that audit enrollments are treated separately + - For audit enrollments, currently will fetch using grades api, + - For non audit, it still calls grades api if pacing !=instructor otherwise calls certificate api + This pacing logic needs cleanup for a more accurate piece of logic since pacing should not be relevant + + Returns: tuple with values: + completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent, passed_timestamp + ''' + is_audit_enrollment = enterprise_enrollment.is_audit_enrollment + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + enterprise_customer_uuid = enterprise_enrollment.enterprise_customer_user.enterprise_customer.uuid + course_id = enterprise_enrollment.course_id + + if is_audit_enrollment: + completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent, passed_timestamp = \ + self.collect_grades_data(enterprise_enrollment, course_details, channel_name) + if incomplete_count == 0 and completed_date_from_api is None: + LOGGER.info(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + 'Setting completed_date to now() for audit course with all non-gated content done.' + )) + completed_date_from_api = timezone.now() + else: + completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent, passed_timestamp = \ + self.collect_certificate_data(enterprise_enrollment, channel_name) + if completed_date_from_api is None: + # means we cannot find a cert for this learner + # we will try getting grades info using the alternative api in this case + # if that also does not exist then we have nothing to report + completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent, passed_timestamp = \ + self.collect_grades_data(enterprise_enrollment, course_details, channel_name) + + # In the past we have been inconsistent about the format/source/typing of the grade_percent value. + # Initial investigations have lead us to believe that grade percents from the source are seemingly more + # accurate now, so we're enforcing float typing. To account for the chance that old documentation is + # right and grade percents are reported as letter grade values, the float conversion is wrapped in a try/except + # in order to both not blow up and also log said instance of letter grade values. + try: + if grade_percent is not None: + grade_percent = float(grade_percent) + except ValueError as exc: + LOGGER.error(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + f'Grade percent is not a valid float: {grade_percent}. Failed with exc: {exc}' + )) + grade_percent = None + + return completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent, passed_timestamp + + def get_incomplete_content_count(self, enterprise_enrollment): + ''' + Fetch incomplete content count using completion blocks LMS api + Will return None for non audit enrollment (but this does not have to be the case necessarily) + ''' + incomplete_count = None + is_audit_enrollment = enterprise_enrollment.is_audit_enrollment + + # The decision to not get incomplete count for non audit enrollments can be questioned + # Right now we don't use this number for non audit. But this condition can be just removed + # if we decide we need this for non audit enrollments too + if not is_audit_enrollment: + return incomplete_count + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + course_id = enterprise_enrollment.course_id + + user = User.objects.get(pk=lms_user_id) + completion_summary = get_completion_summary(course_id, user) + incomplete_count = completion_summary.get('incomplete_count') + + return incomplete_count + + def export(self, **kwargs): + """ + Collect learner data for the ``EnterpriseCustomer`` where data sharing consent is granted. + If BOTH learner_to_transmit and course_run_id are present, collected data returned is narrowed to + that learner and course. If either param is absent or None, ALL data will be collected. + + Yields a learner data object for each enrollment, containing: + + * ``enterprise_enrollment``: ``EnterpriseCourseEnrollment`` object. + * ``user_email``: PII User/learner email string + * ``content_title``: Course title string + * ``completed_date``: datetime instance containing the course/enrollment completion date; None if not complete. + "Course completion" occurs for instructor-paced courses when course certificates are issued, and + for self-paced courses, when the course end date is passed, or when the learner achieves a passing grade. + Currently unused as it gets overridden from the grader data. + * ``grade``: string grade recorded for the learner in the course. + * ``learner_to_transmit``: OPTIONAL User, filters exported data + * ``course_run_id``: OPTIONAL Course key string, filters exported data + + """ + channel_name = kwargs.get('app_label') + lms_user_for_filter = kwargs.get('learner_to_transmit', None) + course_run_id = kwargs.get('course_run_id', None) + grade = kwargs.get('grade', None) + skip_transmitted = kwargs.get('skip_transmitted', True) + TransmissionAudit = kwargs.get('TransmissionAudit', None) + + # Fetch the consenting enrollment data, including the enterprise_customer_user. + # Order by the course_id, to avoid fetching course API data more than we have to. + enrollments_permitted = self._determine_enrollments_permitted( + lms_user_for_filter, + course_run_id, + channel_name, + skip_transmitted, + TransmissionAudit, + grade, + ) + enrollment_ids_to_export = [enrollment.id for enrollment in enrollments_permitted] + + for enterprise_enrollment in enrollments_permitted: + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + user_email = enterprise_enrollment.enterprise_customer_user.user_email + enterprise_customer_uuid = enterprise_enrollment.enterprise_customer_user.enterprise_customer.uuid + course_id = enterprise_enrollment.course_id + + course_details, error_message = LearnerExporterUtility.get_course_details_by_id(course_id) + + if course_details is None: + # Course not found, so we have nothing to report. + LOGGER.error(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + f'get_course_details returned None for EnterpriseCourseEnrollment {enterprise_enrollment.pk}' + f', error_message: {error_message}' + )) + continue + + # For audit courses, check if 100% completed + # which we define as: no non-gated content is remaining + incomplete_count = self.get_incomplete_content_count(enterprise_enrollment) + + ( + completed_date_from_api, grade_from_api, + is_passing_from_api, grade_percent, passed_timestamp + ) = self.get_grades_summary( + course_details, + enterprise_enrollment, + channel_name, + incomplete_count, + ) + + if completed_date_from_api: + if is_passing_from_api: + progress_status = 'Passed' + else: + progress_status = 'Failed' + else: + progress_status = 'In Progress' + + # Apply the Source of Truth for Grades + # Note: Only completed records are transmitted by the completion transmitter + # therefore even non complete grading/cert records are exported here. + _is_course_completed = is_course_completed( + enterprise_enrollment, + is_passing_from_api, + incomplete_count, + passed_timestamp, + ) + records = self.get_learner_data_records( + enterprise_enrollment=enterprise_enrollment, + user_email=user_email, + completed_date=completed_date_from_api, + grade=grade_from_api, + content_title=course_details.display_name, + progress_status=progress_status, + course_completed=_is_course_completed, + grade_percent=grade_percent, + ) + + if records: + # There are some cases where we won't receive a record from the above + # method; right now, that should only happen if we have an Enterprise-linked + # user for the integrated channel, and transmission of that user's + # data requires an upstream user identifier that we don't have (due to a + # failure of SSO or similar). In such a case, `get_learner_data_record` + # would return None, and we'd simply skip yielding it here. + for record in records: + # Because we export a course and course run under the same enrollment, we can only remove the + # enrollment from the list of enrollments to export, once. + try: + enrollment_ids_to_export.pop(enrollment_ids_to_export.index(enterprise_enrollment.id)) + except ValueError: + pass + + yield record + + def _filter_out_pre_transmitted_enrollments( + self, + enrollments_to_process, + channel_name, + grade, + transmission_audit + ): + """ + Given an enrollments_to_process, returns only enrollments that are not already transmitted + """ + included_enrollments = set() + for enterprise_enrollment in enrollments_to_process: + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + enterprise_customer_uuid = enterprise_enrollment.enterprise_customer_user.enterprise_customer.uuid + course_id = enterprise_enrollment.course_id + + if transmission_audit and \ + is_already_transmitted( + transmission_audit, + enterprise_enrollment.id, + self.enterprise_configuration.id, + grade, + detect_grade_updated=self.INCLUDE_GRADE_FOR_COMPLETION_AUDIT_CHECK, + ): + # We've already sent a completion status for this enrollment + LOGGER.info(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + 'Skipping export of previously sent enterprise enrollment. ' + 'EnterpriseCourseEnrollment: {enterprise_enrollment_id}'.format( + enterprise_enrollment_id=enterprise_enrollment.id + ))) + continue + included_enrollments.add(enterprise_enrollment) + return included_enrollments + + def get_enrollments_to_process(self, lms_user_for_filter, course_run_id, channel_name): + """ + Fetches list of EnterpriseCourseEnrollments ordered by course_id. + List is filtered by learner and course_run_id if both are provided + + lms_user_for_filter: If None, data for ALL courses and learners will be returned + course_run_id: If None, data for ALL courses and learners will be returned + + """ + enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related( + 'enterprise_customer_user' + ).filter( + enterprise_customer_user__enterprise_customer=self.enterprise_customer, + enterprise_customer_user__active=True, + ) + if lms_user_for_filter and course_run_id: + enrollment_queryset = enrollment_queryset.filter( + course_id=course_run_id, + enterprise_customer_user__user_id=lms_user_for_filter.id, + ) + LOGGER.info(generate_formatted_log( + channel_name, self.enterprise_customer.uuid, lms_user_for_filter, course_run_id, + 'get_enrollments_to_process run for single learner and course.')) + enrollment_queryset = enrollment_queryset.order_by('course_id') + # return resolved list instead of queryset + return list(enrollment_queryset) + + def get_learner_assessment_data_records( + self, + enterprise_enrollment, + assessment_grade_data + ): + """ + Generate a learner assessment data transmission audit with fields properly filled in. + Returns a list of LearnerDataTransmissionAudit objects. + + enterprise_enrollment: the ``EnterpriseCourseEnrollment`` object we are getting a learner's data for. + assessment_grade_data: A dict with keys corresponding to different edX course subsections. + See _collect_assessment_grades_data for the formatted data returned as the value for a given key. + """ + TransmissionAudit = apps.get_model('integrated_channel', 'GenericLearnerDataTransmissionAudit') + user_subsection_audits = [] + # Create an audit for each of the subsections in the course data. + for subsection_data in assessment_grade_data.values(): + subsection_percent_grade = subsection_data.get('grade') + subsection_id = subsection_data.get('subsection_id') + # Sanity check for a grade to report + if not subsection_percent_grade or not subsection_id: + continue + + user_subsection_audits.append(TransmissionAudit( + plugin_configuration_id=self.enterprise_configuration.id, + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + enterprise_course_enrollment_id=enterprise_enrollment.id, + course_id=enterprise_enrollment.course_id, + subsection_id=subsection_id, + grade=subsection_percent_grade, + )) + + return user_subsection_audits + + def get_learner_data_records( + self, + enterprise_enrollment, + completed_date=None, + grade=None, + course_completed=False, + grade_percent=None, + content_title=None, + user_email=None, + progress_status=None, + + ): # pylint: disable=unused-argument + """ + Generate a learner data transmission audit with fields properly filled in. + """ + TransmissionAudit = apps.get_model('integrated_channel', 'GenericLearnerDataTransmissionAudit') + completed_timestamp = None + if completed_date is not None: + completed_timestamp = parse_datetime_to_epoch_millis(completed_date) + course_id = get_course_id_for_enrollment(enterprise_enrollment) + # We only want to send one record per enrollment and course, so we check if one exists first. + learner_transmission_record = TransmissionAudit.objects.filter( + enterprise_course_enrollment_id=enterprise_enrollment.id, + course_id=course_id, + ).first() + if learner_transmission_record is None: + learner_transmission_record = TransmissionAudit( + plugin_configuration_id=self.enterprise_configuration.id, + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + enterprise_course_enrollment_id=enterprise_enrollment.id, + course_id=course_id, + course_completed=course_completed, + completed_timestamp=completed_timestamp, + grade=grade, + user_email=user_email, + content_title=content_title, + progress_status=progress_status, + ) + # We return one record here, with the course key, that was sent to the integrated channel. + return [learner_transmission_record] + + def collect_certificate_data(self, enterprise_enrollment, channel_name): + """ + Collect the learner completion data from the course certificate. + + Used for Instructor-paced courses. + + If no certificate is found, then returns the completed_date = None, grade = In Progress, on the idea that a + certificate will eventually be generated. + + Args: + enterprise_enrollment (EnterpriseCourseEnrollment): the enterprise enrollment record for which we need to + collect completion/grade data, + channel_name: labeled for relevant integrated channel this is being called for to enhance logging. + + Returns: + completed_date: Date the course was completed, this is None if course has not been completed. + grade: Current grade in the course. + percent_grade: The current percent grade in the course. + is_passing: Boolean indicating if the grade is a passing grade or not. + passed_timestamp: Timestamp when learner obtained passing grade. + """ + + course_id = enterprise_enrollment.course_id + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + enterprise_customer_uuid = enterprise_enrollment.enterprise_customer_user.enterprise_customer.uuid + user = User.objects.get(pk=lms_user_id) + passed_timestamp = None + + completed_date = None + grade = self.grade_incomplete + is_passing = False + percent_grade = None + + try: + certificate = get_course_certificate(course_id, user) + except InvalidKeyError: + certificate = None + LOGGER.error(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + 'get_course_certificate failed. Certificate fetch failed due to invalid course_id for' + f' EnterpriseCourseEnrollment: {enterprise_enrollment}. Data export will continue without grade.' + )) + + if not certificate: + return completed_date, grade, is_passing, percent_grade, passed_timestamp + + LOGGER.info(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + f'get_course_certificate certificate={certificate}' + )) + + # get_certificate_for_user has an optional formatting argument which defaults to True + # edx-platform/blob/6b9eb122dd5b018cfeffc120a70e503b2c159c0b/lms/djangoapps/certificates/api.py#L128 + # when True, the certifiate's `created_date` is transformed into just `created` + completed_date = certificate.get('created', None) + # guard against the default formatting argument changing + if not completed_date: + completed_date = certificate.get('created_date', None) + if completed_date: + completed_date = parse_datetime(str(completed_date)) + + # also get passed_timestamp which is used to line up completion logic with analytics + persistent_grade = get_persistent_grade(course_id, user) + passed_timestamp = persistent_grade.passed_timestamp if persistent_grade is not None else None + + if (not completed_date) and passed_timestamp: + LOGGER.info(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + 'get_course_certificate misisng created/created_date, ' + 'but there is a passed_timestamp so using that' + )) + completed_date = datetime.fromtimestamp((passed_timestamp / 1000), tz=timezone.utc) + elif not completed_date and not passed_timestamp: + LOGGER.info(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + 'get_course_certificate misisng created/created_date, ' + 'no passed_timestamp so defaulting to timezone.now(), ' + f'certificate={certificate}' + )) + completed_date = timezone.now() + + # For consistency with _collect_grades_data, we only care about Pass/Fail grades. This could change. + is_passing = certificate.get('is_passing') + percent_grade = certificate.get('grade') + grade = self.grade_passing if is_passing else self.grade_failing + + return completed_date, grade, is_passing, percent_grade, passed_timestamp + + def _collect_assessment_grades_data(self, enterprise_enrollment): + """ + Collect a learner's assessment level grade data using an enterprise enrollment, from the Grades API. + + Args: + enterprise_enrollment (EnterpriseCourseEnrollment): the enterprise enrollment record for which we need to + collect subsection grades data + Returns: + Dict: + { + [subsection name]: { + 'grade_category': category, + 'grade': percent grade, + 'assessment_label': label, + 'grade_point_score': points earned on the assignment, + 'grade_points_possible': max possible points on the assignment, + 'subsection_id': subsection module ID + } + + ... + } + """ + if self.grades_api is None: + self.grades_api = GradesApiClient(self.user) + + course_id = enterprise_enrollment.course_id + username = enterprise_enrollment.enterprise_customer_user.user.username + try: + assessment_grades_data = self.grades_api.get_course_assessment_grades(course_id, username) + except HTTPError as err: + if err.response and err.response.status_code == 404: + return {} + raise err + + assessment_grades = {} + for grade in assessment_grades_data: + if not grade.get('attempted'): + continue + assessment_grades[grade.get('subsection_name')] = { + 'grade_category': grade.get('category'), + 'grade': grade.get('percent'), + 'assessment_label': grade.get('label'), + 'grade_point_score': grade.get('score_earned'), + 'grade_points_possible': grade.get('score_possible'), + 'subsection_id': grade.get('module_id') + } + + return assessment_grades + + def collect_grades_data(self, enterprise_enrollment, course_details, channel_name): + """ + Collect the learner completion data from the Grades API. + + Used for self-paced courses. + + Args: + enterprise_enrollment (EnterpriseCourseEnrollment): the enterprise enrollment record for which we need to + collect completion/grade data + course_details (CourseOverview): the course details for the course in the enterprise enrollment record. + channel_name: Integrated channel name for improved logging. + + Returns: + completed_date: Date the course was completed, None if course has not been completed. + grade: Current grade in the course. + is_passing: Boolean indicating if the grade is a passing grade or not. + percent_grade: a number between 0 and 100 + passed_timestamp: Timestamp when learner obtained passing grade. + """ + + course_id = enterprise_enrollment.course_id + lms_user_id = enterprise_enrollment.enterprise_customer_user.user_id + user = User.objects.get(pk=lms_user_id) + enterprise_customer_uuid = enterprise_enrollment.enterprise_customer_user.enterprise_customer.uuid + + grades_data = get_single_user_grade(course_id, user) + + if grades_data is None: + LOGGER.warning(generate_formatted_log( + channel_name, enterprise_customer_uuid, lms_user_id, course_id, + f'No grade found for ' + f'EnterpriseCourseEnrollment: {enterprise_enrollment}.' + )) + # if enrollment found, but no grades, we can safely mark as incomplete/in progress + return None, LearnerExporter.GRADE_INCOMPLETE, None, None, None + + # also get passed_timestamp which is used to line up completion logic with analytics + persistent_grade = get_persistent_grade(course_id, user) + passed_timestamp = persistent_grade.passed_timestamp if persistent_grade is not None else None + + # Prepare to process the course end date and pass/fail grade + course_end_date = course_details.end + now = timezone.now() + is_passing = grades_data.passed + + # We can consider a course complete if: + # * the course's end date has passed + if course_end_date is not None and course_end_date < now: + completed_date = course_end_date + grade = self.grade_passing if is_passing else self.grade_failing + grade = self.grade_audit if enterprise_enrollment.is_audit_enrollment else grade + + # * Or, the learner has a passing grade (as of now) + elif is_passing: + completed_date = now + grade = self.grade_passing + + # Otherwise, the course is still in progress + else: + completed_date = None + grade = self.grade_incomplete + + percent_grade = grades_data.percent + + return completed_date, grade, is_passing, percent_grade, passed_timestamp + + +class LearnerExporterUtility: + """ Utility class namespace for accessing Django objects in a common way. """ + + @staticmethod + def lms_user_id_for_ent_course_enrollment_id(enterprise_course_enrollment_id): + """ Returns the ID of the LMS User for the EnterpriseCourseEnrollment id passed in + or None if EnterpriseCourseEnrollment not found """ + try: + return EnterpriseCourseEnrollment.objects.get( + id=enterprise_course_enrollment_id).enterprise_customer_user.user_id + except EnterpriseCourseEnrollment.DoesNotExist: + return None + + @staticmethod + def get_course_details_by_id(course_id): + ''' + Convenience method to fetch course details or None (if not found) + ''' + course_details = None + error_message = None + try: + course_details = get_course_details(course_id) + except (InvalidKeyError, CourseOverview.DoesNotExist): + error_message = f'get_course_details failed for course_id {course_id}' + + return course_details, error_message diff --git a/channel_integrations/integrated_channel/management/__init__.py b/channel_integrations/integrated_channel/management/__init__.py new file mode 100644 index 0000000..113cb83 --- /dev/null +++ b/channel_integrations/integrated_channel/management/__init__.py @@ -0,0 +1,3 @@ +""" +Enterprise Integrated Channel management commands and related functions. +""" diff --git a/channel_integrations/integrated_channel/management/commands/__init__.py b/channel_integrations/integrated_channel/management/commands/__init__.py new file mode 100644 index 0000000..a0bc4ab --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/__init__.py @@ -0,0 +1,159 @@ +""" +Enterprise Integrated Channel management commands. +""" + +from collections import OrderedDict + +from django.core.management.base import CommandError +from django.utils.translation import gettext as _ + +from enterprise.models import EnterpriseCustomer +from channel_integrations.blackboard.models import BlackboardEnterpriseCustomerConfiguration +from channel_integrations.canvas.models import CanvasEnterpriseCustomerConfiguration +from channel_integrations.cornerstone.models import CornerstoneEnterpriseCustomerConfiguration +from channel_integrations.degreed2.models import Degreed2EnterpriseCustomerConfiguration +from channel_integrations.degreed.models import DegreedEnterpriseCustomerConfiguration +from channel_integrations.moodle.models import MoodleEnterpriseCustomerConfiguration +from channel_integrations.sap_success_factors.models import SAPSuccessFactorsEnterpriseCustomerConfiguration + +# Mapping between the channel code and the channel configuration class +INTEGRATED_CHANNEL_CHOICES = OrderedDict([ + (integrated_channel_class.channel_code(), integrated_channel_class) + for integrated_channel_class in ( + BlackboardEnterpriseCustomerConfiguration, + CanvasEnterpriseCustomerConfiguration, + CornerstoneEnterpriseCustomerConfiguration, + Degreed2EnterpriseCustomerConfiguration, + MoodleEnterpriseCustomerConfiguration, + SAPSuccessFactorsEnterpriseCustomerConfiguration, + ) +]) + +ASSESSMENT_LEVEL_REPORTING_INTEGRATED_CHANNEL_CHOICES = OrderedDict([ + (integrated_channel_class.channel_code(), integrated_channel_class) + for integrated_channel_class in ( + BlackboardEnterpriseCustomerConfiguration, + CanvasEnterpriseCustomerConfiguration, + ) +]) + +CONTENT_METADATA_JOB_INTEGRATED_CHANNEL_CHOICES = OrderedDict([ + (integrated_channel_class.channel_code(), integrated_channel_class) + for integrated_channel_class in ( + BlackboardEnterpriseCustomerConfiguration, + CanvasEnterpriseCustomerConfiguration, + CornerstoneEnterpriseCustomerConfiguration, + Degreed2EnterpriseCustomerConfiguration, + MoodleEnterpriseCustomerConfiguration, + SAPSuccessFactorsEnterpriseCustomerConfiguration, + ) +]) + + +class IntegratedChannelCommandMixin: + """ + Contains common functionality for the IntegratedChannel management commands. + """ + + def add_arguments(self, parser): + """ + Adds the optional arguments: ``--enterprise_customer``, ``--channel`` + """ + parser.add_argument( + '--enterprise_customer', + dest='enterprise_customer', + default=None, + metavar='ENTERPRISE_CUSTOMER_UUID', + help=_('Transmit data for only this EnterpriseCustomer. ' + 'Omit this option to transmit to all EnterpriseCustomers with active integrated channels.'), + ) + parser.add_argument( + '--channel', + dest='channel', + default='', + metavar='INTEGRATED_CHANNEL', + help=_('Transmit data to this IntegrateChannel. ' + 'Omit this option to transmit to all configured, active integrated channels.'), + choices=list(INTEGRATED_CHANNEL_CHOICES.keys()), + ) + + def get_integrated_channels(self, options): + """ + Generates a list of active integrated channels for active customers, filtered from the given options. + + Raises errors when invalid options are encountered. + + Note: in order to retrieve in active configurations, the option `prevent_disabled_configurations` must be set to + false. For other available configurations, see ``add_arguments`` for all accepted options. + """ + assessment_level_support = options.get('assessment_level_support', False) + content_metadata_job_support = options.get('content_metadata_job_support', False) + channel_classes = self.get_channel_classes( + options.get('channel'), + assessment_level_support=assessment_level_support, + content_metadata_job_support=content_metadata_job_support, + ) + filter_kwargs = { + 'enterprise_customer__active': True, + } + if options.get('prevent_disabled_configurations', True): + filter_kwargs['active'] = True + + enterprise_customer = self.get_enterprise_customer(options.get('enterprise_customer')) + + if enterprise_customer: + filter_kwargs['enterprise_customer'] = enterprise_customer + + for channel_class in channel_classes: + yield from channel_class.objects.filter(**filter_kwargs) + + @staticmethod + def get_enterprise_customer(uuid): + """ + Returns the enterprise customer requested for the given uuid, None if not. + + Raises CommandError if uuid is invalid. + """ + if uuid is None: + return None + try: + return EnterpriseCustomer.active_customers.get(uuid=uuid) + except EnterpriseCustomer.DoesNotExist as no_customer_exception: + raise CommandError( + _('Enterprise customer {uuid} not found, or not active').format(uuid=uuid) + ) from no_customer_exception + + @staticmethod + def get_channel_classes(channel_code, assessment_level_support=False, content_metadata_job_support=False): + """ + Assemble a list of integrated channel classes to transmit to. + + If a valid channel type was provided, use it. + + Otherwise, use all the available channel types. + """ + if assessment_level_support: + channel_choices = ASSESSMENT_LEVEL_REPORTING_INTEGRATED_CHANNEL_CHOICES + elif content_metadata_job_support: + channel_choices = CONTENT_METADATA_JOB_INTEGRATED_CHANNEL_CHOICES + else: + channel_choices = INTEGRATED_CHANNEL_CHOICES + + if channel_code: + # Channel code is case-insensitive + channel_code = channel_code.upper() + + if channel_code not in channel_choices: + raise CommandError(_('Invalid integrated channel: {channel}').format(channel=channel_code)) + + channel_classes = [channel_choices[channel_code]] + else: + channel_classes = list(channel_choices.values()) + + return channel_classes + + +class IntegratedChannelCommandUtils(IntegratedChannelCommandMixin): + """ + This is a wrapper class to avoid using mixin in methods + """ diff --git a/channel_integrations/integrated_channel/management/commands/assign_skills_to_degreed_courses.py b/channel_integrations/integrated_channel/management/commands/assign_skills_to_degreed_courses.py new file mode 100644 index 0000000..6082b39 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/assign_skills_to_degreed_courses.py @@ -0,0 +1,128 @@ +""" +Assign skills to degreed courses +""" +from logging import getLogger + +from requests.exceptions import ConnectionError, RequestException, Timeout # pylint: disable=redefined-builtin + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError + +from enterprise.api_client.enterprise_catalog import EnterpriseCatalogApiClient +from channel_integrations.degreed2.client import Degreed2APIClient +from channel_integrations.exceptions import ClientError +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.utils import generate_formatted_log + +User = auth.get_user_model() +LOGGER = getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Add skill metadata to existing Degreed courses. + + ./manage.py lms assign_skills_to_degreed_courses + """ + + def add_arguments(self, parser): + """ + Add required arguments to the parser. + """ + parser.add_argument( + '--catalog_user', + dest='catalog_user', + required=True, + metavar='ENTERPRISE_CATALOG_API_USERNAME', + help='Use this user to access the Enterprise Catalog API.' + ) + super().add_arguments(parser) + + def _prepare_json_payload_for_skills_endpoint(self, course_skills): + """ + Prepares a json payload for skills in the Degreed expected format. + """ + course_skills_json = [] + for skill in course_skills: + skill_data = {"type": "skills", "id": skill} + course_skills_json.append(skill_data) + return { + "data": course_skills_json + } + + def handle(self, *args, **options): + """ + Update all existing Degreed courses to assign skills metadata. + """ + options['channel'] = 'DEGREED2' + username = options['catalog_user'] + + try: + user = User.objects.get(username=username) + except User.DoesNotExist as no_user_error: + raise CommandError('A user with the username {} was not found.'.format(username)) from no_user_error + + enterprise_catalog_client = EnterpriseCatalogApiClient(user) + channel_integrations = self.get_integrated_channels(options) + for degreed_channel_config in channel_integrations: + enterprise_customer = degreed_channel_config.enterprise_customer + enterprise_customer_catalogs = degreed_channel_config.customer_catalogs_to_transmit or \ + enterprise_customer.enterprise_customer_catalogs.all() + try: + content_metadata_in_catalogs = enterprise_catalog_client.get_content_metadata( + enterprise_customer, + enterprise_customer_catalogs + ) + except (RequestException, ConnectionError, Timeout) as exc: + LOGGER.exception( + 'Failed to retrieve enterprise catalogs content metadata due to: [%s]', str(exc) + ) + continue + + degreed_client = Degreed2APIClient(degreed_channel_config) + LOGGER.info( + generate_formatted_log( + degreed_channel_config.channel_code(), + enterprise_customer.uuid, + None, + None, + f'[Degreed Skills] Attempting to assign skills for customer {enterprise_customer.slug}' + ) + ) + + for content_item in content_metadata_in_catalogs: + course_id = content_item.get('key', None) + course_skills = content_item.get('skill_names', []) + + # if we get empty list of skills, there's no point making API call to Degreed. + if not course_skills: + continue + + json_payload = self._prepare_json_payload_for_skills_endpoint(course_skills) + + # assign skills metadata to degreed course by first fetching degreed course id + try: + degreed_client.assign_course_skills(course_id, json_payload) + except ClientError as error: + LOGGER.error( + generate_formatted_log( + degreed_channel_config.channel_code(), + enterprise_customer.uuid, + None, + None, + f'Degreed2APIClient assign_course_skills failed for course {course_id} ' + f'with message: {error.message}' + ) + ) + continue + except RequestException as error: + LOGGER.error( + generate_formatted_log( + degreed_channel_config.channel_code(), + enterprise_customer.uuid, + None, + None, + f'Degreed2APIClient request to assign skills failed with message: {error.message}' + ) + ) + continue diff --git a/channel_integrations/integrated_channel/management/commands/backfill_course_end_dates.py b/channel_integrations/integrated_channel/management/commands/backfill_course_end_dates.py new file mode 100644 index 0000000..db3f4c0 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/backfill_course_end_dates.py @@ -0,0 +1,41 @@ +""" +Update all courses associated with canvas customer configs to show end dates +""" + +from django.apps import apps +from django.contrib import auth +from django.core.management.base import BaseCommand + +from channel_integrations.canvas.client import CanvasAPIClient +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin + +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Update content transmission items to have their respective catalog's uuid. + + ./manage.py lms backfill_course_end_dates + """ + def handle(self, *args, **options): + """ + Update all past content transmission items to show end dates. + """ + options['prevent_disabled_configurations'] = False + options['channel'] = 'CANVAS' + + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + + # get every past transmitted course on canvas channels + for canvas_channel in self.get_integrated_channels(options): + transmitted_course_ids = ContentMetadataItemTransmission.objects.filter( + enterprise_customer=canvas_channel.enterprise_customer, + integrated_channel_code='CANVAS', + remote_deleted_at__isnull=True, + ).values('content_id') + + CanvasAPIClient(canvas_channel).update_participation_types(transmitted_course_ids) diff --git a/channel_integrations/integrated_channel/management/commands/backfill_missing_csod_foreign_keys.py b/channel_integrations/integrated_channel/management/commands/backfill_missing_csod_foreign_keys.py new file mode 100644 index 0000000..c335da2 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/backfill_missing_csod_foreign_keys.py @@ -0,0 +1,76 @@ +""" +Backfill missing audit record foreign keys for Cornerstone. +""" +import logging + +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils.translation import gettext as _ + +from channel_integrations.cornerstone.models import ( + CornerstoneEnterpriseCustomerConfiguration, + CornerstoneLearnerDataTransmissionAudit, +) +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.utils import batch_by_pk + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Management command which backfills missing audit record foreign keys for cornerstone. + """ + help = _(''' + Backfill missing audit record foreign keys for Cornerstone. + ''') + + def find_csod_config_by_subdomain(self, customer_subdomain): + """ + Given the subdomain from a CornerstoneLearnerDataTransmissionAudit record, search the + CornerstoneEnterpriseCustomerConfiguration records for one with a matching base_url. + Raise an exception when we dont find exactly one (missing config or duplicate configs are bad) + """ + # real prod data often has a config like `https://edx.csod.com` alongside `https://edx-stg.csod.com` + # if we just did a plain `icontains` using `edx` subdomain, we'd get the staging config too + # expanding the subdomain into a proper url prefix lets us get a more exact match. + # we require these urls be https + subdomain_formatted_as_url_prefix = f'https://{customer_subdomain}.' + configs = CornerstoneEnterpriseCustomerConfiguration.objects.filter( + cornerstone_base_url__icontains=subdomain_formatted_as_url_prefix, + ) + if len(configs) > 1: + LOGGER.error(f'multiple ({len(configs)}) configs found for "{customer_subdomain}" when expecting just 1') + return None + else: + return configs.first() + + def backfill_join_keys(self): + """ + For each audit record kind, find all the records in batch, then lookup the appropriate + enterprise_customer_uuid and/or plugin_config_id + """ + try: + only_missing_ld_fks = Q(plugin_configuration_id__isnull=True) + for audit_record_batch in batch_by_pk(CornerstoneLearnerDataTransmissionAudit, extra_filter=only_missing_ld_fks): # pylint: disable=line-too-long + for audit_record in audit_record_batch: + config = self.find_csod_config_by_subdomain(audit_record.subdomain) + if config is None: + LOGGER.error(f'cannot find CSOD config for subdomain "{audit_record.subdomain}"') + continue + ent_cust_uuid = config.enterprise_customer.uuid + LOGGER.info(f'CornerstoneLearnerDataTransmissionAudit <{audit_record.pk}> ' + f'enterprise_customer_uuid={ent_cust_uuid}, ' + f'plugin_configuration_id={config.id}') + audit_record.enterprise_customer_uuid = ent_cust_uuid + audit_record.plugin_configuration_id = config.id + audit_record.save() + except Exception as exc: + LOGGER.exception('backfill_missing_csod_foreign_keys failed', exc_info=exc) + raise exc + + def handle(self, *args, **options): + """ + Backfill missing audit record foreign keys. + """ + self.backfill_join_keys() diff --git a/channel_integrations/integrated_channel/management/commands/backfill_missing_foreign_keys.py b/channel_integrations/integrated_channel/management/commands/backfill_missing_foreign_keys.py new file mode 100644 index 0000000..6c0e8b8 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/backfill_missing_foreign_keys.py @@ -0,0 +1,134 @@ +""" +Backfill missing audit record foreign keys. +""" +import logging + +from django.apps import apps +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils.translation import gettext as _ + +from channel_integrations.blackboard.models import ( + BlackboardEnterpriseCustomerConfiguration, + BlackboardLearnerAssessmentDataTransmissionAudit, + BlackboardLearnerDataTransmissionAudit, +) +from channel_integrations.canvas.models import ( + CanvasEnterpriseCustomerConfiguration, + CanvasLearnerAssessmentDataTransmissionAudit, + CanvasLearnerDataTransmissionAudit, +) +from channel_integrations.cornerstone.models import ( + CornerstoneEnterpriseCustomerConfiguration, + CornerstoneLearnerDataTransmissionAudit, +) +from channel_integrations.degreed2.models import ( + Degreed2EnterpriseCustomerConfiguration, + Degreed2LearnerDataTransmissionAudit, +) +from channel_integrations.degreed.models import ( + DegreedEnterpriseCustomerConfiguration, + DegreedLearnerDataTransmissionAudit, +) +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.models import ( + ContentMetadataItemTransmission, + GenericEnterpriseCustomerPluginConfiguration, + GenericLearnerDataTransmissionAudit, +) +from channel_integrations.moodle.models import MoodleEnterpriseCustomerConfiguration, MoodleLearnerDataTransmissionAudit +from channel_integrations.sap_success_factors.models import ( + SAPSuccessFactorsEnterpriseCustomerConfiguration, + SapSuccessFactorsLearnerDataTransmissionAudit, +) +from channel_integrations.utils import batch_by_pk + +MODELS = { + 'MOODLE': [MoodleEnterpriseCustomerConfiguration, MoodleLearnerDataTransmissionAudit], + 'CSOD': [CornerstoneEnterpriseCustomerConfiguration, CornerstoneLearnerDataTransmissionAudit], + 'BLACKBOARD': [BlackboardEnterpriseCustomerConfiguration, BlackboardLearnerDataTransmissionAudit], + 'BLACKBOARD_ASMT': [BlackboardEnterpriseCustomerConfiguration, BlackboardLearnerAssessmentDataTransmissionAudit], + 'CANVAS': [CanvasEnterpriseCustomerConfiguration, CanvasLearnerDataTransmissionAudit], + 'CANVAS_ASMT': [CanvasEnterpriseCustomerConfiguration, CanvasLearnerAssessmentDataTransmissionAudit], + 'DEGREED': [DegreedEnterpriseCustomerConfiguration, DegreedLearnerDataTransmissionAudit], + 'DEGREED2': [Degreed2EnterpriseCustomerConfiguration, Degreed2LearnerDataTransmissionAudit], + 'GENERIC': [GenericEnterpriseCustomerPluginConfiguration, GenericLearnerDataTransmissionAudit], + 'SAP': [SAPSuccessFactorsEnterpriseCustomerConfiguration, SapSuccessFactorsLearnerDataTransmissionAudit], +} + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Management command which backfills missing audit record foreign keys. + """ + help = _(''' + Backfill missing audit record foreign keys. + ''') + + def find_ent_cust(self, enrollment_id): + """ + Given an enterprise_course_enrollment id, walk the joins to EnterpriseCustomer + """ + EnterpriseCourseEnrollment = apps.get_model('enterprise', 'EnterpriseCourseEnrollment') + EnterpriseCustomerUser = apps.get_model('enterprise', 'EnterpriseCustomerUser') + EnterpriseCustomer = apps.get_model('enterprise', 'EnterpriseCustomer') + try: + ece = EnterpriseCourseEnrollment.objects.get(pk=enrollment_id) + ecu = EnterpriseCustomerUser.objects.get(pk=ece.enterprise_customer_user_id) + ec = EnterpriseCustomer.objects.get(pk=ecu.enterprise_customer_id) + return ec + except ObjectDoesNotExist: + return None + + def backfill_join_keys(self): + """ + For each audit record kind, find all the records in batch, then lookup the appropriate + enterprise_customer_uuid and/or plugin_config_id + """ + try: + for channel_code, (ConfigModel, LearnerAuditModel) in MODELS.items(): + LOGGER.info(f'{LearnerAuditModel.__name__}') + # make reentrant ie pickup where we've left off in case the job needs to be restarted + # only need to check plugin config OR enterprise customer uuid + only_missing_ld_fks = Q(plugin_configuration_id__isnull=True) + for audit_record_batch in batch_by_pk(LearnerAuditModel, extra_filter=only_missing_ld_fks): + for audit_record in audit_record_batch: + enterprise_customer = self.find_ent_cust(audit_record.enterprise_course_enrollment_id) + if enterprise_customer is None: + continue + # nobody currently has more than 1 config across all kinds + config = ConfigModel.objects.filter(enterprise_customer=enterprise_customer).first() + if config is None: + continue + LOGGER.info(f'{LearnerAuditModel.__name__} <{audit_record.pk}> ' + f'enterprise_customer_uuid={enterprise_customer.uuid}, ' + f'plugin_configuration_id={config.id}') + audit_record.enterprise_customer_uuid = enterprise_customer.uuid + audit_record.plugin_configuration_id = config.id + audit_record.save() + # migrate the content_metadata for this channel code, the _AS ones will be empty, effectively a skip + only_missing_cm_fks = Q(integrated_channel_code=channel_code, plugin_configuration_id__isnull=True) + # make reentrant ie pickup where we've left off in case the job needs to be restarted + for audit_record_batch in self.batch_by_pk(ContentMetadataItemTransmission, extra_filter=only_missing_cm_fks): # pylint: disable=line-too-long + for audit_record in audit_record_batch: + if audit_record.enterprise_customer is None: + continue + config = ConfigModel.objects.filter(enterprise_customer=audit_record.enterprise_customer).first() # pylint: disable=line-too-long + if config is None: + continue + LOGGER.info(f'ContentMetadataItemTransmission {channel_code} <{audit_record.pk}> ' + f'plugin_configuration_id={config.id}') + audit_record.plugin_configuration_id = config.id + audit_record.save() + except Exception as exc: + LOGGER.exception('backfill_missing_foreign_keys failed', exc_info=exc) + raise exc + + def handle(self, *args, **options): + """ + Backfill missing audit record foreign keys. + """ + self.backfill_join_keys() diff --git a/channel_integrations/integrated_channel/management/commands/backfill_remote_action_timestamps.py b/channel_integrations/integrated_channel/management/commands/backfill_remote_action_timestamps.py new file mode 100644 index 0000000..8e83a69 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/backfill_remote_action_timestamps.py @@ -0,0 +1,60 @@ +""" +Backfill the new remote_created_at and remote_updated_at content audit record values. +""" +import logging + +from django.apps import apps +from django.contrib import auth +from django.core.management.base import BaseCommand +from django.db.models import Q + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.utils import batch_by_pk, generate_formatted_log + +User = auth.get_user_model() + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Update content transmission items to have the new remote_created_at and remote_updated_at values. + + ./manage.py lms backfill_remote_action_timestamps + """ + + def handle(self, *args, **options): + """ + Update all past content transmission items remote_created_at and remote_updated_at + """ + + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + + no_remote_created_at = Q(remote_created_at__isnull=True) + for items_batch in batch_by_pk(ContentMetadataItemTransmission, extra_filter=no_remote_created_at): + for item in items_batch: + try: + item.remote_created_at = item.created + item.remote_updated_at = item.modified + item.save() + LOGGER.info(generate_formatted_log( + item.integrated_channel_code, + item.enterprise_customer.uuid, + None, + item.content_id, + f'ContentMetadataItemTransmission <{item.id}> ' + f'remote_created_at={item.remote_created_at}, ' + f'remote_updated_at={item.remote_updated_at}' + )) + except Exception: # pylint: disable=broad-except + LOGGER.exception(generate_formatted_log( + item.integrated_channel_code, + item.enterprise_customer.uuid, + None, + item.content_id, + f'ContentMetadataItemTransmission <{item.id}> ' + 'error backfilling remote_created_at & remote_updated_at' + )) diff --git a/channel_integrations/integrated_channel/management/commands/cleanup_duplicate_assignment_records.py b/channel_integrations/integrated_channel/management/commands/cleanup_duplicate_assignment_records.py new file mode 100644 index 0000000..5d5f484 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/cleanup_duplicate_assignment_records.py @@ -0,0 +1,60 @@ +""" +Remove duplicate transmitted assignments for the integrated channels. +""" + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import cleanup_duplicate_assignment_records + +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Management command which removes duplicated assignment records transmitted to the IntegratedChannel(s) configured + for the given EnterpriseCustomer. + + Collect the enterprise enrollments with data sharing consent, and ensure deduping of assignments for each unique + course that has previously been transmitted. + + Note: this management command is currently only configured to work with only the Canvas integrated channel. + """ + help = _(''' + Verify and remove any duplicated assignments transmitted to the IntegratedChannel(s) configured for the given + EnterpriseCustomer. + ''') + + def add_arguments(self, parser): + """ + Add required --api_user argument to the parsetest_learner_data_multiple_coursesr. + """ + parser.add_argument( + '--api_user', + dest='api_user', + required=True, + metavar='LMS_API_USERNAME', + help=_('Username of a user authorized to fetch grades from the LMS API.'), + ) + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + De-duplicate assignments transmitted for the EnterpriseCustomer(s) + """ + # Ensure that we were given an api_user name, and that User exists. + api_username = options['api_user'] + + # For now we only need/want this command to run with Canvas + options['channel'] = 'CANVAS' + try: + User.objects.get(username=api_username) + except User.DoesNotExist as no_user_error: + raise CommandError( + _('A user with the username {username} was not found.').format(username=api_username) + ) from no_user_error + + for canvas_channel in self.get_integrated_channels(options): + cleanup_duplicate_assignment_records.delay(api_username, canvas_channel.channel_code(), canvas_channel.pk) diff --git a/channel_integrations/integrated_channel/management/commands/mark_learner_transmissions_transmitted_true.py b/channel_integrations/integrated_channel/management/commands/mark_learner_transmissions_transmitted_true.py new file mode 100644 index 0000000..ffc4fda --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/mark_learner_transmissions_transmitted_true.py @@ -0,0 +1,48 @@ + +""" +Mark already transmitted LearnerDataTransmission as is_trasmitted=True for all integrated channels +""" + +from logging import getLogger + +from django.apps import apps +from django.core.management.base import BaseCommand + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin + +LOGGER = getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Mark already transmitted LearnerDataTransmission as is_trasmitted=True for all integrated channels + """ + + def handle(self, *args, **options): + """ + Mark already transmitted LearnerDataTransmission as is_trasmitted=True + """ + channel_learner_audit_models = [ + ('moodle', 'MoodleLearnerDataTransmissionAudit'), + ('blackboard', 'BlackboardLearnerDataTransmissionAudit'), + ('blackboard', 'BlackboardLearnerAssessmentDataTransmissionAudit'), + ('canvas', 'CanvasLearnerDataTransmissionAudit'), + ('degreed2', 'Degreed2LearnerDataTransmissionAudit'), + ('degreed', 'DegreedLearnerDataTransmissionAudit'), + ('sap_success_factors', 'SapSuccessFactorsLearnerDataTransmissionAudit'), + ('cornerstone', 'CornerstoneLearnerDataTransmissionAudit'), + ('canvas', 'CanvasLearnerAssessmentDataTransmissionAudit'), + ] + for app_label, model_name in channel_learner_audit_models: + model_class = apps.get_model(app_label=app_label, model_name=model_name) + LOGGER.info( + f'Started: setting {model_name}.is_transmitted set to True' + ) + model_class.objects.filter( + error_message='', + status__lt=400, + ).update(is_transmitted=True) + + LOGGER.info( + f'Finished: setting {model_name}.is_transmitted set to True' + ) diff --git a/channel_integrations/integrated_channel/management/commands/mark_orphaned_content_metadata_audits.py b/channel_integrations/integrated_channel/management/commands/mark_orphaned_content_metadata_audits.py new file mode 100644 index 0000000..684cde0 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/mark_orphaned_content_metadata_audits.py @@ -0,0 +1,28 @@ +""" +Mark all content metadata audit records not directly connected to a customer's catalogs as orphaned. +""" +import logging + +from django.core.management.base import BaseCommand + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import mark_orphaned_content_metadata_audit + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Mark all content metadata audit records not directly connected to a customer's catalogs as orphaned. + + ./manage.py lms mark_orphaned_content_metadata_audits + """ + + def handle(self, *args, **options): + """ + Mark all content metadata audit records not directly connected to a customer's catalogs as orphaned. + """ + try: + mark_orphaned_content_metadata_audit.delay() + except Exception as exc: # pylint: disable=broad-except + LOGGER.exception(f'Failed to mark orphaned content metadata audits. Task failed with exception: {exc}') diff --git a/channel_integrations/integrated_channel/management/commands/remove_duplicate_learner_transmission_audit_records.py b/channel_integrations/integrated_channel/management/commands/remove_duplicate_learner_transmission_audit_records.py new file mode 100644 index 0000000..8233740 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/remove_duplicate_learner_transmission_audit_records.py @@ -0,0 +1,67 @@ +""" +Transmits consenting enterprise learner data to the integrated channels. +""" +from logging import getLogger + +from django.apps import apps +from django.contrib import auth +from django.core.management.base import BaseCommand +from django.db import transaction +from django.db.models import Max +from django.utils.translation import gettext as _ + +User = auth.get_user_model() +LOGGER = getLogger(__name__) + + +class Command(BaseCommand): + """ + Management command which removes the duplicated transmission audit records for integration channels + """ + help = _(''' + Transmit Enterprise learner course completion data for the given EnterpriseCustomer. + ''') + + def handle(self, *args, **options): + """ + Remove the duplicated transmission audit records for integration channels. + """ + # Multiple transmission records were being saved against single enterprise_course_enrollment_id in case + # transmission fails against course and course run id. Job of this management command is to keep the latest + # record for enterprise_course_enrollment_id that doesn't start with "course-v1: and delete all other records." + channel_learner_audit_models = [ + ('moodle', 'MoodleLearnerDataTransmissionAudit'), + ('blackboard', 'BlackboardLearnerDataTransmissionAudit'), + ('canvas', 'CanvasLearnerDataTransmissionAudit'), + ('degreed2', 'Degreed2LearnerDataTransmissionAudit'), + ('sap_success_factors', 'SapSuccessFactorsLearnerDataTransmissionAudit'), + ] + for app_label, model_name in channel_learner_audit_models: + model_class = apps.get_model(app_label=app_label, model_name=model_name) + + latest_records_without_prefix = ( + model_class.objects.exclude(course_id__startswith='course-v1:') + .values('enterprise_course_enrollment_id').annotate(most_recent_transmission_id=Max('id')) + ) + + LOGGER.info( + f'{app_label} channel has {latest_records_without_prefix.count()} records without prefix' + ) + + # Delete all duplicate records for each enterprise_course_enrollment_id + with transaction.atomic(): + for entry in latest_records_without_prefix: + enterprise_course_enrollment_id = entry['enterprise_course_enrollment_id'] + most_recent_transmission_id = entry['most_recent_transmission_id'] + + # Delete all records except the latest one without "course-v1:" + duplicate_records_to_delete = ( + model_class.objects + .filter(enterprise_course_enrollment_id=enterprise_course_enrollment_id) + .exclude(id=most_recent_transmission_id) + ) + LOGGER.info( + f'{app_label} channel - {duplicate_records_to_delete.count()} duplicate records are deleted ' + f' for enrollment id {enterprise_course_enrollment_id}' + ) + duplicate_records_to_delete.delete() diff --git a/channel_integrations/integrated_channel/management/commands/remove_null_catalog_transmission_audits.py b/channel_integrations/integrated_channel/management/commands/remove_null_catalog_transmission_audits.py new file mode 100644 index 0000000..7e0b55c --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/remove_null_catalog_transmission_audits.py @@ -0,0 +1,30 @@ +""" +Remove content transmission audit records that do not contain a catalog UUID. +""" +import logging + +from django.core.management.base import BaseCommand + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import remove_null_catalog_transmission_audits + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Remove content transmission audit records that do not contain a catalog UUID. + ./manage.py lms remove_null_catalog_transmission_audits + """ + + def handle(self, *args, **options): + """ + Filter content transmission audit records that do not contain a catalog UUID and remove them. + """ + try: + remove_null_catalog_transmission_audits.delay() + except Exception as exc: # pylint: disable=broad-except + LOGGER.exception( + f'''Failed to remove content transmission audits that do not + contain a catalog UUID. Task failed with exception: {exc}''' + ) diff --git a/channel_integrations/integrated_channel/management/commands/remove_stale_integrated_channel_api_logs.py b/channel_integrations/integrated_channel/management/commands/remove_stale_integrated_channel_api_logs.py new file mode 100644 index 0000000..daeebf2 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/remove_stale_integrated_channel_api_logs.py @@ -0,0 +1,41 @@ +""" +Deletes records from the IntegratedChannelAPIRequestLogs model that are older than one month.. +""" +from datetime import timedelta +from logging import getLogger + +from django.contrib import auth +from django.core.management.base import BaseCommand +from django.utils import timezone +from django.utils.translation import gettext as _ + +from channel_integrations.utils import integrated_channel_request_log_model + +User = auth.get_user_model() +LOGGER = getLogger(__name__) + + +class Command(BaseCommand): + """ + Management command to delete old records from the IntegratedChannelAPIRequestLogs model. + """ + help = _(''' + This management command deletes records from the IntegratedChannelAPIRequestLogs model that are older than one month + ''') + + def add_arguments(self, parser): + """ + Adds custom arguments to the parser. + """ + parser.add_argument('time_duration', nargs='?', type=int, default=30, + help='The duration in days for deleting old records. Default is 30 days.') + + def handle(self, *args, **options): + """ + Remove the duplicated transmission audit records for integration channels. + """ + time_duration = options['time_duration'] + time_threshold = timezone.now() - timedelta(days=time_duration) + deleted_count, _ = integrated_channel_request_log_model().objects.filter(created__lt=time_threshold).delete() + + LOGGER.info(f"Deleting records from IntegratedChannelAPIRequestLogs. Total records to delete: {deleted_count}") diff --git a/channel_integrations/integrated_channel/management/commands/reset_csod_remote_deleted_at.py b/channel_integrations/integrated_channel/management/commands/reset_csod_remote_deleted_at.py new file mode 100644 index 0000000..bd467f3 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/reset_csod_remote_deleted_at.py @@ -0,0 +1,62 @@ +""" +Mark for re-send all CSOD content transmission with a remote_deleted_at but no api_response_status_code +""" +import logging + +from django.apps import apps +from django.contrib import auth +from django.core.management.base import BaseCommand +from django.db.models import Q + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.utils import batch_by_pk, generate_formatted_log + +User = auth.get_user_model() + +LOGGER = logging.getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Mark for re-send all CSOD content transmission with a remote_deleted_at but no api_response_status_code + + ./manage.py lms reset_csod_remote_deleted_at + """ + + def handle(self, *args, **options): + """ + Mark for re-send all CSOD content transmission with a remote_deleted_at + """ + + ContentMetadataItemTransmission = apps.get_model( + 'integrated_channel', + 'ContentMetadataItemTransmission' + ) + + csod_deleted_at = Q( + integrated_channel_code='CSOD', + remote_deleted_at__isnull=False + ) + + for items_batch in batch_by_pk(ContentMetadataItemTransmission, extra_filter=csod_deleted_at): + for item in items_batch: + try: + item.remote_deleted_at = None + item.save() + LOGGER.info(generate_formatted_log( + item.integrated_channel_code, + item.enterprise_customer.uuid, + None, + item.content_id, + f'integrated_channel_content_transmission_id={item.id}, ' + 'setting remote_deleted_at to None' + )) + except Exception: # pylint: disable=broad-except + LOGGER.exception(generate_formatted_log( + item.integrated_channel_code, + item.enterprise_customer.uuid, + None, + item.content_id, + f'integrated_channel_content_transmission_id={item.id}, ' + 'error setting remote_deleted_at to None' + )) diff --git a/channel_integrations/integrated_channel/management/commands/reset_sapsf_learner_transmissions.py b/channel_integrations/integrated_channel/management/commands/reset_sapsf_learner_transmissions.py new file mode 100644 index 0000000..5d5d350 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/reset_sapsf_learner_transmissions.py @@ -0,0 +1,69 @@ +""" +Reset SAPSF learner transmissions between two dates. +""" + +from django.apps import apps +from django.core.management.base import BaseCommand +from django.utils.dateparse import parse_datetime +from django.utils.translation import gettext as _ + + +class Command(BaseCommand): + """ + Management command which resets SAPSF learner course completion data between two dates. + That would allow us to resend course completion data. + + `./manage.py lms reset_sapsf_learner_transmissions + --start_datetime=2020-01-14T00:00:00Z --end_datetime=2020-01-14T15:11:00Z` + """ + help = _(''' + Reset SAPSF learner transmissions for the given EnterpriseCustomer and Channel between two dates. + ''') + + def add_arguments(self, parser): + """ + Add required --start_datetime and --end_datetime arguments to the parser. + """ + parser.add_argument( + '--start_datetime', + dest='start_datetime', + required=True, + help=_('Start date and time in YYYY-MM-DDTHH:MM:SSZ format.'), + ) + + parser.add_argument( + '--end_datetime', + dest='end_datetime', + required=True, + help=_('End date and time in YYYY-MM-DDTHH:MM:SSZ format.'), + ) + + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + Resets SAPSF learner course completion data between two dates. + """ + start_datetime = parse_datetime(options['start_datetime']) + end_datetime = parse_datetime(options['end_datetime']) + + if not start_datetime or not end_datetime: + self.stdout.write(self.style.ERROR("FAILED: start or end dates times are not valid")) + return + + SapSuccessFactorsLearnerDataTransmissionAudit = apps.get_model( + 'sap_success_factors', + 'SapSuccessFactorsLearnerDataTransmissionAudit' + ) + enrollment_ids = SapSuccessFactorsLearnerDataTransmissionAudit.objects.filter( + created__gte=start_datetime, created__lte=end_datetime + ).values_list('enterprise_course_enrollment_id', flat=True) + + for enrollment_id in enrollment_ids: + SapSuccessFactorsLearnerDataTransmissionAudit.objects.filter( + enterprise_course_enrollment_id=enrollment_id, + error_message='' + ).update(error_message='Invalid data sent', status='400') + self.stdout.write( + self.style.SUCCESS('Successfully updated transmissions with these enrollment id ["%s"]' % enrollment_id) + ) diff --git a/channel_integrations/integrated_channel/management/commands/transmit_content_metadata.py b/channel_integrations/integrated_channel/management/commands/transmit_content_metadata.py new file mode 100644 index 0000000..763b418 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/transmit_content_metadata.py @@ -0,0 +1,54 @@ +""" +Transmits information about an enterprise's course catalog to connected IntegratedChannels +""" + +from logging import getLogger + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import transmit_content_metadata + +LOGGER = getLogger(__name__) +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Transmit courseware data to the IntegratedChannel(s) linked to any or all EnterpriseCustomers. + """ + + def add_arguments(self, parser): + """ + Add required arguments to the parser. + """ + parser.add_argument( + '--catalog_user', + dest='catalog_user', + required=True, + metavar='ENTERPRISE_CATALOG_API_USERNAME', + help='Use this user to access the Course Catalog API.' + ) + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + Transmit the courseware data for the EnterpriseCustomer(s) to the active integration channels. + """ + username = options['catalog_user'] + options['content_metadata_job_support'] = True + + # Before we do a whole bunch of database queries, make sure that the user we were passed exists. + try: + User.objects.get(username=username) + except User.DoesNotExist as no_user_error: + raise CommandError('A user with the username {} was not found.'.format(username)) from no_user_error + + channels = self.get_integrated_channels(options) + + for channel in channels: + channel_code = channel.channel_code() + channel_pk = channel.pk + # NOTE pass arguments as named kwargs for use in lock key + transmit_content_metadata.delay(username=username, channel_code=channel_code, channel_pk=channel_pk) diff --git a/channel_integrations/integrated_channel/management/commands/transmit_learner_data.py b/channel_integrations/integrated_channel/management/commands/transmit_learner_data.py new file mode 100644 index 0000000..2467873 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/transmit_learner_data.py @@ -0,0 +1,58 @@ +""" +Transmits consenting enterprise learner data to the integrated channels. +""" + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import transmit_learner_data + +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Management command which transmits learner course completion data to the IntegratedChannel(s) configured for the + given EnterpriseCustomer. + + Collect the enterprise learner data for enrollments with data sharing consent, and transmit each to the + EnterpriseCustomer's configured IntegratedChannel(s). + """ + help = _(''' + Transmit Enterprise learner course completion data for the given EnterpriseCustomer. + ''') + stealth_options = ('enterprise_customer_slug', 'user1', 'user2') + + def add_arguments(self, parser): + """ + Add required --api_user argument to the parser. + """ + parser.add_argument( + '--api_user', + dest='api_user', + required=True, + metavar='LMS_API_USERNAME', + help=_('Username of a user authorized to fetch grades from the LMS API.'), + ) + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + Transmit the learner data for the EnterpriseCustomer(s) to the active integration channels. + """ + # Ensure that we were given an api_user name, and that User exists. + api_username = options['api_user'] + try: + User.objects.get(username=api_username) + except User.DoesNotExist as no_user_error: + raise CommandError( + _('A user with the username {username} was not found.').format(username=api_username) + ) from no_user_error + + # Transmit the learner data to each integrated channel + for integrated_channel in self.get_integrated_channels(options): + # NOTE pass arguments as named kwargs for use in lock key + transmit_learner_data.delay( + username=api_username, channel_code=integrated_channel.channel_code(), channel_pk=integrated_channel.pk) diff --git a/channel_integrations/integrated_channel/management/commands/transmit_subsection_learner_data.py b/channel_integrations/integrated_channel/management/commands/transmit_subsection_learner_data.py new file mode 100644 index 0000000..8daeb6d --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/transmit_subsection_learner_data.py @@ -0,0 +1,61 @@ +""" +Transmits consenting enterprise learner data to the integrated channels. +""" + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import transmit_subsection_learner_data + +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Management command which transmits learner course completion data to the IntegratedChannel(s) configured for the + given EnterpriseCustomer. + + Collect the enterprise learner data for enrollments with data sharing consent, and transmit each to the + EnterpriseCustomer's configured IntegratedChannel(s). + """ + help = _(''' + Transmit Enterprise learner assessment level reporting data for the given EnterpriseCustomer. + ''') + stealth_options = ('enterprise_customer_slug', 'user1', 'user2') + + def add_arguments(self, parser): + """ + Add required --api_user argument to the parser. + """ + parser.add_argument( + '--api_user', + dest='api_user', + required=True, + metavar='LMS_API_USERNAME', + help=_('Username of a user authorized to fetch grades from the LMS API.'), + ) + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + Transmit the assessment level learner data for the EnterpriseCustomer(s) to the active integration channels. + """ + # Ensure that we were given an api_user name, and that User exists. + api_username = options['api_user'] + options['assessment_level_support'] = True + try: + User.objects.get(username=api_username) + except User.DoesNotExist as no_user_error: + raise CommandError( + _('A user with the username {username} was not found.').format(username=api_username) + ) from no_user_error + + # Transmit the learner data to each integrated channel + for integrated_channel in self.get_integrated_channels(options): + transmit_subsection_learner_data.delay( + api_username, + integrated_channel.channel_code(), + integrated_channel.pk + ) diff --git a/channel_integrations/integrated_channel/management/commands/unlink_inactive_sap_learners.py b/channel_integrations/integrated_channel/management/commands/unlink_inactive_sap_learners.py new file mode 100644 index 0000000..936a2c4 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/unlink_inactive_sap_learners.py @@ -0,0 +1,31 @@ +""" +Unlink inactive enterprise learners of SAP Success Factors from related EnterpriseCustomer(s). +""" + +from logging import getLogger + +from django.core.management.base import BaseCommand + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import unlink_inactive_learners + +LOGGER = getLogger(__name__) + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Unlink inactive enterprise learners of SAP Success Factors from all related EnterpriseCustomer(s). + """ + + def handle(self, *args, **options): + """ + Unlink inactive EnterpriseCustomer(s) SAP learners. + """ + channels = self.get_integrated_channels(options) + + for channel in channels: + channel_code = channel.channel_code() + channel_pk = channel.pk + if channel_code == 'SAP': + # Transmit the learner data to each integrated channel + unlink_inactive_learners.delay(channel_code, channel_pk) diff --git a/channel_integrations/integrated_channel/management/commands/update_content_transmission_catalogs.py b/channel_integrations/integrated_channel/management/commands/update_content_transmission_catalogs.py new file mode 100644 index 0000000..20cb768 --- /dev/null +++ b/channel_integrations/integrated_channel/management/commands/update_content_transmission_catalogs.py @@ -0,0 +1,50 @@ +""" +Fetch and update all content metadata transmission audits with their respective catalog's uuid. +""" + +from django.contrib import auth +from django.core.management.base import BaseCommand, CommandError + +from channel_integrations.integrated_channel.management.commands import IntegratedChannelCommandMixin +from channel_integrations.integrated_channel.tasks import update_content_transmission_catalog + +User = auth.get_user_model() + + +class Command(IntegratedChannelCommandMixin, BaseCommand): + """ + Update content transmission items to have their respective catalog's uuid. + """ + + def add_arguments(self, parser): + """ + Add required arguments to the parser. + """ + parser.add_argument( + '--catalog_user', + dest='catalog_user', + required=True, + metavar='ENTERPRISE_CATALOG_API_USERNAME', + help='Use this user to access the Course Catalog API.' + ) + super().add_arguments(parser) + + def handle(self, *args, **options): + """ + Update all past content transmission items to have their respective catalog's uuid. + """ + username = options['catalog_user'] + options['prevent_disabled_configurations'] = False + # Before we do a whole bunch of database queries, make sure that the user we were passed exists. + try: + User.objects.get(username=username) + except User.DoesNotExist as no_user_error: + raise CommandError('A user with the username {} was not found.'.format(username)) from no_user_error + + channels = self.get_integrated_channels(options) + + for channel in channels: + + channel_code = channel.channel_code() + channel_pk = channel.pk + update_content_transmission_catalog.delay(username, channel_code, channel_pk) diff --git a/channel_integrations/integrated_channel/migrations/__init__.py b/channel_integrations/integrated_channel/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel_integrations/integrated_channel/models.py b/channel_integrations/integrated_channel/models.py new file mode 100644 index 0000000..f18cc0f --- /dev/null +++ b/channel_integrations/integrated_channel/models.py @@ -0,0 +1,1002 @@ +""" +Database models for Enterprise Integrated Channel. +""" + +import json +import logging + +from jsonfield.fields import JSONField + +from django.contrib import auth +from django.core.exceptions import ValidationError +from django.db import models +from django.db.models import Q +from django.db.models.query import QuerySet +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from model_utils.models import TimeStampedModel + +from enterprise.constants import TRANSMISSION_MARK_CREATE, TRANSMISSION_MARK_DELETE, TRANSMISSION_MARK_UPDATE +from enterprise.models import EnterpriseCustomer, EnterpriseCustomerCatalog +from enterprise.utils import localized_utcnow +from channel_integrations.integrated_channel.exporters.content_metadata import ContentMetadataExporter +from channel_integrations.integrated_channel.exporters.learner_data import LearnerExporter +from channel_integrations.integrated_channel.transmitters.content_metadata import ContentMetadataTransmitter +from channel_integrations.integrated_channel.transmitters.learner_data import LearnerTransmitter +from channel_integrations.utils import channel_code_to_app_label, convert_comma_separated_string_to_list + +LOGGER = logging.getLogger(__name__) +User = auth.get_user_model() +LAST_24_HRS = timezone.now() - timezone.timedelta(hours=24) + + +def set_default_display_name(*args, **kw): + """ + post_save signal reciever to set default display name + wired up in EnterpriseCustomerPluginConfiguration.__init_subclass__ + """ + this_display_name = kw['instance'].display_name + # check if display_name is None, empty, or just spaces + if not (this_display_name and this_display_name.strip()): + kw['instance'].display_name = kw['instance'].generate_default_display_name() + kw['instance'].save() + + +class SoftDeletionQuerySet(QuerySet): + """ + Soft deletion query set. + """ + + def delete(self): + return super().update(deleted_at=localized_utcnow()) + + def hard_delete(self): + return super().delete() + + def revive(self): + return super().update(deleted_at=None) + + +class SoftDeletionManager(models.Manager): + """ + Soft deletion manager overriding a model's query set in order to soft delete. + """ + use_for_related_fields = True + + def __init__(self, *args, **kwargs): + self.alive_only = kwargs.pop('alive_only', True) + super().__init__(*args, **kwargs) + + def get_queryset(self): + if self.alive_only: + return SoftDeletionQuerySet(self.model, using=self._db, hints=self._hints).filter(deleted_at=None) + return SoftDeletionQuerySet(self.model) + + def delete(self): + return self.get_queryset().delete() + + def hard_delete(self): + return self.get_queryset().hard_delete() + + def revive(self): + return self.get_queryset().revive() + + +class SoftDeletionModel(TimeStampedModel): + """ + Soft deletion model that sets a particular entries `deleted_at` field instead of removing the entry on delete. + Use `hard_delete()` to permanently remove entries. + """ + deleted_at = models.DateTimeField(blank=True, null=True) + + objects = SoftDeletionManager() + all_objects = SoftDeletionManager(alive_only=False) + + class Meta: + abstract = True + + +class EnterpriseCustomerPluginConfiguration(SoftDeletionModel): + """ + Abstract base class for information related to integrating with external systems for an enterprise customer. + + EnterpriseCustomerPluginConfiguration should be extended by configuration models in other integrated channel + apps to provide uniformity across different integrated channels. + + The configuration provides default exporters and transmitters if the ``get_x_data_y`` methods aren't + overridden, where ``x`` and ``y`` are (learner, course) and (exporter, transmitter) respectively. + """ + + display_name = models.CharField( + max_length=255, + blank=True, + default='', + help_text=_("A configuration nickname.") + ) + + enterprise_customer = models.ForeignKey( + EnterpriseCustomer, + blank=False, + null=False, + help_text=_("Enterprise Customer associated with the configuration."), + on_delete=models.deletion.CASCADE + ) + + idp_id = models.CharField( + max_length=255, + blank=True, + default='', + help_text=_("If provided, will be used as IDP slug to locate remote id for learners") + ) + + active = models.BooleanField( + blank=False, + null=False, + help_text=_("Is this configuration active?"), + ) + + dry_run_mode_enabled = models.BooleanField( + blank=False, + null=False, + default=False, + help_text=_("Is this configuration in dry-run mode? (experimental)"), + ) + + show_course_price = models.BooleanField( + blank=False, + null=False, + default=False, + help_text=_("Displays course price"), + ) + + transmission_chunk_size = models.IntegerField( + default=500, + help_text=_("The maximum number of data items to transmit to the integrated channel with each request.") + ) + + channel_worker_username = models.CharField( + max_length=255, + blank=True, + default='', + help_text=_("Enterprise channel worker username to get JWT tokens for authenticating LMS APIs."), + ) + catalogs_to_transmit = models.TextField( + blank=True, + default='', + help_text=_( + "A comma-separated list of catalog UUIDs to transmit. If blank, all customer catalogs will be transmitted. " + "If there are overlapping courses in the customer catalogs, the overlapping course metadata will be " + "selected from the newest catalog." + ), + ) + disable_learner_data_transmissions = models.BooleanField( + default=False, + verbose_name="Disable Learner Data Transmission", + help_text=_("When set to True, the configured customer will no longer receive learner data transmissions, both" + " scheduled and signal based") + ) + last_sync_attempted_at = models.DateTimeField( + help_text='The DateTime of the most recent Content or Learner data record sync attempt', + blank=True, + null=True + ) + last_content_sync_attempted_at = models.DateTimeField( + help_text='The DateTime of the most recent Content data record sync attempt', + blank=True, + null=True + ) + last_learner_sync_attempted_at = models.DateTimeField( + help_text='The DateTime of the most recent Learner data record sync attempt', + blank=True, + null=True + ) + last_sync_errored_at = models.DateTimeField( + help_text='The DateTime of the most recent failure of a Content or Learner data record sync attempt', + blank=True, + null=True + ) + last_content_sync_errored_at = models.DateTimeField( + help_text='The DateTime of the most recent failure of a Content data record sync attempt', + blank=True, + null=True + ) + last_learner_sync_errored_at = models.DateTimeField( + help_text='The DateTime of the most recent failure of a Learner data record sync attempt', + blank=True, + null=True + ) + last_modified_at = models.DateTimeField( + help_text='The DateTime of the last change made to this configuration.', + auto_now=True, + blank=True, + null=True + ) + + class Meta: + abstract = True + + @classmethod + def __init_subclass__(cls, **kwargs): + """ + Finds every subclass and wires up the signal reciever to set default display name when blank + """ + super().__init_subclass__(**kwargs) + models.signals.post_save.connect(set_default_display_name, sender=cls) + + def delete(self, *args, **kwargs): + self.deleted_at = localized_utcnow() + self.save() + + def revive(self): + self.deleted_at = None + self.save() + + def hard_delete(self): + return super().delete() + + def clean(self): + invalid_uuids = [] + for uuid in convert_comma_separated_string_to_list(self.catalogs_to_transmit): + try: + EnterpriseCustomerCatalog.objects.get(uuid=uuid, enterprise_customer=self.enterprise_customer) + except (EnterpriseCustomerCatalog.DoesNotExist, ValidationError): + invalid_uuids.append(str(uuid)) + if invalid_uuids: + raise ValidationError( + { + 'catalogs_to_transmit': [ + "These are the invalid uuids: {invalid_uuids}".format(invalid_uuids=invalid_uuids) + ] + } + ) + + def fetch_orphaned_content_audits(self): + """ + Helper method attached to customer configs to fetch all orphaned content metadata audits not linked to the + customer's catalogs. + """ + enterprise_customer_catalogs = self.customer_catalogs_to_transmit or \ + self.enterprise_customer.enterprise_customer_catalogs.all() + + customer_catalog_uuids = enterprise_customer_catalogs.values_list('uuid', flat=True) + + non_existent_catalogs_filter = Q(enterprise_customer_catalog_uuid__in=customer_catalog_uuids) + null_catalogs_filter = Q(enterprise_customer_catalog_uuid__isnull=True) + + return ContentMetadataItemTransmission.objects.filter( + integrated_channel_code=self.channel_code(), + enterprise_customer=self.enterprise_customer, + remote_deleted_at__isnull=True, + remote_created_at__isnull=False, + ).filter(~non_existent_catalogs_filter | null_catalogs_filter) + + def update_content_synced_at(self, action_happened_at, was_successful): + """ + Given the last time a Content record sync was attempted and status update the appropriate timestamps. + """ + update_fields = [] + if self.last_sync_attempted_at is None or action_happened_at > self.last_sync_attempted_at: + self.last_sync_attempted_at = action_happened_at + update_fields.append('last_sync_attempted_at') + if self.last_content_sync_attempted_at is None or action_happened_at > self.last_content_sync_attempted_at: + self.last_content_sync_attempted_at = action_happened_at + update_fields.append('last_content_sync_attempted_at') + if not was_successful: + if self.last_sync_errored_at is None or action_happened_at > self.last_sync_errored_at: + self.last_sync_errored_at = action_happened_at + update_fields.append('last_sync_errored_at') + if self.last_content_sync_errored_at is None or action_happened_at > self.last_content_sync_errored_at: + self.last_content_sync_errored_at = action_happened_at + update_fields.append('last_content_sync_errored_at') + if update_fields: + return self.save(update_fields=update_fields) + else: + return self + + def update_learner_synced_at(self, action_happened_at, was_successful): + """ + Given the last time a Learner record sync was attempted and status update the appropriate timestamps. + """ + update_fields = [] + if self.last_sync_attempted_at is None or action_happened_at > self.last_sync_attempted_at: + self.last_sync_attempted_at = action_happened_at + update_fields.append('last_sync_attempted_at') + if self.last_learner_sync_attempted_at is None or action_happened_at > self.last_learner_sync_attempted_at: + self.last_learner_sync_attempted_at = action_happened_at + update_fields.append('last_learner_sync_attempted_at') + if not was_successful: + if self.last_sync_errored_at is None or action_happened_at > self.last_sync_errored_at: + self.last_sync_errored_at = action_happened_at + update_fields.append('last_sync_errored_at') + if self.last_learner_sync_errored_at is None or action_happened_at > self.last_learner_sync_errored_at: + self.last_learner_sync_errored_at = action_happened_at + update_fields.append('last_learner_sync_errored_at') + if update_fields: + return self.save(update_fields=update_fields) + else: + return self + + @property + def is_valid(self): + """ + Returns whether or not the configuration is valid and ready to be activated + + Args: + obj: The instance of EnterpriseCustomerConfiguration + being rendered with this admin form. + """ + missing_items = {'missing': []} + incorrect_items = {'incorrect': []} + return missing_items, incorrect_items + + @property + def channel_worker_user(self): + """ + default worker username for channel + """ + worker_username = self.channel_worker_username if self.channel_worker_username else 'enterprise_channel_worker' + return User.objects.filter(username=worker_username).first() + + @property + def customer_catalogs_to_transmit(self): + """ + Return the list of EnterpriseCustomerCatalog objects. + """ + catalogs_list = [] + if self.catalogs_to_transmit: + catalogs_list = EnterpriseCustomerCatalog.objects.filter( + uuid__in=convert_comma_separated_string_to_list(self.catalogs_to_transmit) + ) + return catalogs_list + + @staticmethod + def channel_code(): + """ + Returns an capitalized identifier for this channel class, unique among subclasses. + """ + raise NotImplementedError('Implemented in concrete subclass.') + + @classmethod + def get_class_by_channel_code(cls, channel_code): + """ + Return the `EnterpriseCustomerPluginConfiguration` implementation for the particular channel_code, or None + """ + for a_cls in cls.__subclasses__(): + if a_cls.channel_code().lower() == channel_code.lower(): + return a_cls + return None + + def generate_default_display_name(self): + """ + Returns a default display namem which can be overriden by a subclass. + """ + return f'{self.channel_code()} {self.id}' + + def get_learner_data_exporter(self, user): + """ + Returns the class that can serialize the learner course completion data to the integrated channel. + """ + return LearnerExporter(user, self) + + def get_learner_data_transmitter(self): + """ + Returns the class that can transmit the learner course completion data to the integrated channel. + """ + return LearnerTransmitter(self) + + def get_content_metadata_exporter(self, user): + """ + Returns a class that can retrieve and transform content metadata to the schema + expected by the integrated channel. + """ + return ContentMetadataExporter(user, self) + + def get_content_metadata_transmitter(self): + """ + Returns a class that can transmit the content metadata to the integrated channel. + """ + return ContentMetadataTransmitter(self) + + def transmit_learner_data(self, user): + """ + Iterate over each learner data record and transmit it to the integrated channel. + """ + exporter = self.get_learner_data_exporter(user) + transmitter = self.get_learner_data_transmitter() + transmitter.transmit(exporter) + + def transmit_single_learner_data(self, **kwargs): + """ + Iterate over single learner data record and transmit it to the integrated channel. + """ + exporter = self.get_learner_data_exporter(self.channel_worker_user) + transmitter = self.get_learner_data_transmitter() + transmitter.transmit(exporter, **kwargs) + + def transmit_content_metadata(self, user): + """ + Transmit content metadata to integrated channel. + """ + exporter = self.get_content_metadata_exporter(user) + transmitter = self.get_content_metadata_transmitter() + transmitter.transmit(*exporter.export()) + + def transmit_single_subsection_learner_data(self, **kwargs): + """ + Transmit a single subsection learner data record to the integrated channel. + """ + exporter = self.get_learner_data_exporter(self.channel_worker_user) + transmitter = self.get_learner_data_transmitter() + transmitter.single_learner_assessment_grade_transmit(exporter, **kwargs) + + def transmit_subsection_learner_data(self, user): + """ + Iterate over each assessment learner data record and transmit them to the integrated channel. + """ + exporter = self.get_learner_data_exporter(user) + transmitter = self.get_learner_data_transmitter() + transmitter.assessment_level_transmit(exporter) + + def cleanup_duplicate_assignment_records(self, user): + """ + Remove duplicated assessments transmitted through the integrated channel. + """ + exporter = self.get_learner_data_exporter(user) + transmitter = self.get_learner_data_transmitter() + transmitter.deduplicate_assignment_records_transmit(exporter) + + def update_content_transmission_catalog(self, user): + """ + Update transmission audits to contain the content's associated catalog uuid. + """ + exporter = self.get_content_metadata_exporter(user) + exporter.update_content_transmissions_catalog_uuids() + + +class GenericEnterpriseCustomerPluginConfiguration(EnterpriseCustomerPluginConfiguration): + """ + A generic implementation of EnterpriseCustomerPluginConfiguration which can be instantiated + """ + + def __str__(self): + """ + Return human-readable string representation. + """ + return "".format( + enterprise_name=self.enterprise_customer.name + ) + + @staticmethod + def channel_code(): + """ + Returns an capitalized identifier for this channel class, unique among subclasses. + """ + return 'GENERIC' + + +class ApiResponseRecord(TimeStampedModel): + """ + Api response data for learner and content metadata transmissions + + .. no_pii; + """ + status_code = models.PositiveIntegerField( + help_text='The most recent remote API call response HTTP status code', + blank=True, + null=True + ) + body = models.TextField( + help_text='The most recent remote API call response body', + blank=True, + null=True + ) + + +class LearnerDataTransmissionAudit(TimeStampedModel): + """ + The payload we send to an integrated channel at a given point in time for an enterprise course enrollment. + + .. pii: The user_email model field contains PII. Declaring "retained" because I don't know if it's retired. + .. pii_types: email_address + .. pii_retirement: retained + """ + + # TODO: index customer uuid + plugin coinfig id together, with enrollment id? + enterprise_customer_uuid = models.UUIDField(blank=True, null=True) + user_email = models.CharField(max_length=255, blank=True, null=True) + plugin_configuration_id = models.IntegerField(blank=True, null=True) + enterprise_course_enrollment_id = models.IntegerField(blank=True, null=True, db_index=True) + course_id = models.CharField(max_length=255, blank=False, null=False) + content_title = models.CharField(max_length=255, default=None, null=True, blank=True) + course_completed = models.BooleanField(default=True) + progress_status = models.CharField(max_length=255, blank=True) + completed_timestamp = models.DateTimeField(blank=True, null=True) + instructor_name = models.CharField(max_length=255, blank=True) + grade = models.FloatField(blank=True, null=True) + total_hours = models.FloatField(null=True, blank=True) + subsection_id = models.CharField(max_length=255, blank=True, null=True, db_index=True) + subsection_name = models.CharField(max_length=255, blank=False, null=True) + status = models.CharField(max_length=100, blank=True, null=True) + error_message = models.TextField(blank=True, null=True) + is_transmitted = models.BooleanField(default=False) + friendly_status_message = models.CharField( + help_text='A user-friendly API response status message.', + max_length=255, + default=None, + null=True, + blank=True + ) + api_record = models.OneToOneField( + ApiResponseRecord, + blank=True, + null=True, + on_delete=models.CASCADE, + help_text=_('Data pertaining to the transmissions API request response.') + ) + + class Meta: + abstract = True + app_label = 'integrated_channel' + + def __str__(self): + """ + Return a human-readable string representation of the object. + """ + return ( + f'' + f', grade: {self.grade}' + f', completed_timestamp: {self.completed_timestamp}' + f', enterprise_customer_uuid: {self.enterprise_customer_uuid}' + f', course_completed: {self.course_completed}' + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + @property + def provider_id(self): + """ + Fetch ``provider_id`` from global configuration settings + """ + return None + + @classmethod + def audit_type(cls): + """ + The base learner data transmission audit type - defaults to `completion` + """ + return "completion" + + @classmethod + def get_completion_class_by_channel_code(cls, channel_code): + """ + Return the `LearnerDataTransmissionAudit` implementation related to completion reporting for + the particular channel_code, or None + """ + app_label = channel_code_to_app_label(channel_code) + for a_cls in cls.__subclasses__(): + if a_cls._meta.app_label == app_label and a_cls.audit_type() == "completion": + return a_cls + return None + + def serialize(self, *args, **kwargs): + """ + Return a JSON-serialized representation. + + Sort the keys so the result is consistent and testable. + + # TODO: When we refactor to use a serialization flow consistent with how course metadata + # is serialized, remove the serialization here and make the learner data exporter handle the work. + """ + return json.dumps(self._payload_data(), sort_keys=True) + + def _payload_data(self): + """ + Convert the audit record's fields into SAP SuccessFactors key/value pairs. + """ + return { + 'courseID': self.course_id, + 'courseCompleted': 'true' if self.course_completed else 'false', + 'completedTimestamp': self.completed_timestamp, + 'grade': self.grade, + } + + +class GenericLearnerDataTransmissionAudit(LearnerDataTransmissionAudit): + """ + A generic implementation of LearnerDataTransmissionAudit which can be instantiated + """ + class Meta: + app_label = 'integrated_channel' + + def __str__(self): + """ + Return a human-readable string representation of the object. + """ + return ( + ''.format( + transmission_id=self.id, + enrollment=self.enterprise_course_enrollment_id, + course_id=self.course_id + ) + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + +class ContentMetadataItemTransmission(TimeStampedModel): + """ + A content metadata item that has been transmitted to an integrated channel. + + This model can be queried to find the content metadata items that have been + transmitted to an integrated channel. It is used to synchronize the content + metadata items available in an enterprise's catalog with the integrated channel. + + .. no_pii: + """ + class Meta: + index_together = [('enterprise_customer', 'integrated_channel_code', 'plugin_configuration_id', 'content_id')] + unique_together = (('integrated_channel_code', 'plugin_configuration_id', 'content_id'),) + + enterprise_customer = models.ForeignKey(EnterpriseCustomer, on_delete=models.CASCADE) + integrated_channel_code = models.CharField(max_length=30) + plugin_configuration_id = models.PositiveIntegerField(blank=True, null=True) + content_id = models.CharField(max_length=255) + content_title = models.CharField(max_length=255, default=None, null=True, blank=True) + channel_metadata = JSONField() + content_last_changed = models.DateTimeField( + help_text='Date of the last time the enterprise catalog associated with this metadata item was updated', + blank=True, + null=True + ) + enterprise_customer_catalog_uuid = models.UUIDField( + help_text='The enterprise catalog that this metadata item was derived from', + blank=True, + null=True, + ) + remote_deleted_at = models.DateTimeField( + help_text='Date when the content transmission was deleted in the remote API', + blank=True, + null=True + ) + remote_created_at = models.DateTimeField( + help_text='Date when the content transmission was created in the remote API', + blank=True, + null=True + ) + remote_errored_at = models.DateTimeField( + help_text='Date when the content transmission was failed in the remote API.', + blank=True, + null=True + ) + remote_updated_at = models.DateTimeField( + help_text='Date when the content transmission was last updated in the remote API', + blank=True, + null=True + ) + api_response_status_code = models.PositiveIntegerField( + help_text='The most recent remote API call response HTTP status code', + blank=True, + null=True + ) + friendly_status_message = models.CharField( + help_text='A user-friendly API response status message.', + max_length=255, + default=None, + null=True, + blank=True + ) + marked_for = models.CharField( + help_text='Flag marking a record as needing a form of transmitting', + max_length=32, + blank=True, + null=True + ) + api_record = models.OneToOneField( + ApiResponseRecord, + blank=True, + null=True, + on_delete=models.CASCADE, + help_text=_('Data pertaining to the transmissions API request response.') + ) + + @classmethod + def deleted_transmissions(cls, enterprise_customer, plugin_configuration_id, integrated_channel_code, content_id): + """ + Return any pre-existing records for this customer/plugin/content which was previously deleted + """ + query = Q( + enterprise_customer=enterprise_customer, + plugin_configuration_id=plugin_configuration_id, + content_id=content_id, + integrated_channel_code=integrated_channel_code, + remote_deleted_at__isnull=False, + ) + query.add( + Q(remote_errored_at__lt=LAST_24_HRS) + | Q(remote_errored_at__isnull=True) + | Q(remote_errored_at__lt=enterprise_customer.modified), Q.AND) + return ContentMetadataItemTransmission.objects.filter(query) + + @classmethod + def incomplete_create_transmissions( + cls, + enterprise_customer, + plugin_configuration_id, + integrated_channel_code, + content_id + ): + """ + Return any pre-existing records for this customer/plugin/content which was created but never sent or failed + """ + in_db_but_unsent_query = Q( + enterprise_customer=enterprise_customer, + plugin_configuration_id=plugin_configuration_id, + content_id=content_id, + integrated_channel_code=integrated_channel_code, + remote_created_at__isnull=True, + remote_updated_at__isnull=True, + remote_deleted_at__isnull=True, + remote_errored_at__isnull=True, + ) + in_db_but_failed_to_send_query = Q( + enterprise_customer=enterprise_customer, + plugin_configuration_id=plugin_configuration_id, + content_id=content_id, + integrated_channel_code=integrated_channel_code, + remote_created_at__isnull=False, + remote_updated_at__isnull=True, + remote_deleted_at__isnull=True, + api_response_status_code__gte=400, + ) + in_db_but_failed_to_send_query.add( + Q(remote_errored_at__lt=LAST_24_HRS) | Q(remote_errored_at__isnull=True) | + Q(remote_errored_at__lt=enterprise_customer.modified), Q.AND) + in_db_but_unsent_query.add(in_db_but_failed_to_send_query, Q.OR) + return ContentMetadataItemTransmission.objects.filter(in_db_but_unsent_query) + + @classmethod + def incomplete_update_transmissions( + cls, + enterprise_customer, + plugin_configuration_id, + integrated_channel_code, + content_id + ): + """ + Return any pre-existing records for this customer/plugin/content which was updated but never sent or failed + """ + in_db_but_failed_to_send_query = Q( + enterprise_customer=enterprise_customer, + plugin_configuration_id=plugin_configuration_id, + content_id=content_id, + integrated_channel_code=integrated_channel_code, + remote_created_at__isnull=False, + remote_updated_at__isnull=False, + remote_deleted_at__isnull=True, + api_response_status_code__gte=400, + ) + in_db_but_failed_to_send_query.add( + Q(remote_errored_at__lt=LAST_24_HRS) | Q(remote_errored_at__isnull=True) | + Q(remote_errored_at__lt=enterprise_customer.modified), Q.AND) + return ContentMetadataItemTransmission.objects.filter(in_db_but_failed_to_send_query) + + @classmethod + def incomplete_delete_transmissions( + cls, + enterprise_customer, + plugin_configuration_id, + integrated_channel_code, + content_id + ): + """ + Return any pre-existing records for this customer/plugin/content which was deleted but never sent or failed + """ + in_db_but_failed_to_send_query = Q( + enterprise_customer=enterprise_customer, + plugin_configuration_id=plugin_configuration_id, + content_id=content_id, + integrated_channel_code=integrated_channel_code, + remote_created_at__isnull=False, + remote_deleted_at__isnull=False, + api_response_status_code__gte=400, + ) + in_db_but_failed_to_send_query.add( + Q(remote_errored_at__lt=LAST_24_HRS) + | Q(remote_errored_at__isnull=True) + | Q(remote_errored_at__lt=enterprise_customer.modified), Q.AND) + return ContentMetadataItemTransmission.objects.filter(in_db_but_failed_to_send_query) + + def _mark_transmission(self, mark_for): + """ + Helper method to tag a transmission for any operation + """ + self.marked_for = mark_for + self.save() + + def mark_for_create(self): + """ + Mark a transmission for creation + """ + self._mark_transmission(TRANSMISSION_MARK_CREATE) + + def mark_for_update(self): + """ + Mark a transmission for update + """ + self._mark_transmission(TRANSMISSION_MARK_UPDATE) + + def mark_for_delete(self): + """ + Mark a transmission for delete + """ + self._mark_transmission(TRANSMISSION_MARK_DELETE) + + def remove_marked_for(self): + """ + Remove and mark on a transmission + """ + self._mark_transmission(None) + + def prepare_to_recreate(self, content_last_changed, enterprise_customer_catalog_uuid): + """ + Prepare a deleted or unsent record to be re-created in the remote API by resetting dates and audit fields + """ + # maintaining status code on the transmission record to aid with querying + self.api_response_status_code = None + if self.api_record: + self.api_record.body = None + self.api_record.status_code = None + self.remote_deleted_at = None + self.remote_created_at = None + self.remote_updated_at = None + self.channel_metadata = None + self.content_last_changed = content_last_changed + self.enterprise_customer_catalog_uuid = enterprise_customer_catalog_uuid + self.marked_for = TRANSMISSION_MARK_CREATE + self.save() + return self + + def __str__(self): + """ + Return a human-readable string representation of the object. + """ + return ( + ''.format( + content_id=self.content_id, + customer=self.enterprise_customer, + channel=self.integrated_channel_code + ) + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + +class OrphanedContentTransmissions(TimeStampedModel): + """ + A model to track content metadata transmissions that were successfully sent to the integrated channel but then + subsequently were orphaned by a removal of their associated catalog from the customer. + """ + class Meta: + index_together = [('integrated_channel_code', 'plugin_configuration_id', 'resolved')] + + integrated_channel_code = models.CharField(max_length=30) + plugin_configuration_id = models.PositiveIntegerField(blank=False, null=False) + content_id = models.CharField(max_length=255, blank=False, null=False) + transmission = models.ForeignKey( + ContentMetadataItemTransmission, + related_name='orphaned_record', + on_delete=models.CASCADE, + ) + resolved = models.BooleanField(default=False) + + +class IntegratedChannelAPIRequestLogs(TimeStampedModel): + """ + A model to track basic information about every API call we make from the integrated channels. + """ + + enterprise_customer = models.ForeignKey( + EnterpriseCustomer, on_delete=models.CASCADE + ) + enterprise_customer_configuration_id = models.IntegerField( + blank=False, + null=False, + help_text="ID from the EnterpriseCustomerConfiguration model", + ) + endpoint = models.URLField( + blank=False, + max_length=255, + null=False, + ) + payload = models.TextField(blank=False, null=False) + time_taken = models.FloatField(blank=False, null=False) + status_code = models.PositiveIntegerField( + help_text="API call response HTTP status code", blank=True, null=True + ) + response_body = models.TextField( + help_text="API call response body", blank=True, null=True + ) + channel_name = models.TextField( + help_text="Name of the integrated channel associated with this API call log record.", + blank=True + ) + + class Meta: + app_label = "integrated_channel" + verbose_name_plural = "Integrated channels API request logs" + + def __str__(self): + """ + Return a human-readable string representation of the object. + """ + return ( + f"" + f", endpoint: {self.endpoint}" + f", time_taken: {self.time_taken}" + f", response_body: {self.response_body}" + f", status_code: {self.status_code}" + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + @classmethod + def store_api_call( + cls, + enterprise_customer, + enterprise_customer_configuration_id, + endpoint, + payload, + time_taken, + status_code, + response_body, + channel_name + ): + """ + Creates new record in IntegratedChannelAPIRequestLogs table. + """ + try: + record = cls( + enterprise_customer=enterprise_customer, + enterprise_customer_configuration_id=enterprise_customer_configuration_id, + endpoint=endpoint, + payload=payload, + time_taken=time_taken, + status_code=status_code, + response_body=response_body, + channel_name=channel_name + ) + record.save() + except Exception as e: # pylint: disable=broad-except + LOGGER.error( + f"store_api_call raised error while storing API call: {e}" + f"enterprise_customer={enterprise_customer}" + f"enterprise_customer_configuration_id={enterprise_customer_configuration_id}," + f"endpoint={endpoint}" + f"payload={payload}" + f"time_taken={time_taken}" + f"status_code={status_code}" + f"response_body={response_body}" + f"channel_name={channel_name}" + ) diff --git a/channel_integrations/integrated_channel/tasks.py b/channel_integrations/integrated_channel/tasks.py new file mode 100644 index 0000000..6535405 --- /dev/null +++ b/channel_integrations/integrated_channel/tasks.py @@ -0,0 +1,481 @@ +""" +Celery tasks for integrated channel management commands. +""" + +import time +from functools import wraps + +from celery import shared_task +from celery.utils.log import get_task_logger +from edx_django_utils.monitoring import set_code_owner_attribute + +from django.conf import settings +from django.contrib import auth +from django.core.cache import cache +from django.utils import timezone + +from enterprise.utils import get_enterprise_uuids_for_user_and_course +from channel_integrations.integrated_channel.constants import TASK_LOCK_EXPIRY_SECONDS +from channel_integrations.integrated_channel.management.commands import ( + INTEGRATED_CHANNEL_CHOICES, + IntegratedChannelCommandUtils, +) +from channel_integrations.integrated_channel.models import ContentMetadataItemTransmission, OrphanedContentTransmissions +from channel_integrations.utils import generate_formatted_log + +LOGGER = get_task_logger(__name__) +User = auth.get_user_model() + + +def locked(expiry_seconds, lock_name_kwargs): + """ + A decorator to wrap a method in a cache-based lock with a cache-key derrived from function name and selected kwargs + """ + def task_decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): # lint-amnesty, pylint: disable=inconsistent-return-statements + cache_key = f'{func.__name__}' + for key in lock_name_kwargs: + cache_key += f'-{key}:{kwargs.get(key)}' + if cache.add(cache_key, "true", expiry_seconds): + exception = None + try: + LOGGER.info('Locking task in cache with key: %s for %s seconds', cache_key, expiry_seconds) + return func(*args, **kwargs) + except Exception as error: # lint-amnesty, pylint: disable=broad-except + LOGGER.exception(error) + exception = error + finally: + LOGGER.info('Unlocking task in cache with key: %s', cache_key) + cache.delete(cache_key) + if exception: + LOGGER.error(f'Re-raising exception from inside locked task: {type(exception).__name__}') + raise exception + else: + LOGGER.info('Task with key %s already exists in cache', cache_key) + return None + return wrapper + return task_decorator + + +def _log_batch_task_start(task_name, channel_code, job_user_id, integrated_channel_full_config, extra_message=''): + """ + Logs a consistent message on the start of a batch integrated channel task. + """ + LOGGER.info( + '[Integrated Channel: {channel_name}] Batch {task_name} started ' + '(api user: {job_user_id}). Configuration: {configuration}. {details}'.format( + channel_name=channel_code, + task_name=task_name, + job_user_id=job_user_id, + configuration=integrated_channel_full_config, + details=extra_message + )) + + +def _log_batch_task_finish(task_name, channel_code, job_user_id, + integrated_channel_full_config, duration_seconds, extra_message=''): + """ + Logs a consistent message on the end of a batch integrated channel task. + """ + + LOGGER.info( + '[Integrated Channel: {channel_name}] Batch {task_name} finished in {duration_seconds} ' + '(api user: {job_user_id}). Configuration: {configuration}. {details}'.format( + channel_name=channel_code, + task_name=task_name, + job_user_id=job_user_id, + configuration=integrated_channel_full_config, + duration_seconds=duration_seconds, + details=extra_message + )) + + +@shared_task +@set_code_owner_attribute +def remove_null_catalog_transmission_audits(): + """ + Task to remove content transmission audit records that do not contain a catalog UUID. + """ + start = time.time() + + _log_batch_task_start('remove_null_catalog_transmission_audits', None, None, None) + + deleted_null_catalog_uuids = ContentMetadataItemTransmission.objects.filter( + enterprise_customer_catalog_uuid=None + ).delete() + + duration_seconds = time.time() - start + _log_batch_task_finish( + 'remove_null_catalog_transmission_audits', + channel_code=None, + job_user_id=None, + integrated_channel_full_config=None, + duration_seconds=duration_seconds, + extra_message=f"{deleted_null_catalog_uuids[0]} transmission audits with no catalog UUIDs removed" + ) + + +@shared_task +@set_code_owner_attribute +def remove_duplicate_transmission_audits(): + """ + Task to remove duplicate transmission audits, keeping the most recently modified one. + """ + start = time.time() + _log_batch_task_start('remove_duplicate_transsmision_audits', None, None, None) + unique_transmissions = ContentMetadataItemTransmission.objects.values_list( + 'content_id', + 'plugin_configuration_id', + 'integrated_channel_code', + ).distinct() + duplicates_found = 0 + for unique_transmission in unique_transmissions: + content_id = unique_transmission[0] + duplicates = ContentMetadataItemTransmission.objects.filter( + id__in=ContentMetadataItemTransmission.objects.filter( + content_id=content_id, + plugin_configuration_id=unique_transmission[1], + integrated_channel_code=unique_transmission[2] + ).values_list('id', flat=True) + ).order_by('-modified') + # Subtract one because we're keeping the most recently modified one + num_duplicates = duplicates.count() - 1 + duplicates_found += num_duplicates + # Mysql doesn't support taking the count of a sliced queryset + duplicates_to_delete = duplicates[1:] + + dry_run_flag = getattr(settings, "DRY_RUN_MODE_REMOVE_DUP_TRANSMISSION_AUDIT", True) + LOGGER.info( + f"remove_duplicate_transmission_audits task dry run mode set to: {dry_run_flag}" + ) + if dry_run_flag: + LOGGER.info( + f"Found {num_duplicates} duplicate content transmission audits for course: {content_id}" + ) + else: + LOGGER.info(f'Beginning to delete duplicate content transmission audits for course: {content_id}') + for duplicate in duplicates_to_delete: + LOGGER.info(f"Deleting duplicate transmission audit: {duplicate.id}") + duplicate.delete() + + duration_seconds = time.time() - start + _log_batch_task_finish( + 'remove_duplicate_transsmision_audits', + channel_code=None, + job_user_id=None, + integrated_channel_full_config=None, + duration_seconds=duration_seconds, + extra_message=f"{duplicates_found} duplicates found" + ) + + +@shared_task +@set_code_owner_attribute +def mark_orphaned_content_metadata_audit(): + """ + Task to mark content metadata audits as orphaned if they are not linked to any customer catalogs. + """ + start = time.time() + _log_batch_task_start('mark_orphaned_content_metadata_audit', None, None, None) + + orphaned_metadata_audits = ContentMetadataItemTransmission.objects.none() + # Go over each integrated channel + for individual_channel in INTEGRATED_CHANNEL_CHOICES.values(): + try: + # Iterate through each configuration for the channel + for config in individual_channel.objects.all(): + # fetch orphaned content + orphaned_metadata_audits |= config.fetch_orphaned_content_audits() + except Exception as exc: # pylint: disable=broad-except + LOGGER.exception( + f'[Integrated Channel] mark_orphaned_content_metadata_audit failed with exception {exc}.', + exc_info=True + ) + # Generate orphaned content records for each fetched audit record + for orphaned_metadata_audit in orphaned_metadata_audits: + OrphanedContentTransmissions.objects.get_or_create( + integrated_channel_code=orphaned_metadata_audit.integrated_channel_code, + plugin_configuration_id=orphaned_metadata_audit.plugin_configuration_id, + transmission=orphaned_metadata_audit, + content_id=orphaned_metadata_audit.content_id, + ) + + duration = time.time() - start + _log_batch_task_finish( + 'mark_orphaned_content_metadata_audit', + channel_code=None, + job_user_id=None, + integrated_channel_full_config=None, + duration_seconds=duration, + extra_message=f'Orphaned content metadata audits marked: {orphaned_metadata_audits.count()}' + ) + + +@shared_task +@set_code_owner_attribute +@locked(expiry_seconds=TASK_LOCK_EXPIRY_SECONDS, lock_name_kwargs=['channel_code', 'channel_pk']) +def transmit_content_metadata(username, channel_code, channel_pk): + """ + Task to send content metadata to each linked integrated channel. + + Arguments: + username (str): The username of the User for making API requests to retrieve content metadata. + channel_code (str): Capitalized identifier for the integrated channel. + channel_pk (str): Primary key for identifying integrated channel. + + """ + start = time.time() + api_user = User.objects.get(username=username) + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + + _log_batch_task_start('transmit_content_metadata', channel_code, api_user.id, integrated_channel) + + try: + integrated_channel.transmit_content_metadata(api_user) + except Exception: # pylint: disable=broad-except + LOGGER.exception( + '[Integrated Channel: {channel_name}] Batch transmit_content_metadata failed with exception. ' + '(api user: {job_user_id}). Configuration: {configuration}'.format( + channel_name=channel_code, + job_user_id=api_user.id, + configuration=integrated_channel + ), exc_info=True) + + duration = time.time() - start + _log_batch_task_finish('transmit_content_metadata', channel_code, api_user.id, integrated_channel, duration) + + +@shared_task +@set_code_owner_attribute +@locked(expiry_seconds=TASK_LOCK_EXPIRY_SECONDS, lock_name_kwargs=['channel_code', 'channel_pk']) +def transmit_learner_data(username, channel_code, channel_pk): + """ + Task to send learner data to a linked integrated channel. + + Arguments: + username (str): The username of the User to be used for making API requests for learner data. + channel_code (str): Capitalized identifier for the integrated channel + channel_pk (str): Primary key for identifying integrated channel + """ + start = time.time() + api_user = User.objects.get(username=username) + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + _log_batch_task_start('transmit_learner_data', channel_code, api_user.id, integrated_channel) + + # Note: learner data transmission code paths don't raise any uncaught exception, + # so we don't need a broad try-except block here. + integrated_channel.transmit_learner_data(api_user) + + duration = time.time() - start + _log_batch_task_finish('transmit_learner_data', channel_code, api_user.id, integrated_channel, duration) + + +@shared_task +@set_code_owner_attribute +def cleanup_duplicate_assignment_records(username, channel_code, channel_pk): + """ + Task to remove transmitted duplicate assignment records of provided integrated channel. + + Arguments: + username (str): The username of the User to be used for making API requests for learner data. + channel_code (str): Capitalized identifier for the integrated channel + channel_pk (str): Primary key for identifying integrated channel + """ + start = time.time() + api_user = User.objects.get(username=username) + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + _log_batch_task_start('cleanup_duplicate_assignment_records', channel_code, api_user.id, integrated_channel) + + integrated_channel.cleanup_duplicate_assignment_records(api_user) + duration = time.time() - start + _log_batch_task_finish( + 'cleanup_duplicate_assignment_records', + channel_code, + api_user.id, + integrated_channel, + duration + ) + + +@shared_task +@set_code_owner_attribute +def update_content_transmission_catalog(username, channel_code, channel_pk): + """ + Task to retrieve all transmitted content items under a specific channel and update audits to contain the content's + associated catalog. + + Arguments: + username (str): The username of the User to be used for making API requests for learner data. + channel_code (str): Capitalized identifier for the integrated channel + channel_pk (str): Primary key for identifying integrated channel + """ + start = time.time() + api_user = User.objects.get(username=username) + + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + + _log_batch_task_start('update_content_transmission_catalog', channel_code, api_user.id, integrated_channel) + + integrated_channel.update_content_transmission_catalog(api_user) + duration = time.time() - start + _log_batch_task_finish( + 'update_content_transmission_catalog', + channel_code, + api_user.id, + integrated_channel, + duration + ) + + +@shared_task +@set_code_owner_attribute +def transmit_single_learner_data(username, course_run_id): + """ + Task to send single learner data to each linked integrated channel. + + Arguments: + username (str): The username of the learner whose data it should send. + course_run_id (str): The course run id of the course it should send data for. + """ + user = User.objects.get(username=username) + enterprise_customer_uuids = get_enterprise_uuids_for_user_and_course(user, course_run_id, is_customer_active=True) + + # Transmit the learner data to each integrated channel for each related customer. + # Starting Export. N customer is usually 1 but multiple are supported in codebase. + for enterprise_customer_uuid in enterprise_customer_uuids: + channel_utils = IntegratedChannelCommandUtils() + enterprise_integrated_channels = channel_utils.get_integrated_channels( + {'channel': None, 'enterprise_customer': enterprise_customer_uuid} + ) + for channel in enterprise_integrated_channels: + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel.channel_code()].objects.get(pk=channel.pk) + + LOGGER.info(generate_formatted_log( + integrated_channel.channel_code(), + enterprise_customer_uuid, + user.id, + course_run_id, + 'transmit_single_learner_data started.' + )) + + integrated_channel.transmit_single_learner_data( + learner_to_transmit=user, + course_run_id=course_run_id, + completed_date=timezone.now(), + grade='Pass', + is_passing=True + ) + LOGGER.info(generate_formatted_log( + integrated_channel.channel_code(), + enterprise_customer_uuid, + user.id, + course_run_id, + "transmit_single_learner_data finished." + )) + + +@shared_task +@set_code_owner_attribute +def transmit_single_subsection_learner_data(username, course_run_id, subsection_id, grade): + """ + Task to send an assessment level learner data record to each linked + integrated channel. This task is fired off + when an enterprise learner completes a subsection of their course, and + only sends the data for that sub-section. + + Arguments: + username (str): The username of the learner whose data it should send. + course_run_id (str): The course run id of the course it should send data for. + subsection_id (str): The completed subsection id whose grades are being reported. + grade (str): The grade received, used to ensure we are not sending duplicate transmissions. + """ + + user = User.objects.get(username=username) + enterprise_customer_uuids = get_enterprise_uuids_for_user_and_course(user, course_run_id, is_customer_active=True) + channel_utils = IntegratedChannelCommandUtils() + + # Transmit the learner data to each integrated channel for each related customer. + # Starting Export. N customer is usually 1 but multiple are supported in codebase. + for enterprise_customer_uuid in enterprise_customer_uuids: + enterprise_integrated_channels = channel_utils.get_integrated_channels( + {'channel': None, 'enterprise_customer': enterprise_customer_uuid, 'assessment_level_support': True} + ) + + for channel in enterprise_integrated_channels: + start = time.time() + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel.channel_code()].objects.get(pk=channel.pk) + + LOGGER.info(generate_formatted_log( + channel.channel_code(), + enterprise_customer_uuid, + user.id, + course_run_id, + 'transmit_single_subsection_learner_data for Subsection_id: {} started.'.format(subsection_id) + )) + + integrated_channel.transmit_single_subsection_learner_data( + learner_to_transmit=user, + course_run_id=course_run_id, + grade=grade, + subsection_id=subsection_id + ) + + duration = time.time() - start + LOGGER.info(generate_formatted_log( + None, + enterprise_customer_uuid, + user.id, + course_run_id, + 'transmit_single_subsection_learner_data for channels {channels} and for Subsection_id: ' + '{subsection_id} finished in {duration}s.'.format( + channels=[c.channel_code() for c in enterprise_integrated_channels], + subsection_id=subsection_id, + duration=duration) + )) + + +@shared_task +@set_code_owner_attribute +@locked(expiry_seconds=TASK_LOCK_EXPIRY_SECONDS, lock_name_kwargs=['channel_code', 'channel_pk']) +def transmit_subsection_learner_data(job_username, channel_code, channel_pk): + """ + Task to send assessment level learner data to a linked integrated channel. + + Arguments: + job_username (str): The username of the User making API requests for learner data. + channel_code (str): Capitalized identifier for the integrated channel + channel_pk (str): Primary key for identifying integrated channel + """ + start = time.time() + api_user = User.objects.get(username=job_username) + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + _log_batch_task_start('transmit_subsection_learner_data', channel_code, api_user.id, integrated_channel) + + # Exceptions during transmission are caught and saved within the audit so no need to try/catch here + integrated_channel.transmit_subsection_learner_data(api_user) + duration = time.time() - start + _log_batch_task_finish('transmit_subsection_learner_data', channel_code, api_user.id, integrated_channel, duration) + + +@shared_task +@set_code_owner_attribute +def unlink_inactive_learners(channel_code, channel_pk): + """ + Task to unlink inactive learners of provided integrated channel. + + Arguments: + channel_code (str): Capitalized identifier for the integrated channel + channel_pk (str): Primary key for identifying integrated channel + """ + start = time.time() + integrated_channel = INTEGRATED_CHANNEL_CHOICES[channel_code].objects.get(pk=channel_pk) + + _log_batch_task_start('unlink_inactive_learners', channel_code, None, integrated_channel) + + # Note: learner data transmission code paths don't raise any uncaught exception, so we don't need a broad + # try-except block here. + integrated_channel.unlink_inactive_learners() + + duration = time.time() - start + _log_batch_task_finish('unlink_inactive_learners', channel_code, None, integrated_channel, duration) diff --git a/channel_integrations/integrated_channel/transmitters/__init__.py b/channel_integrations/integrated_channel/transmitters/__init__.py new file mode 100644 index 0000000..ef1a278 --- /dev/null +++ b/channel_integrations/integrated_channel/transmitters/__init__.py @@ -0,0 +1,31 @@ +""" +Package for generic data transmitters which send serialized data to integrated channels. +""" + + +class Transmitter: + """ + Interface for transmitting data to an integrated channel. + + The interface contains the following method(s): + + transmit(payload) + payload - The ``Exporter`` object expected to implement an ``export`` method that returns serialized data. + """ + + def __init__(self, enterprise_configuration, client=None): + """ + Prepares a configuration and a client to be used to transmit data to an integrated channel. + + Arguments: + * enterprise_configuration - The configuration connecting an enterprise to an integrated channel. + * client - The REST API client that'll transmit serialized data. + """ + self.enterprise_configuration = enterprise_configuration + self.client = client(enterprise_configuration) if client else None + + def transmit(self, create_payload, update_payload, delete_payload): + """ + The abstract interface method for sending exported data to an integrated channel through its API client. + """ + raise NotImplementedError('Implement in concrete subclass transmitter.') diff --git a/channel_integrations/integrated_channel/transmitters/content_metadata.py b/channel_integrations/integrated_channel/transmitters/content_metadata.py new file mode 100644 index 0000000..1514233 --- /dev/null +++ b/channel_integrations/integrated_channel/transmitters/content_metadata.py @@ -0,0 +1,264 @@ +""" +Generic content metadata transmitter for integrated channels. +""" + +import functools +import json +import logging +from itertools import islice + +import requests + +from django.apps import apps +from django.conf import settings + +from enterprise.utils import localized_utcnow, truncate_string +from channel_integrations.exceptions import ClientError +from channel_integrations.integrated_channel.client import IntegratedChannelApiClient +from channel_integrations.integrated_channel.transmitters import Transmitter +from channel_integrations.utils import chunks, encode_binary_data_for_logging, generate_formatted_log + +LOGGER = logging.getLogger(__name__) + + +class ContentMetadataTransmitter(Transmitter): + """ + Used to transmit content metadata to an integrated channel. + """ + + # a 'magic number' to designate an unknown error + UNKNOWN_ERROR_HTTP_STATUS_CODE = 555 + + def __init__(self, enterprise_configuration, client=IntegratedChannelApiClient): + """ + By default, use the abstract integrated channel API client which raises an error when used if not subclassed. + """ + super().__init__( + enterprise_configuration=enterprise_configuration, + client=client + ) + self._transmit_create = functools.partial( + self._transmit_action, + client_method=self.client.create_content_metadata, + action_name='create', + ) + self._transmit_update = functools.partial( + self._transmit_action, + client_method=self.client.update_content_metadata, + action_name='update', + ) + self._transmit_delete = functools.partial( + self._transmit_action, + client_method=self.client.delete_content_metadata, + action_name='delete', + ) + + def _log_info(self, msg, course_or_course_run_key=None): + LOGGER.info( + generate_formatted_log( + channel_name=self.enterprise_configuration.channel_code(), + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=course_or_course_run_key, + plugin_configuration_id=self.enterprise_configuration.id, + message=msg + ) + ) + + def _log_error(self, msg, course_or_course_run_key=None): + LOGGER.error( + generate_formatted_log( + channel_name=self.enterprise_configuration.channel_code(), + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=course_or_course_run_key, + plugin_configuration_id=self.enterprise_configuration.id, + message=msg + ) + ) + + def _log_info_for_each_item_map(self, item_map, msg): + for content_id, transmission in item_map.items(): + self._log_info( + f'integrated_channel_content_transmission_id={transmission.id}, ' + f'{msg}', + course_or_course_run_key=content_id + ) + + def transmit(self, create_payload, update_payload, delete_payload): + """ + Transmit content metadata items to the integrated channel. Save or update content metadata records according to + the type of transmission. + """ + delete_payload_results = self._transmit_delete(delete_payload) + + create_payload_results = self._transmit_create(create_payload) + + update_payload_results = self._transmit_update(update_payload) + return create_payload_results, update_payload_results, delete_payload_results + + def _prepare_items_for_transmission(self, channel_metadata_items): + """ + Perform any necessary modifications to content metadata item + data structure before transmission. This can be overridden by + subclasses to add any data structure wrappers expected by the + integrated channel. + """ + return channel_metadata_items + + def _prepare_items_for_delete(self, channel_metadata_items): + """ + Perform any necessary modifications to content metadata item + data structure before delete. This can be overridden by + subclasses to add any data structure wrappers expected by the + integrated channel. + """ + return channel_metadata_items + + def _serialize_items(self, channel_metadata_items): + """ + Serialize content metadata items for a create transmission to the integrated channel. + """ + return json.dumps( + self._prepare_items_for_transmission(channel_metadata_items), + sort_keys=True + ).encode('utf-8') + + def _filter_api_response(self, response, content_id): # pylint: disable=unused-argument + """ + Filter the response from the integrated channel API client. + This can be overridden by subclasses to parse the response + expected by the integrated channel. + """ + return response + + def _transmit_action(self, content_metadata_item_map, client_method, action_name): # pylint: disable=too-many-statements + """ + Do the work of calling the appropriate client method, saving the results, and updating + the appropriate timestamps + """ + results = [] + chunk_items = chunks(content_metadata_item_map, self.enterprise_configuration.transmission_chunk_size) + transmission_limit = settings.INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT.get( + self.enterprise_configuration.channel_code() + ) + + # If we're deleting, fetch all orphaned, unresolved content transmissions + is_delete_action = action_name == 'delete' + successfully_removed_content_keys = [] + + for chunk in islice(chunk_items, transmission_limit): + json_payloads = [item.channel_metadata for item in list(chunk.values())] + serialized_chunk = self._serialize_items(json_payloads) + if self.enterprise_configuration.dry_run_mode_enabled: + enterprise_customer_uuid = self.enterprise_configuration.enterprise_customer.uuid + channel_code = self.enterprise_configuration.channel_code() + for key, item in chunk.items(): + payload = item.channel_metadata + serialized_payload = self._serialize_items([payload]) + encoded_serialized_payload = encode_binary_data_for_logging(serialized_payload) + LOGGER.info(generate_formatted_log( + channel_code, + enterprise_customer_uuid, + None, + key, + f'dry-run mode content metadata ' + f'skipping "{action_name}" action for content metadata transmission ' + f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}' + )) + continue + + response_status_code = None + response_body = None + try: + response_status_code, response_body = client_method(serialized_chunk) + except ClientError as exc: + LOGGER.exception(exc) + response_status_code = exc.status_code + response_body = str(exc) + self._log_error( + f"Failed to {action_name} [{len(chunk)}] content metadata items for integrated channel " + f"[{self.enterprise_configuration.enterprise_customer.name}] " + f"[{self.enterprise_configuration.channel_code()}]. " + f"Task failed with message [{response_body}] and status code [{response_status_code}]" + ) + except requests.exceptions.RequestException as exc: + LOGGER.exception(exc) + if exc.response: + response_status_code = exc.response.status_code + response_body = exc.response.text + else: + response_status_code = self.UNKNOWN_ERROR_HTTP_STATUS_CODE + response_body = str(exc) + self._log_error( + f"Failed to {action_name} [{len(chunk)}] content metadata items for integrated channel " + f"[{self.enterprise_configuration.enterprise_customer.name}] " + f"[{self.enterprise_configuration.channel_code()}]. " + f"Task failed with message [{str(exc)}] and status code [{response_status_code}]" + ) + except Exception as exc: # pylint: disable=broad-except + LOGGER.exception(exc) + response_status_code = self.UNKNOWN_ERROR_HTTP_STATUS_CODE + response_body = str(exc) + self._log_error( + f"Failed to {action_name} [{len(chunk)}] content metadata items for integrated channel " + f"[{self.enterprise_configuration.enterprise_customer.name}] " + f"[{self.enterprise_configuration.channel_code()}]. " + f"Task failed with message [{response_body}]" + ) + finally: + action_happened_at = localized_utcnow() + for content_id, transmission in chunk.items(): + transmission.api_response_status_code = response_status_code + was_successful = response_status_code < 300 + api_content_response = response_body + if was_successful: + api_content_response = self._filter_api_response(api_content_response, content_id) + (api_content_response, was_truncated) = truncate_string(api_content_response) + if was_truncated: + self._log_info( + f'integrated_channel_content_transmission_id={transmission.id}, ' + f'api response truncated', + course_or_course_run_key=content_id + ) + if transmission.api_record: + transmission.api_record.body = api_content_response + transmission.api_record.status_code = response_status_code + transmission.api_record.save() + else: + ApiResponseRecord = apps.get_model( + 'integrated_channel', + 'ApiResponseRecord' + ) + transmission.api_record = ApiResponseRecord.objects.create( + body=api_content_response, status_code=response_status_code + ) + if action_name == 'create': + transmission.remote_created_at = action_happened_at + elif action_name == 'update': + transmission.remote_updated_at = action_happened_at + elif is_delete_action: + transmission.remote_deleted_at = action_happened_at + if was_successful: + successfully_removed_content_keys.append(transmission.content_id) + if was_successful: + transmission.remove_marked_for() + transmission.remote_errored_at = None + else: + transmission.remote_errored_at = action_happened_at + transmission.save() + self.enterprise_configuration.update_content_synced_at(action_happened_at, was_successful) + results.append(transmission) + + if is_delete_action and successfully_removed_content_keys: + # Mark any successfully deleted, orphaned content transmissions as resolved + OrphanedContentTransmissions = apps.get_model( + 'integrated_channel', + 'OrphanedContentTransmissions' + ) + orphaned_items = OrphanedContentTransmissions.objects.filter( + integrated_channel_code=self.enterprise_configuration.channel_code(), + plugin_configuration_id=self.enterprise_configuration.id, + resolved=False, + ) + orphaned_items.filter(content_id__in=successfully_removed_content_keys).update(resolved=True) + + return results diff --git a/channel_integrations/integrated_channel/transmitters/learner_data.py b/channel_integrations/integrated_channel/transmitters/learner_data.py new file mode 100644 index 0000000..07a5a6d --- /dev/null +++ b/channel_integrations/integrated_channel/transmitters/learner_data.py @@ -0,0 +1,488 @@ +""" +Generic learner data transmitter for integrated channels. +""" + +import logging +from http import HTTPStatus + +from django.apps import apps + +from enterprise.utils import localized_utcnow +from channel_integrations.exceptions import ClientError +from channel_integrations.integrated_channel.channel_settings import ChannelSettingsMixin +from channel_integrations.integrated_channel.client import IntegratedChannelApiClient +from channel_integrations.integrated_channel.exporters.learner_data import LearnerExporterUtility +from channel_integrations.integrated_channel.transmitters import Transmitter +from channel_integrations.utils import encode_data_for_logging, generate_formatted_log, is_already_transmitted + +LOGGER = logging.getLogger(__name__) + + +class LearnerTransmitter(Transmitter, ChannelSettingsMixin): + """ + A generic learner data transmitter. + + It may be subclassed by specific integrated channel learner data transmitters for + each integrated channel's particular learner data transmission requirements and expectations. + """ + + def __init__(self, enterprise_configuration, client=IntegratedChannelApiClient): + """ + By default, use the abstract integrated channel API client which raises an error when used if not subclassed. + """ + super().__init__( + enterprise_configuration=enterprise_configuration, + client=client + ) + + def _generate_common_params(self, **kwargs): + """ Pulls labeled common params out of kwargs """ + app_label = kwargs.get('app_label', 'integrated_channel') + enterprise_customer_uuid = self.enterprise_configuration.enterprise_customer.uuid or None + lms_user_id = kwargs.get('learner_to_transmit', None) + return app_label, enterprise_customer_uuid, lms_user_id + + def single_learner_assessment_grade_transmit(self, exporter, **kwargs): + """ + Send an assessment level grade information to the integrated channel using the client. + + Args: + exporter: The ``LearnerExporter`` instance used to send to the integrated channel. + kwargs: Contains integrated channel-specific information for customized transmission variables. + - app_label: The app label of the integrated channel for whom to store learner data records for. + - model_name: The name of the specific learner data record model to use. + - remote_user_id: The remote ID field name on the audit model that will map to the learner. + """ + app_label, enterprise_customer_uuid, lms_user_id = self._generate_common_params(**kwargs) + TransmissionAudit = apps.get_model( + app_label=app_label, + model_name=kwargs.get('model_name', 'GenericLearnerDataTransmissionAudit'), + ) + kwargs.update( + TransmissionAudit=TransmissionAudit, + ) + kwargs.update(channel_name=app_label) + + if self.enterprise_configuration.disable_learner_data_transmissions: + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + None, + "Single learner data assessment level transmission skipped as customer's configuration has marked " + "learner data reporting as disabled." + )) + return + + # Even though we're transmitting a learner, they can have records per assessment (multiple per course). + for learner_data in exporter.single_assessment_level_export(**kwargs): + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'create_assessment_reporting started for ' + f'integrated_channel_enterprise_enrollment_id={learner_data.enterprise_course_enrollment_id}' + )) + + serialized_payload = learner_data.serialize(enterprise_configuration=self.enterprise_configuration) + + if self.enterprise_configuration.dry_run_mode_enabled: + remote_id = getattr(learner_data, kwargs.get('remote_user_id')) + encoded_serialized_payload = encode_data_for_logging(serialized_payload) + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'dry-run mode ' + 'skipping create_assessment_reporting for enrollment ' + f'integrated_channel_enterprise_enrollment_id={learner_data.enterprise_course_enrollment_id}, ' + f'integrated_channel_remote_user_id={remote_id}, ' + f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}' + )) + continue + + try: + code, body = self.client.create_assessment_reporting( + getattr(learner_data, kwargs.get('remote_user_id')), + serialized_payload + ) + except ClientError as client_error: + code = client_error.status_code + body = client_error.message + self.process_transmission_error( + learner_data, + client_error, + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + + except Exception: + # Log additional data to help debug failures but just have Exception bubble + self._log_exception_supplemental_data( + learner_data, + 'create_assessment_reporting', + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + raise + + learner_data.status = str(code) + learner_data.error_message = body if code >= 400 else '' + if code < 400 and learner_data.error_message == '': + learner_data.is_transmitted = True + learner_data.save() + + def assessment_level_transmit(self, exporter, **kwargs): + """ + Send all assessment level grade information under an enterprise enrollment to the integrated channel using the + client. + + Args: + exporter: The learner assessment data exporter used to send to the integrated channel. + kwargs: Contains integrated channel-specific information for customized transmission variables. + - app_label: The app label of the integrated channel for whom to store learner data records for. + - model_name: The name of the specific learner data record model to use. + - remote_user_id: The remote ID field name of the learner on the audit model. + """ + app_label, enterprise_customer_uuid, _ = self._generate_common_params(**kwargs) + TransmissionAudit = apps.get_model( + app_label=app_label, + model_name=kwargs.get('model_name', 'GenericLearnerDataTransmissionAudit'), + ) + kwargs.update( + TransmissionAudit=TransmissionAudit, + ) + + if self.enterprise_configuration.disable_learner_data_transmissions: + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + None, + None, + "Bulk learner data assessment level transmission skipped as customer's configuration has marked " + "learner data reporting as disabled." + )) + return + + # Retrieve learner data for each existing enterprise enrollment under the enterprise customer + # and transmit the data according to the current enterprise configuration. + for learner_data in exporter.bulk_assessment_level_export(): + serialized_payload = learner_data.serialize(enterprise_configuration=self.enterprise_configuration) + enterprise_enrollment_id = learner_data.enterprise_course_enrollment_id + lms_user_id = LearnerExporterUtility.lms_user_id_for_ent_course_enrollment_id( + enterprise_enrollment_id + ) + + # Check the last transmission for the current enrollment and see if the old grades match the new ones + # note: the property self.include_grade_for_completion_audit_check comes from ChannelSettingsMixin + if is_already_transmitted( + TransmissionAudit, + enterprise_enrollment_id, + self.enterprise_configuration.id, + learner_data.grade, + learner_data.subsection_id, + detect_grade_updated=self.INCLUDE_GRADE_FOR_COMPLETION_AUDIT_CHECK, + ): + # We've already sent a completion status for this enrollment + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'Skipping previously sent ' + f'integrated_channel_enterprise_enrollment_id={enterprise_enrollment_id}' + )) + continue + + if self.enterprise_configuration.dry_run_mode_enabled: + remote_id = getattr(learner_data, kwargs.get('remote_user_id')) + encoded_serialized_payload = encode_data_for_logging(serialized_payload) + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'dry-run mode ' + 'skipping create_assessment_reporting for enrollment ' + f'integrated_channel_enterprise_enrollment_id={enterprise_enrollment_id}, ' + f'integrated_channel_remote_user_id={remote_id}, ' + f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}' + )) + continue + + try: + code, body = self.client.create_assessment_reporting( + getattr(learner_data, kwargs.get('remote_user_id')), + serialized_payload + ) + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'create_assessment_reporting successfully completed for ' + f'integrated_channel_enterprise_enrollment_id={enterprise_enrollment_id}' + )) + except ClientError as client_error: + code = client_error.status_code + body = client_error.message + self.process_transmission_error( + learner_data, + client_error, + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + + except Exception: + # Log additional data to help debug failures but just have Exception bubble + self._log_exception_supplemental_data( + learner_data, + 'create_assessment_reporting', + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + raise + + learner_data.status = str(code) + learner_data.error_message = body if code >= 400 else '' + learner_data.is_transmitted = code < 400 + learner_data.save() + + def transmit(self, payload, **kwargs): # pylint: disable=arguments-differ + """ + Send a completion status call to the integrated channel using the client. + + Args: + payload: The learner data exporter. + kwargs: Contains integrated channel-specific information for customized transmission variables. + - app_label: The app label of the integrated channel for whom to store learner data records for. + - model_name: The name of the specific learner data record model to use. + - remote_user_id: The remote ID field name of the learner on the audit model. + """ + app_label, enterprise_customer_uuid, _ = self._generate_common_params(**kwargs) + TransmissionAudit = apps.get_model( + app_label=app_label, + model_name=kwargs.get('model_name', 'GenericLearnerDataTransmissionAudit'), + ) + kwargs.update( + TransmissionAudit=TransmissionAudit, + ) + + if self.enterprise_configuration.disable_learner_data_transmissions: + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + None, + None, + "Completion level learner data transmission skipped as customer's configuration has marked learner data" + " reporting as disabled." + )) + return + + # Since we have started sending courses to integrated channels instead of course runs, + # we need to attempt to send transmissions with course keys and course run ids in order to + # ensure that we account for whether courses or course runs exist in the integrated channel. + # The exporters have been changed to return multiple transmission records to attempt, + # one by course key and one by course run id. + # If the transmission with the course key succeeds, the next one will get skipped. + # If it fails, the one with the course run id will be attempted and (presumably) succeed. + + for learner_data in payload.export(**kwargs): + serialized_payload = learner_data.serialize(enterprise_configuration=self.enterprise_configuration) + + enterprise_enrollment_id = learner_data.enterprise_course_enrollment_id + lms_user_id = LearnerExporterUtility.lms_user_id_for_ent_course_enrollment_id( + enterprise_enrollment_id + ) + + if (not learner_data.course_completed and + not getattr(self.enterprise_configuration, 'enable_incomplete_progress_transmission', False)): + # The user has not completed the course and enable_incomplete_progress_transmission is not set, + # so we shouldn't send a completion status call + remote_id = getattr(learner_data, kwargs.get('remote_user_id')) + encoded_serialized_payload = encode_data_for_logging(serialized_payload) + continue + + grade = getattr(learner_data, 'grade', None) + if is_already_transmitted( + TransmissionAudit, + enterprise_enrollment_id, + self.enterprise_configuration.id, + grade, + detect_grade_updated=self.INCLUDE_GRADE_FOR_COMPLETION_AUDIT_CHECK, + ): + # We've already sent a completion status for this enrollment + continue + + if self.enterprise_configuration.dry_run_mode_enabled: + remote_id = getattr(learner_data, kwargs.get('remote_user_id')) + encoded_serialized_payload = encode_data_for_logging(serialized_payload) + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'dry-run mode ' + f'integrated_channel_enterprise_enrollment_id={enterprise_enrollment_id}, ' + f'integrated_channel_remote_user_id={remote_id}, ' + f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}' + )) + continue + + try: + code, body = self.client.create_course_completion( + getattr(learner_data, kwargs.get('remote_user_id')), + serialized_payload + ) + if code >= HTTPStatus.BAD_REQUEST.value: + raise ClientError(f'Client create_course_completion failed: {body}', code) + + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id, + 'Successfully sent completion status call for enterprise enrollment ' + f'integrated_channel_enterprise_enrollment_id={enterprise_enrollment_id}' + )) + except ClientError as client_error: + code = client_error.status_code + body = client_error.message + self.process_transmission_error( + learner_data, + client_error, + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + + except Exception: + # Log additional data to help debug failures but have Exception bubble + self._log_exception_supplemental_data( + learner_data, + 'create_assessment_reporting', + app_label, + enterprise_customer_uuid, + lms_user_id, + learner_data.course_id + ) + raise + + action_happened_at = localized_utcnow() + was_successful = code < 300 + learner_data.status = str(code) + learner_data.error_message = body if not was_successful else '' + learner_data.is_transmitted = was_successful + learner_data.save() + self.enterprise_configuration.update_learner_synced_at(action_happened_at, was_successful) + + def deduplicate_assignment_records_transmit(self, exporter, **kwargs): + """ + Remove duplicated assessments sent to the integrated channel using the client. + + Args: + exporter: The learner completion data payload to send to the integrated channel. + kwargs: Contains integrated channel-specific information for customized transmission variables. + - app_label: The app label of the integrated channel for whom to store learner data records for. + - model_name: The name of the specific learner data record model to use. + - remote_user_id: The remote ID field name of the learner on the audit model. + """ + app_label, enterprise_customer_uuid, _ = self._generate_common_params(**kwargs) + courses = exporter.export_unique_courses() + + if self.enterprise_configuration.dry_run_mode_enabled: + LOGGER.info(generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + None, + None, + 'dry-run mode ' + 'skipping deduplicate_assignment_records_transmit' + )) + return + + code, body = self.client.cleanup_duplicate_assignment_records(courses) + + if code >= 400: + LOGGER.exception( + generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + None, + None, + f'{app_label} Deduping assignments transmission experienced a failure, ' + f'received the error message: {body}' + ) + ) + else: + LOGGER.info( + generate_formatted_log( + self.enterprise_configuration.channel_code(), + enterprise_customer_uuid, + None, + None, + f'{app_label} Deduping assignments transmission finished successfully, ' + f'received message: {body}' + ) + ) + + def _log_exception_supplemental_data(self, learner_data, operation_name, + integrated_channel_name, enterprise_customer_uuid, learner_id, course_id): + """ Logs extra payload and parameter data to help debug which learner data caused an exception. """ + LOGGER.exception(generate_formatted_log( + self.enterprise_configuration.channel_code(), enterprise_customer_uuid, learner_id, course_id, + '{operation_name} {integrated_channel_name} failed with Exception for ' + 'enterprise enrollment {enrollment_id} with payload {payload}'.format( + operation_name=operation_name, + integrated_channel_name=integrated_channel_name, + enrollment_id=learner_data.enterprise_course_enrollment_id, + payload=learner_data + )), exc_info=True) + + def handle_transmission_error(self, learner_data, client_exception): + """ + Subclasses who wish to do additional processing of transmission error + before logging, can do so in overrides of this method + """ + + def process_transmission_error( + self, + learner_data, + client_exception, + integrated_channel_name, + enterprise_customer_uuid, + learner_id, + course_id, + ): + """ + applies any needed processing of transmission error before logging it + subclasses should override handle_transmission_error if they want to do additional + processing of the error before it's logged + """ + self.handle_transmission_error( + learner_data, + client_exception, + ) + serialized_payload = learner_data.serialize(enterprise_configuration=self.enterprise_configuration) + encoded_serialized_payload = encode_data_for_logging(serialized_payload) + LOGGER.exception( + generate_formatted_log( + self.enterprise_configuration.channel_code(), enterprise_customer_uuid, learner_id, course_id, + f'Failed to send completion status call for {integrated_channel_name} ' + f'integrated_channel_enterprise_enrollment_id={learner_data.enterprise_course_enrollment_id}, ' + f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}, ' + f'Error message: {client_exception.message} ' + f'Error status code: {client_exception.status_code}' + ) + ) diff --git a/channel_integrations/lms_utils.py b/channel_integrations/lms_utils.py new file mode 100644 index 0000000..759bb14 --- /dev/null +++ b/channel_integrations/lms_utils.py @@ -0,0 +1,131 @@ +""" +A utility collection for calls from integrated_channels to LMS APIs +If integrated_channels calls LMS APIs, put them here for better tracking. +""" +from opaque_keys.edx.keys import CourseKey + +try: + from lms.djangoapps.certificates.api import get_certificate_for_user + from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory + from lms.djangoapps.grades.models import PersistentCourseGrade + from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +except ImportError: + get_certificate_for_user = None + CourseGradeFactory = None + CourseOverview = None + PersistentCourseGrade = None + +try: + from lms.djangoapps.courseware.courses import get_course_blocks_completion_summary +except ImportError: + get_course_blocks_completion_summary = None + +from enterprise.utils import NotConnectedToOpenEdX + + +def get_persistent_grade(course_key, user): + """ + Get the persistent course grade record for this course and user, or None + """ + try: + grade = PersistentCourseGrade.read(user.id, course_key) + except PersistentCourseGrade.DoesNotExist: + return None + return grade + + +def get_course_certificate(course_key, user): + """ + A course certificate for a user (must be a django.contrib.auth.User instance). + If there is a problem loading the get_certificate_for_user function, throws NotConnectedToOpenEdX + If issues with course_key string, throws a InvalidKeyError + Arguments: + course_key (string): course key + user (django.contrib.auth.User): user instance + Returns a certificate as a dict, for example: + { + "username": "bob", + "course_id": "edX/DemoX/Demo_Course", + "certificate_type": "verified", + "created_date": "2015-12-03T13:14:28+0000", + "status": "downloadable", + "is_passing": true, + "download_url": "http://www.example.com/cert.pdf", + "grade": "0.98" + } + """ + if not get_certificate_for_user: + raise NotConnectedToOpenEdX( + 'To use this function, this package must be ' + 'installed in an Open edX environment.' + ) + course_id = CourseKey.from_string(course_key) + user_cert = get_certificate_for_user(username=user.username, course_key=course_id) + return user_cert + + +def get_single_user_grade(course_key, user): + """ + Returns a grade for the user (must be a django.contrib.auth.User instance). + If there is a problem loading the CourseGradeFactory class, throws NotConnectedToOpenEdX + If issues with course_key string, throws a InvalidKeyError + Args: + course_key (string): string course key + user (django.contrib.auth.User): user instance + + Returns: + A CourseGrade object with at least these fields: + - percent (Number) + - passed (Boolean) + """ + if not CourseGradeFactory: + raise NotConnectedToOpenEdX( + 'To use this function, this package must be ' + 'installed in an Open edX environment.' + ) + course_id = CourseKey.from_string(course_key) + course_grade = CourseGradeFactory().read(user, course_key=course_id, create_if_needed=False) + return course_grade + + +def get_course_details(course_key): + """ + Args: + course_key (string): string course key + Returns: + course_overview: (openedx.core.djangoapps.content.course_overviews.models.CourseOverview) + + If there is a problem loading the CourseOverview class, throws NotConnectedToOpenEdX + If issues with course_key string, throws a InvalidKeyError + If course details not found, throws CourseOverview.DoesNotExist + """ + if not CourseOverview: + raise NotConnectedToOpenEdX( + 'To use this function, this package must be ' + 'installed in an Open edX environment.' + ) + course_id = CourseKey.from_string(course_key) + course_overview = CourseOverview.get_from_id(course_id) + return course_overview + + +def get_completion_summary(course_key, user): + """ + Fetch completion summary for course + user using course blocks completions api + Args: + course_key (string): string course key + user (django.contrib.auth.User): user instance + Returns: + object containing fields: complete_count, incomplete_count, locked_count + + If there is a problem loading the CourseOverview class, throws NotConnectedToOpenEdX + If issues with course_key string, throws a InvalidKeyError + If course details not found, throws CourseOverview.DoesNotExist + """ + if not get_course_blocks_completion_summary: + raise NotConnectedToOpenEdX( + 'To use get_course_blocks_completion_summary() function, this package must be ' + 'installed in an Open edX environment.' + ) + course_id = CourseKey.from_string(course_key) + return get_course_blocks_completion_summary(course_id, user) diff --git a/channel_integrations/utils.py b/channel_integrations/utils.py new file mode 100644 index 0000000..f22bfcd --- /dev/null +++ b/channel_integrations/utils.py @@ -0,0 +1,565 @@ +""" +Utilities common to different integrated channels. +""" + +import base64 +import itertools +import json +import math +import re +from datetime import datetime, timedelta +from itertools import islice +from logging import getLogger +from string import Formatter +from urllib.parse import urlparse + +import pytz +import requests + +from django.apps import apps +from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q +from django.utils import timezone +from django.utils.html import strip_tags + +from enterprise.utils import parse_datetime_handle_invalid, parse_lms_api_datetime +from channel_integrations.catalog_service_utils import get_course_run_for_enrollment + +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +UNIX_MIN_DATE_STRING = '1970-01-01T00:00:00Z' +UNIX_MAX_DATE_STRING = '2038-01-19T03:14:07Z' + +LOGGER = getLogger(__name__) + + +def encode_data_for_logging(data): + """ + Converts input into URL-safe, utf-8 encoded, base64 encoded output + If the input is other than a string, it is dumped to json + """ + if not isinstance(data, str): + data = json.dumps(data) + return base64.urlsafe_b64encode(data.encode("utf-8")).decode('utf-8') + + +def encode_binary_data_for_logging(data): + """ + Converts binary input into URL-safe, utf-8 encoded, base64 encoded output. + If the input is binary (bytes), it is first decoded to utf-8, then dumped to JSON, + and finally, base64 encoded. + """ + if not isinstance(data, str): + try: + data = json.dumps(data.decode('utf-8')) + except (UnicodeDecodeError, AttributeError): + # Handle decoding errors or attribute errors (e.g., if 'data' is not bytes) + data = json.dumps(data) + return base64.urlsafe_b64encode(data.encode("utf-8")).decode('utf-8') + + +def parse_datetime_to_epoch(datestamp, magnitude=1.0): + """ + Convert an ISO-8601 datetime string to a Unix epoch timestamp in some magnitude. + + By default, returns seconds. + """ + parsed_datetime = parse_lms_api_datetime(datestamp) + time_since_epoch = parsed_datetime - UNIX_EPOCH + return int(time_since_epoch.total_seconds() * magnitude) + + +def strip_html_tags(text, strip_entities=True): + """ + Return (str): Text without any html tags and entities. + + Args: + text (str): text having html tags + strip_entities (bool): If set to True html entities are also stripped + """ + text = strip_tags(text) + if strip_entities: + text = re.sub(r'&([a-zA-Z]{4,5}|#[0-9]{2,4});', '', text) + return text + + +def parse_datetime_to_epoch_millis(datestamp): + """ + Convert an ISO-8601 datetime string to a Unix epoch timestamp in milliseconds. + """ + return parse_datetime_to_epoch(datestamp, magnitude=1000.0) + + +def current_time_is_in_interval(start, end): + """ + Determine whether the current time is on the interval [start, end]. + """ + interval_start = parse_lms_api_datetime(start or UNIX_MIN_DATE_STRING) + interval_end = parse_lms_api_datetime(end or UNIX_MAX_DATE_STRING) + return interval_start <= timezone.now() <= interval_end + + +def chunks(dictionary, chunk_size): + """ + Yield successive n-sized chunks from dictionary. + """ + iterable = iter(dictionary) + for __ in range(0, len(dictionary), chunk_size): + yield {key: dictionary[key] for key in islice(iterable, chunk_size)} + + +def strfdelta(tdelta, fmt='{D:02}d {H:02}h {M:02}m {S:02}s', input_type='timedelta'): + """ + Convert a datetime.timedelta object or a regular number to a custom-formatted string. + + This function works like the strftime() method works for datetime.datetime + objects. + + The fmt argument allows custom formatting to be specified. Fields can + include seconds, minutes, hours, days, and weeks. Each field is optional. + + Arguments: + tdelta (datetime.timedelta, int): time delta object containing the duration or an integer + to go with the input_type. + fmt (str): Expected format of the time delta. place holders can only be one of the following. + 1. D to extract days from time delta + 2. H to extract hours from time delta + 3. M to extract months from time delta + 4. S to extract seconds from timedelta + input_type (str): The input_type argument allows tdelta to be a regular number instead of the + default, which is a datetime.timedelta object. + Valid input_type strings: + 1. 's', 'seconds', + 2. 'm', 'minutes', + 3. 'h', 'hours', + 4. 'd', 'days', + 5. 'w', 'weeks' + Returns: + (str): timedelta object interpolated into a string following the given format. + + Examples: + '{D:02}d {H:02}h {M:02}m {S:02}s' --> '05d 08h 04m 02s' (default) + '{W}w {D}d {H}:{M:02}:{S:02}' --> '4w 5d 8:04:02' + '{D:2}d {H:2}:{M:02}:{S:02}' --> ' 5d 8:04:02' + '{H}h {S}s' --> '72h 800s' + """ + # Convert tdelta to integer seconds. + if input_type == 'timedelta': + remainder = int(tdelta.total_seconds()) + elif input_type in ['s', 'seconds']: + remainder = int(tdelta) + elif input_type in ['m', 'minutes']: + remainder = int(tdelta) * 60 + elif input_type in ['h', 'hours']: + remainder = int(tdelta) * 3600 + elif input_type in ['d', 'days']: + remainder = int(tdelta) * 86400 + elif input_type in ['w', 'weeks']: + remainder = int(tdelta) * 604800 + else: + raise ValueError( + 'input_type is not valid. Valid input_type strings are: "timedelta", "s", "m", "h", "d", "w"' + ) + + f = Formatter() + desired_fields = [field_tuple[1] for field_tuple in f.parse(fmt)] + possible_fields = ('W', 'D', 'H', 'M', 'S') + constants = {'W': 604800, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1} + values = {} + + for field in possible_fields: + if field in desired_fields and field in constants: + values[field], remainder = divmod(remainder, constants[field]) + + return f.format(fmt, **values) + + +def convert_comma_separated_string_to_list(comma_separated_string): + """ + Convert the comma separated string to a valid list. + """ + return list({item.strip() for item in comma_separated_string.split(",") if item.strip()}) + + +def get_image_url(content_metadata_item): + """ + Return the image URI of the content item. + """ + image_url = '' + if content_metadata_item['content_type'] == 'program': + image_url = content_metadata_item.get('card_image_url') + elif content_metadata_item['content_type'] in ['course', 'courserun']: + image_url = content_metadata_item.get('image_url') + + return image_url + + +def is_already_transmitted( + transmission, + enterprise_enrollment_id, + enterprise_configuration_id, + grade, + subsection_id=None, + detect_grade_updated=True, +): + """ + Returns: Boolean indicating if completion date for given enrollment is already sent of not. + + Args: + transmission: TransmissionAudit model to search enrollment in + enterprise_enrollment_id: enrollment id + grade: 'Pass' or 'Fail' status + subsection_id (Optional): The id of the subsection, needed if transmitting assessment level grades as there can + be multiple per course. + detect_grade_updated: default True. if this is False, method does not take into account grade changes + """ + try: + already_transmitted = transmission.objects.filter( + enterprise_course_enrollment_id=enterprise_enrollment_id, + plugin_configuration_id=enterprise_configuration_id, + is_transmitted=True, + ) + if subsection_id: + already_transmitted = already_transmitted.filter(subsection_id=subsection_id) + + latest_transmitted_tx = already_transmitted.latest('id') + if detect_grade_updated: + return latest_transmitted_tx and getattr(latest_transmitted_tx, 'grade', None) == grade + return latest_transmitted_tx is not None + except transmission.DoesNotExist: + return False + + +def get_duration_from_estimated_hours(estimated_hours): + """ + Return the duration in {hours}:{minutes}:00 corresponding to estimated hours as int or float. + """ + if estimated_hours and isinstance(estimated_hours, (int, float)): + fraction, whole_number = math.modf(estimated_hours) + hours = "{:02d}".format(int(whole_number)) + minutes = "{:02d}".format(int(60 * fraction)) + duration = "{hours}:{minutes}:00".format(hours=hours, minutes=minutes) + return duration + + return None + + +def get_courserun_duration_in_hours(course_run): + """ + Return the approximate number of hours required to complete this course run. + """ + min_effort = course_run.get('min_effort') + max_effort = course_run.get('max_effort') + weeks_to_complete = course_run.get('weeks_to_complete') + if min_effort and max_effort and weeks_to_complete: + average_hours_per_week = (min_effort + max_effort) / 2.0 + total_hours = weeks_to_complete * average_hours_per_week + return math.ceil(total_hours) + else: + return 0 + + +def get_subjects_from_content_metadata(content_metadata_item): + """ + Returns a list of subject names for the content metadata item. + + Subjects in the content metadata item are represented by either: + - a list of strings, e.g. ['Communication'] + - a list of objects, e.g. [{'name': 'Communication'}] + + Arguments: + - content_metadata_item (dict): a dictionary for the content metadata item + + Returns: + - list: a list of subject names as strings + """ + metadata_subjects = content_metadata_item.get('subjects') or [] + subjects = set() + + for subject in metadata_subjects: + if isinstance(subject, str): + subjects.add(subject) + continue + + subject_name = subject.get('name') + if subject_name: + subjects.add(subject_name) + + return list(subjects) + + +def generate_formatted_log( + channel_name=None, + enterprise_customer_uuid=None, + lms_user_id=None, + course_or_course_run_key=None, + message=None, + plugin_configuration_id=None, +): + """ + Formats and returns a standardized message for the integrated channels. + All fields are optional. + + Arguments: + - channel_name (str): The name of the integrated channel + - enterprise_customer_uuid (str): UUID of the relevant EnterpriseCustomer + - lms_user_id (str): The LMS User id (if applicable) related to the message + - course_or_course_run_key (str): The course key (if applicable) for the message + - message (str): The string to be formatted and logged + - plugin_configuration_id (str): The configuration id related to the message + + """ + return f'integrated_channel={channel_name}, '\ + f'integrated_channel_enterprise_customer_uuid={enterprise_customer_uuid}, '\ + f'integrated_channel_lms_user={lms_user_id}, '\ + f'integrated_channel_course_key={course_or_course_run_key}, '\ + f'integrated_channel_plugin_configuration_id={plugin_configuration_id}, {message}' + + +def log_exception(enterprise_configuration, msg, course_or_course_run_key=None): + LOGGER.exception( + generate_formatted_log( + channel_name=enterprise_configuration.channel_code(), + enterprise_customer_uuid=enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=course_or_course_run_key, + plugin_configuration_id=enterprise_configuration.id, + message=msg + ) + ) + + +def refresh_session_if_expired( + oauth_access_token_function, + session=None, + expires_at=None, +): + """ + Instantiate a new session object for use in connecting with integrated channel. + Or, return an updated session if provided session has expired. + Suitable for use with oauth supporting servers that use bearer token: Canvas, Blackboard etc. + + Arguments: + - oauth_access_token_function (function): access token fetch function + - session (requests.Session): a session object. Pass None if creating new session + - expires_at: the expiry date of the session if known. None is interpreted as expired. + + Each enterprise customer connecting to channel should have a single client session. + Will only create a new session if token expiry has been reached + If a new session is being created, closes the session first + + Returns (tuple) with values: + - session: newly created session or an updated session (can be stored for later use) + - expires_at: new expiry date to be stored for later use + If session has not expired, or not updated for any reason, just returns the input values of + session and expires_at + """ + now = datetime.utcnow() + if session is None or expires_at is None or now >= expires_at: + # need new session if session expired, or not initialized + if session: + session.close() + # Create a new session with a valid token + oauth_access_token, expires_in = oauth_access_token_function() + new_session = requests.Session() + new_session.headers['Authorization'] = 'Bearer {}'.format(oauth_access_token) + new_session.headers['content-type'] = 'application/json' + # expiry expected after `expires_in` seconds + if expires_in is not None: + new_expires_at = datetime.utcnow() + timedelta(seconds=expires_in) + else: + new_expires_at = None + return new_session, new_expires_at + return session, expires_at + + +def get_upgrade_deadline(course_run): + """ + Returns upgrade_deadline of a verified seat if found. Otherwise returns None. + """ + for seat in course_run.get('seats', []): + if seat.get('type') == 'verified': + return parse_datetime_handle_invalid(seat.get('upgrade_deadline')) + return None + + +def is_course_completed(enterprise_enrollment, is_passing, incomplete_count, passed_timestamp=None): + ''' + For non audit, this requires passing and passed_timestamp + For audit enrollment, returns True if: + - for non upgradable course: + - no more non-gated content is left + - for upgradable course: + - the verified upgrade deadline has passed AND no more non-gated content is left + ''' + if enterprise_enrollment.is_audit_enrollment: + if incomplete_count is None: + raise ValueError('Incomplete count is required if using audit enrollment') + course_run = get_course_run_for_enrollment(enterprise_enrollment) + upgrade_deadline = get_upgrade_deadline(course_run) + if upgrade_deadline is None: + return incomplete_count == 0 + else: + # for upgradable course check deadline passed as well + now = datetime.now(pytz.UTC) + return incomplete_count == 0 and upgrade_deadline < now + return passed_timestamp is not None and is_passing + + +def is_valid_url(url): + """ + Return where the specified URL is a valid absolute url. + """ + if len(url) == 0: + return True + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except ValueError: + return False + + +def batch_by_pk(ModelClass, extra_filter=Q(), batch_size=10000): + """ + yield per batch efficiently + using limit/offset does a lot of table scanning to reach higher offsets + this scanning can be slow on very large tables + if you order by pk, you can use the pk as a pivot rather than offset + this utilizes the index, which is faster than scanning to reach offset + Example usage: + csod_only_filter = Q(integrated_channel_code='CSOD') + for items_batch in batch_by_pk(ContentMetadataItemTransmission, extra_filter=csod_only_filter): + for item in items_batch: + ... + """ + qs = ModelClass.objects.filter(extra_filter).order_by('pk')[:batch_size] + while qs.exists(): + yield qs + # qs.last() doesn't work here because we've already sliced + # loop through so we eventually grab the last one + for item in qs: + start_pk = item.pk + qs = ModelClass.objects.filter(pk__gt=start_pk).filter(extra_filter).order_by('pk')[:batch_size] + + +def truncate_item_dicts(items_to_create, items_to_update, items_to_delete, combined_maximum_size): + """ + given the item collections to create, update, and delete on the remote LMS side, truncate the + collections to keep under a maximum batch size, prioritizing creates, updates, then deletes + """ + # if we have more to work with than the allowed space, slice it up + if len(items_to_create) + len(items_to_delete) + len(items_to_update) > combined_maximum_size: + # prioritize creates, then updates, then deletes + items_to_create = dict(itertools.islice(items_to_create.items(), combined_maximum_size)) + count_left = combined_maximum_size - len(items_to_create) + items_to_update = dict(itertools.islice(items_to_update.items(), count_left)) + count_left = count_left - len(items_to_update) + items_to_delete = dict(itertools.islice(items_to_delete.items(), count_left)) + return items_to_create, items_to_update, items_to_delete + + +def channel_code_to_app_label(channel_code): + """ + Convert an integrated_channel channel_code to app_label. + They can be different, such as in the case of SAP -> sap_success_factors + """ + app_label = channel_code.lower() + if app_label == 'generic': + app_label = 'integrated_channel' + elif app_label == 'sap': + app_label = 'sap_success_factors' + elif app_label == 'csod': + app_label = 'cornerstone' + return app_label + + +def get_enterprise_customer_model(): + """ + Returns the ``EnterpriseCustomer`` class. + """ + return apps.get_model('enterprise', 'EnterpriseCustomer') + + +def integrated_channel_request_log_model(): + """ + Returns the ``IntegratedChannelAPIRequestLogs`` class. + """ + return apps.get_model("integrated_channel", "IntegratedChannelAPIRequestLogs") + + +def get_enterprise_customer_from_enterprise_enrollment(enrollment_id): + """ + Returns the Django ORM enterprise customer object that is associated with an enterprise enrollment ID + """ + EnterpriseCustomer = get_enterprise_customer_model() + try: + ec = EnterpriseCustomer.objects.filter( + enterprise_customer_users__enterprise_enrollments=enrollment_id + ).first() + return ec + except ObjectDoesNotExist: + return None + + +def get_enterprise_client_by_channel_code(channel_code): + """ + Get the appropriate enterprise client based on channel code + """ + # TODO: Other configs + from channel_integrations.canvas.client import CanvasAPIClient # pylint: disable=C0415 + _enterprise_client_model_by_channel_code = { + 'canvas': CanvasAPIClient, + } + return _enterprise_client_model_by_channel_code[channel_code] + + +def stringify_and_store_api_record( + enterprise_customer, + enterprise_customer_configuration_id, + endpoint, + data, + time_taken, + status_code, + response_body, + channel_name +): + """ + Stringify the given data and store the API record in the database. + """ + if data is not None: + # Convert data to string if it's not already a string + if not isinstance(data, str): + try: + # Check if data is a dictionary, list, or tuple then convert to JSON string + if isinstance(data, (dict, list, tuple)): + data = json.dumps(data) + else: + # If it's another type, simply convert to string + data = str(data) + except (TypeError, ValueError) as e: + LOGGER.error( + f"stringify_and_store_api_record: Error occured during stringification: {e}" + f"enterprise_customer={enterprise_customer}" + f"enterprise_customer_configuration_id={enterprise_customer_configuration_id}" + f"channel name={channel_name}" + f"data={data}" + ) + # Store stringified data in the database + try: + integrated_channel_request_log_model().store_api_call( + enterprise_customer=enterprise_customer, + enterprise_customer_configuration_id=enterprise_customer_configuration_id, + endpoint=endpoint, + payload=data, + time_taken=time_taken, + status_code=status_code, + response_body=response_body, + channel_name=channel_name + ) + except Exception as e: # pylint: disable=broad-except + LOGGER.error( + f"stringify_and_store_api_record: Error occured while storing: {e}" + f"enterprise_customer={enterprise_customer}" + f"enterprise_customer_configuration_id={enterprise_customer_configuration_id}" + f"channel name={channel_name}" + f"data={data}" + ) + return data From 2e91618fb209699172a30fa88b3e76d99f42676f Mon Sep 17 00:00:00 2001 From: Muhammad Sameer Amin <35958006+sameeramin@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:25:58 +0500 Subject: [PATCH 2/2] feat: updated apps config and added migrations --- channel_integrations/apps.py | 13 -- .../integrated_channel/apps.py | 1 + .../migrations/0001_initial.py | 151 ++++++++++++++++++ .../integrated_channel/models.py | 17 +- channel_integrations/models.py | 3 - .../enterprise_integrated_channels/base.html | 26 --- default.db | 0 setup.py | 4 +- 8 files changed, 166 insertions(+), 49 deletions(-) delete mode 100644 channel_integrations/apps.py create mode 100644 channel_integrations/integrated_channel/migrations/0001_initial.py delete mode 100644 channel_integrations/models.py delete mode 100644 channel_integrations/templates/enterprise_integrated_channels/base.html create mode 100644 default.db diff --git a/channel_integrations/apps.py b/channel_integrations/apps.py deleted file mode 100644 index 8d8d172..0000000 --- a/channel_integrations/apps.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -channel_integrations Django application initialization. -""" - -from django.apps import AppConfig - - -class ChannelIntegrationsConfig(AppConfig): - """ - Configuration for the channel_integrations Django application. - """ - - name = 'channel_integrations' diff --git a/channel_integrations/integrated_channel/apps.py b/channel_integrations/integrated_channel/apps.py index b3b4646..e80752f 100644 --- a/channel_integrations/integrated_channel/apps.py +++ b/channel_integrations/integrated_channel/apps.py @@ -11,3 +11,4 @@ class IntegratedChannelConfig(AppConfig): """ name = 'channel_integrations.integrated_channel' verbose_name = "Enterprise Integrated Channels" + label = 'channel_integration' diff --git a/channel_integrations/integrated_channel/migrations/0001_initial.py b/channel_integrations/integrated_channel/migrations/0001_initial.py new file mode 100644 index 0000000..a981f8d --- /dev/null +++ b/channel_integrations/integrated_channel/migrations/0001_initial.py @@ -0,0 +1,151 @@ +# Generated by Django 4.2.17 on 2024-12-27 06:17 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import jsonfield.fields +import model_utils.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('enterprise', '0228_alter_defaultenterpriseenrollmentrealization_realized_enrollment'), + ] + + operations = [ + migrations.CreateModel( + name='ApiResponseRecord', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('status_code', models.PositiveIntegerField(blank=True, help_text='The most recent remote API call response HTTP status code', null=True)), + ('body', models.TextField(blank=True, help_text='The most recent remote API call response body', null=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ContentMetadataItemTransmission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('integrated_channel_code', models.CharField(max_length=30)), + ('plugin_configuration_id', models.PositiveIntegerField(blank=True, null=True)), + ('content_id', models.CharField(max_length=255)), + ('content_title', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('channel_metadata', jsonfield.fields.JSONField()), + ('content_last_changed', models.DateTimeField(blank=True, help_text='Date of the last time the enterprise catalog associated with this metadata item was updated', null=True)), + ('enterprise_customer_catalog_uuid', models.UUIDField(blank=True, help_text='The enterprise catalog that this metadata item was derived from', null=True)), + ('remote_deleted_at', models.DateTimeField(blank=True, help_text='Date when the content transmission was deleted in the remote API', null=True)), + ('remote_created_at', models.DateTimeField(blank=True, help_text='Date when the content transmission was created in the remote API', null=True)), + ('remote_errored_at', models.DateTimeField(blank=True, help_text='Date when the content transmission was failed in the remote API.', null=True)), + ('remote_updated_at', models.DateTimeField(blank=True, help_text='Date when the content transmission was last updated in the remote API', null=True)), + ('api_response_status_code', models.PositiveIntegerField(blank=True, help_text='The most recent remote API call response HTTP status code', null=True)), + ('friendly_status_message', models.CharField(blank=True, default=None, help_text='A user-friendly API response status message.', max_length=255, null=True)), + ('marked_for', models.CharField(blank=True, help_text='Flag marking a record as needing a form of transmitting', max_length=32, null=True)), + ('api_record', models.OneToOneField(blank=True, help_text='Data pertaining to the transmissions API request response.', null=True, on_delete=django.db.models.deletion.CASCADE, to='channel_integration.apiresponserecord')), + ('enterprise_customer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='channelintegration_contentmetadataitemtransmission', to='enterprise.enterprisecustomer')), + ], + options={ + 'unique_together': {('integrated_channel_code', 'plugin_configuration_id', 'content_id')}, + 'index_together': {('enterprise_customer', 'integrated_channel_code', 'plugin_configuration_id', 'content_id')}, + }, + ), + migrations.CreateModel( + name='IntegratedChannelAPIRequestLogs', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('enterprise_customer_configuration_id', models.IntegerField(help_text='ID from the EnterpriseCustomerConfiguration model')), + ('endpoint', models.URLField(max_length=255)), + ('payload', models.TextField()), + ('time_taken', models.FloatField()), + ('status_code', models.PositiveIntegerField(blank=True, help_text='API call response HTTP status code', null=True)), + ('response_body', models.TextField(blank=True, help_text='API call response body', null=True)), + ('channel_name', models.TextField(blank=True, help_text='Name of the integrated channel associated with this API call log record.')), + ('enterprise_customer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='channelintegration_integratedchannelapirequestlogs', to='enterprise.enterprisecustomer')), + ], + options={ + 'verbose_name_plural': 'Integrated channels API request logs', + }, + ), + migrations.CreateModel( + name='GenericLearnerDataTransmissionAudit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('enterprise_customer_uuid', models.UUIDField(blank=True, null=True)), + ('user_email', models.CharField(blank=True, max_length=255, null=True)), + ('plugin_configuration_id', models.IntegerField(blank=True, null=True)), + ('enterprise_course_enrollment_id', models.IntegerField(blank=True, db_index=True, null=True)), + ('course_id', models.CharField(max_length=255)), + ('content_title', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('course_completed', models.BooleanField(default=True)), + ('progress_status', models.CharField(blank=True, max_length=255)), + ('completed_timestamp', models.DateTimeField(blank=True, null=True)), + ('instructor_name', models.CharField(blank=True, max_length=255)), + ('grade', models.FloatField(blank=True, null=True)), + ('total_hours', models.FloatField(blank=True, null=True)), + ('subsection_id', models.CharField(blank=True, db_index=True, max_length=255, null=True)), + ('subsection_name', models.CharField(max_length=255, null=True)), + ('status', models.CharField(blank=True, max_length=100, null=True)), + ('error_message', models.TextField(blank=True, null=True)), + ('is_transmitted', models.BooleanField(default=False)), + ('friendly_status_message', models.CharField(blank=True, default=None, help_text='A user-friendly API response status message.', max_length=255, null=True)), + ('api_record', models.OneToOneField(blank=True, help_text='Data pertaining to the transmissions API request response.', null=True, on_delete=django.db.models.deletion.CASCADE, to='channel_integration.apiresponserecord')), + ], + ), + migrations.CreateModel( + name='GenericEnterpriseCustomerPluginConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ('display_name', models.CharField(blank=True, default='', help_text='A configuration nickname.', max_length=255)), + ('idp_id', models.CharField(blank=True, default='', help_text='If provided, will be used as IDP slug to locate remote id for learners', max_length=255)), + ('active', models.BooleanField(help_text='Is this configuration active?')), + ('dry_run_mode_enabled', models.BooleanField(default=False, help_text='Is this configuration in dry-run mode? (experimental)')), + ('show_course_price', models.BooleanField(default=False, help_text='Displays course price')), + ('transmission_chunk_size', models.IntegerField(default=500, help_text='The maximum number of data items to transmit to the integrated channel with each request.')), + ('channel_worker_username', models.CharField(blank=True, default='', help_text='Enterprise channel worker username to get JWT tokens for authenticating LMS APIs.', max_length=255)), + ('catalogs_to_transmit', models.TextField(blank=True, default='', help_text='A comma-separated list of catalog UUIDs to transmit. If blank, all customer catalogs will be transmitted. If there are overlapping courses in the customer catalogs, the overlapping course metadata will be selected from the newest catalog.')), + ('disable_learner_data_transmissions', models.BooleanField(default=False, help_text='When set to True, the configured customer will no longer receive learner data transmissions, both scheduled and signal based', verbose_name='Disable Learner Data Transmission')), + ('last_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Content or Learner data record sync attempt', null=True)), + ('last_content_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Content data record sync attempt', null=True)), + ('last_learner_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Learner data record sync attempt', null=True)), + ('last_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Content or Learner data record sync attempt', null=True)), + ('last_content_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Content data record sync attempt', null=True)), + ('last_learner_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Learner data record sync attempt', null=True)), + ('last_modified_at', models.DateTimeField(auto_now=True, help_text='The DateTime of the last change made to this configuration.', null=True)), + ('enterprise_customer', models.ForeignKey(help_text='Enterprise Customer associated with the configuration.', on_delete=django.db.models.deletion.CASCADE, related_name='channelintegration_enterprisecustomerpluginconfiguration', to='enterprise.enterprisecustomer')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='OrphanedContentTransmissions', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('integrated_channel_code', models.CharField(max_length=30)), + ('plugin_configuration_id', models.PositiveIntegerField()), + ('content_id', models.CharField(max_length=255)), + ('resolved', models.BooleanField(default=False)), + ('transmission', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='orphaned_record', to='channel_integration.contentmetadataitemtransmission')), + ], + options={ + 'index_together': {('integrated_channel_code', 'plugin_configuration_id', 'resolved')}, + }, + ), + ] diff --git a/channel_integrations/integrated_channel/models.py b/channel_integrations/integrated_channel/models.py index f18cc0f..b0e4d08 100644 --- a/channel_integrations/integrated_channel/models.py +++ b/channel_integrations/integrated_channel/models.py @@ -117,6 +117,7 @@ class EnterpriseCustomerPluginConfiguration(SoftDeletionModel): enterprise_customer = models.ForeignKey( EnterpriseCustomer, + related_name='channelintegration_enterprisecustomerpluginconfiguration', blank=False, null=False, help_text=_("Enterprise Customer associated with the configuration."), @@ -537,7 +538,7 @@ class LearnerDataTransmissionAudit(TimeStampedModel): class Meta: abstract = True - app_label = 'integrated_channel' + app_label = 'channel_integration' def __str__(self): """ @@ -613,7 +614,7 @@ class GenericLearnerDataTransmissionAudit(LearnerDataTransmissionAudit): A generic implementation of LearnerDataTransmissionAudit which can be instantiated """ class Meta: - app_label = 'integrated_channel' + app_label = 'channel_integration' def __str__(self): """ @@ -649,7 +650,11 @@ class Meta: index_together = [('enterprise_customer', 'integrated_channel_code', 'plugin_configuration_id', 'content_id')] unique_together = (('integrated_channel_code', 'plugin_configuration_id', 'content_id'),) - enterprise_customer = models.ForeignKey(EnterpriseCustomer, on_delete=models.CASCADE) + enterprise_customer = models.ForeignKey( + EnterpriseCustomer, + related_name='channelintegration_contentmetadataitemtransmission', + on_delete=models.CASCADE + ) integrated_channel_code = models.CharField(max_length=30) plugin_configuration_id = models.PositiveIntegerField(blank=True, null=True) content_id = models.CharField(max_length=255) @@ -912,7 +917,9 @@ class IntegratedChannelAPIRequestLogs(TimeStampedModel): """ enterprise_customer = models.ForeignKey( - EnterpriseCustomer, on_delete=models.CASCADE + EnterpriseCustomer, + related_name='channelintegration_integratedchannelapirequestlogs', + on_delete=models.CASCADE ) enterprise_customer_configuration_id = models.IntegerField( blank=False, @@ -938,7 +945,7 @@ class IntegratedChannelAPIRequestLogs(TimeStampedModel): ) class Meta: - app_label = "integrated_channel" + app_label = 'channel_integration' verbose_name_plural = "Integrated channels API request logs" def __str__(self): diff --git a/channel_integrations/models.py b/channel_integrations/models.py deleted file mode 100644 index a74527c..0000000 --- a/channel_integrations/models.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Database models for channel_integrations. -""" diff --git a/channel_integrations/templates/enterprise_integrated_channels/base.html b/channel_integrations/templates/enterprise_integrated_channels/base.html deleted file mode 100644 index daf311a..0000000 --- a/channel_integrations/templates/enterprise_integrated_channels/base.html +++ /dev/null @@ -1,26 +0,0 @@ - - -{% load i18n %} -{% trans "Dummy text to generate a translation (.po) source file. It is safe to delete this line. It is also safe to delete (load i18n) above if there are no other (trans) tags in the file" %} - -{% comment %} -As the developer of this package, don't place anything here if you can help it -since this allows developers to have interoperability between your template -structure and their own. - -Example: Developer melding the 2SoD pattern to fit inside with another pattern:: - - {% extends "base.html" %} - {% load static %} - - - {% block extra_js %} - - - {% block javascript %} - - {% endblock javascript %} - - {% endblock extra_js %} -{% endcomment %} - diff --git a/default.db b/default.db new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 1574e64..97910a7 100755 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ def is_requirement(line): include_package_data=True, install_requires=load_requirements('requirements/base.in'), - python_requires=">=3.12", + python_requires=">=3.11", license="AGPL 3.0", zip_safe=False, keywords='Python edx', @@ -155,6 +155,6 @@ def is_requirement(line): 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.11', ], )