From 3dd77857caa74b986c92ae47b81cd8432009a131 Mon Sep 17 00:00:00 2001 From: Alec Li Date: Sat, 23 Dec 2023 03:35:35 -0800 Subject: [PATCH 1/2] Initial implementation of export page Add initial query implementations Remove old data export modal --- csm_web/frontend/src/components/App.tsx | 2 + csm_web/frontend/src/components/AutoGrid.tsx | 48 ++ csm_web/frontend/src/components/Home.tsx | 26 +- .../frontend/src/components/course/Course.tsx | 15 +- .../src/components/course/DataExportModal.tsx | 112 ---- .../src/components/data_export/DataExport.tsx | 22 + .../data_export/DataExportTypes.tsx | 83 +++ .../src/components/data_export/ExportPage.tsx | 236 +++++++ .../components/data_export/ExportSelector.tsx | 52 ++ csm_web/frontend/src/css/base/autogrid.scss | 5 + csm_web/frontend/src/css/data-export.scss | 175 ++++++ csm_web/frontend/src/css/home.scss | 16 +- csm_web/frontend/src/utils/api.tsx | 38 +- csm_web/frontend/src/utils/queries/export.tsx | 87 +++ .../static/frontend/img/file-export.svg | 1 + .../frontend/static/frontend/img/refresh.svg | 1 + csm_web/scheduler/urls.py | 1 + csm_web/scheduler/views/__init__.py | 3 +- csm_web/scheduler/views/export.py | 590 ++++++++++++++++++ package-lock.json | 13 + package.json | 1 + 21 files changed, 1386 insertions(+), 141 deletions(-) create mode 100644 csm_web/frontend/src/components/AutoGrid.tsx delete mode 100644 csm_web/frontend/src/components/course/DataExportModal.tsx create mode 100644 csm_web/frontend/src/components/data_export/DataExport.tsx create mode 100644 csm_web/frontend/src/components/data_export/DataExportTypes.tsx create mode 100644 csm_web/frontend/src/components/data_export/ExportPage.tsx create mode 100644 csm_web/frontend/src/components/data_export/ExportSelector.tsx create mode 100644 csm_web/frontend/src/css/base/autogrid.scss create mode 100644 csm_web/frontend/src/css/data-export.scss create mode 100644 csm_web/frontend/src/utils/queries/export.tsx create mode 100644 csm_web/frontend/static/frontend/img/file-export.svg create mode 100644 csm_web/frontend/static/frontend/img/refresh.svg create mode 100644 csm_web/scheduler/views/export.py diff --git a/csm_web/frontend/src/components/App.tsx b/csm_web/frontend/src/components/App.tsx index 6799ce7c..1c321f7a 100644 --- a/csm_web/frontend/src/components/App.tsx +++ b/csm_web/frontend/src/components/App.tsx @@ -8,6 +8,7 @@ import { emptyRoles, Roles } from "../utils/user"; import CourseMenu from "./CourseMenu"; import Home from "./Home"; import Policies from "./Policies"; +import { DataExport } from "./data_export/DataExport"; import { EnrollmentMatcher } from "./enrollment_automation/EnrollmentMatcher"; import { Resources } from "./resource_aggregation/Resources"; import Section from "./section/Section"; @@ -39,6 +40,7 @@ const App = () => { } /> } /> } /> + } /> } /> diff --git a/csm_web/frontend/src/components/AutoGrid.tsx b/csm_web/frontend/src/components/AutoGrid.tsx new file mode 100644 index 00000000..43f0bfb1 --- /dev/null +++ b/csm_web/frontend/src/components/AutoGrid.tsx @@ -0,0 +1,48 @@ +import React from "react"; + +import "../css/base/autogrid.scss"; + +interface AutoColumnsProps { + children?: React.ReactNode[]; +} + +/** + * Automatically format children in balanced columns. + */ +export const AutoGrid = ({ children }: AutoColumnsProps) => { + const gridSize = Math.ceil(Math.sqrt(children?.length ?? 0)); + + if (children == null) { + return null; + } + + const raw_table: React.ReactNode[][] = []; + children.forEach((item, idx) => { + if (idx % gridSize == 0) { + raw_table.push([item]); + } else { + raw_table[raw_table.length - 1].push(item); + } + }); + + // transpose table + const table = raw_table[0].map((_, colIndex) => raw_table.map(row => row[colIndex])); + + return ( +
+ + + {table.map((row, rowIdx) => ( + + {row.map((item, itemIdx) => ( + + ))} + + ))} + +
+ {item} +
+
+ ); +}; diff --git a/csm_web/frontend/src/components/Home.tsx b/csm_web/frontend/src/components/Home.tsx index b8f0298f..da7873e0 100644 --- a/csm_web/frontend/src/components/Home.tsx +++ b/csm_web/frontend/src/components/Home.tsx @@ -8,6 +8,7 @@ import { useCourses } from "../utils/queries/courses"; import { Profile, Course, Role } from "../utils/types"; import LoadingSpinner from "./LoadingSpinner"; +import FileExport from "../../static/frontend/img/file-export.svg"; import PlusIcon from "../../static/frontend/img/plus.svg"; import scssColors from "../css/base/colors-export.module.scss"; @@ -17,6 +18,7 @@ const Home = () => { const { data: courses, isSuccess: coursesLoaded, isError: coursesLoadError } = useCourses(); let content = null; + let headingRight = null; if (profilesLoaded && coursesLoaded) { // loaded, no error const coursesById: Map = new Map(); @@ -52,6 +54,17 @@ const Home = () => { })} ); + + const isCoordinator = profiles!.some(profile => profile.role === Role.COORDINATOR); + + if (isCoordinator) { + headingRight = ( + + + Export + + ); + } } else if (profilesLoadError) { // error during load content =

Profiles not found

; @@ -66,11 +79,14 @@ const Home = () => { return (
-

My courses

- - - Add Course - +
+

My courses

+ + + Add Course + +
+
{headingRight}
{content}
diff --git a/csm_web/frontend/src/components/course/Course.tsx b/csm_web/frontend/src/components/course/Course.tsx index a46a3f14..e1d20cbc 100644 --- a/csm_web/frontend/src/components/course/Course.tsx +++ b/csm_web/frontend/src/components/course/Course.tsx @@ -6,7 +6,6 @@ import { useCourseSections } from "../../utils/queries/courses"; import { Course as CourseType } from "../../utils/types"; import LoadingSpinner from "../LoadingSpinner"; import { CreateSectionModal } from "./CreateSectionModal"; -import { DataExportModal } from "./DataExportModal"; import { SectionCard } from "./SectionCard"; import { SettingsModal } from "./SettingsModal"; import { WhitelistModal } from "./WhitelistModal"; @@ -27,7 +26,6 @@ const DAY_OF_WEEK_ABREVIATIONS: { [day: string]: string } = Object.freeze({ }); const COURSE_MODAL_TYPE = Object.freeze({ - exportData: "csv", createSection: "mksec", whitelist: "whitelist", settings: "settings" @@ -83,9 +81,7 @@ const Course = ({ courses, priorityEnrollment, enrollmentTimes }: CourseProps): * Render the currently chosen modal. */ const renderModal = (): React.ReactElement | null => { - if (whichModal == COURSE_MODAL_TYPE.exportData) { - return setShowModal(false)} />; - } else if (whichModal == COURSE_MODAL_TYPE.createSection) { + if (whichModal == COURSE_MODAL_TYPE.createSection) { return ( Create Section - - - - - ); -}; diff --git a/csm_web/frontend/src/components/data_export/DataExport.tsx b/csm_web/frontend/src/components/data_export/DataExport.tsx new file mode 100644 index 00000000..0e4a3d25 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/DataExport.tsx @@ -0,0 +1,22 @@ +import React, { useState } from "react"; +import { ExportType } from "./DataExportTypes"; +import { ExportPage } from "./ExportPage"; +import { ExportSelector } from "./ExportSelector"; + +export const DataExport = () => { + const [dataExportType, setDataExportType] = useState(null); + + return ( +
+ {dataExportType === null ? ( + { + setDataExportType(exportType); + }} + /> + ) : ( + setDataExportType(null)} /> + )} +
+ ); +}; diff --git a/csm_web/frontend/src/components/data_export/DataExportTypes.tsx b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx new file mode 100644 index 00000000..2562ef74 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx @@ -0,0 +1,83 @@ +/** + * Enum for all the various data export types + */ +export enum ExportType { + STUDENT_DATA = "STUDENT_DATA", + ATTENDANCE_DATA = "ATTENDANCE_DATA", + SECTION_DATA = "SECTION_DATA", + COURSE_DATA = "COURSE_DATA" +} + +/** + * Object for displaying export types in the UI + */ +export const EXPORT_TYPE_DATA = new Map([ + [ExportType.STUDENT_DATA, "Student data"], + [ExportType.ATTENDANCE_DATA, "Attendance data"], + [ExportType.SECTION_DATA, "Section data"], + [ExportType.COURSE_DATA, "Course data"] +]); + +export const EXPORT_COLUMNS: { + [exportType in ExportType]: { + required: { [key: string]: string }; + optional: { [key: string]: string }; + }; +} = { + [ExportType.ATTENDANCE_DATA]: { + required: { + student_email: "Student email", + student_name: "Student name" + }, + optional: { + course_name: "Course name", + active: "Active", + section_id: "Section ID", + mentor_name: "Mentor name", + mentor_email: "Mentor email" + } + }, + [ExportType.COURSE_DATA]: { + required: { + course_name: "Course name" + }, + optional: { + course_id: "Course ID", + description: "Course description", + num_sections: "Section count", + num_students: "Student count", + num_mentors: "Mentor count" + } + }, + [ExportType.SECTION_DATA]: { + required: { + mentor_name: "Mentor name", + mentor_email: "Mentor email" + }, + optional: { + course_name: "Course name", + section_id: "Section ID", + section_times: "Section times", + section_description: "Section description", + num_students: "Student count", + capacity: "Capacity" + } + }, + [ExportType.STUDENT_DATA]: { + required: { + student_email: "Student email", + student_name: "Student name" + }, + optional: { + course_name: "Course name", + active: "Active", + mentor_name: "Mentor name", + mentor_email: "Mentor email", + section_id: "Section ID", + section_times: "Section times", + num_present: "Present attendance count", + num_excused: "Excused absence count", + num_unexcused: "Unexcused absence count" + } + } +}; diff --git a/csm_web/frontend/src/components/data_export/ExportPage.tsx b/csm_web/frontend/src/components/data_export/ExportPage.tsx new file mode 100644 index 00000000..f1c14e49 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/ExportPage.tsx @@ -0,0 +1,236 @@ +import React, { useEffect, useState } from "react"; +import { useProfiles } from "../../utils/queries/base"; +import { useDataExportMutation, useDataExportPreviewMutation } from "../../utils/queries/export"; +import { Role } from "../../utils/types"; +import { AutoGrid } from "../AutoGrid"; +import LoadingSpinner from "../LoadingSpinner"; +import { ExportType, EXPORT_COLUMNS } from "./DataExportTypes"; + +import RefreshIcon from "../../../static/frontend/img/refresh.svg"; + +interface ExportPageProps { + dataExportType: ExportType; + onBack: () => void; +} + +export const ExportPage = ({ dataExportType, onBack }: ExportPageProps) => { + const { data: profiles, isSuccess: profilesLoaded, isError: profilesError } = useProfiles(); + const [includedCourses, setIncludedCourses] = useState([]); + const [includedFields, setIncludedFields] = useState( + Array.from(Object.keys(EXPORT_COLUMNS[dataExportType].optional)) + ); + + const dataExportMutation = useDataExportMutation(); + + useEffect(() => { + if (profiles != null && profilesLoaded) { + const coordinatorProfiles = profiles.filter(profile => profile.role === Role.COORDINATOR); + setIncludedCourses(coordinatorProfiles.map(profile => profile.courseId)); + } + }, [profilesLoaded, profiles]); + + if (profilesError) { + return

Profiles not found

; + } else if (!profilesLoaded) { + return ; + } + + const coordinatorProfiles = profiles.filter(profile => profile.role === Role.COORDINATOR); + + const courseSelection = ( +
+

Select Courses

+
+ + {coordinatorProfiles + .sort((profileA, profileB) => profileA.course.localeCompare(profileB.course)) + .map(profile => ( + + ))} + +
+
+ ); + + const columnFields = EXPORT_COLUMNS[dataExportType]; + const requiredInputs = Object.entries(columnFields.required).map(([key, description]) => ({ + key, + description, + disabled: true + })); + const optionalInputs = Object.entries(columnFields.optional).map(([key, description]) => ({ + key, + description, + disabled: false + })); + const columnInputs = requiredInputs.concat(optionalInputs).map(({ key, description, disabled }) => ( + + )); + + const columnSelection = ( +
+

Select Fields

+
+ {columnInputs} +
+
+ ); + + /** + * Download the data; open a new page with the data + */ + const downloadData = () => { + dataExportMutation.mutate({ + courses: includedCourses, + fields: includedFields, + type: dataExportType + }); + }; + + return ( +
+
+

Export Data

+ +
+
+
{courseSelection}
+
{columnSelection}
+
+ +
+ +
+
+ ); +}; + +const PREVIEW_OPTIONS = [5, 10, 25, 50]; + +interface ExportPagePreviewProps { + courses: number[]; + fields: string[]; + exportType: ExportType; + preview: number; +} + +/** + * Preview of the exported data + */ +const ExportPagePreview = ({ exportType, courses, fields }: ExportPagePreviewProps) => { + const dataExportPreviewMutation = useDataExportPreviewMutation(); + const [preview, setPreview] = useState(10); + const [data, setData] = useState([]); + + const refreshPreview = () => { + if (courses.length == 0) { + return; + } + + dataExportPreviewMutation.mutate( + { + courses: courses, + fields: fields, + type: exportType, + preview: preview + }, + { + onSuccess: dataPreview => { + setData(dataPreview); + } + } + ); + }; + + const handlePreviewSelect = (e: React.ChangeEvent) => { + const selectedValue = parseInt(e.target.value); + setPreview(selectedValue); + }; + + return ( +
+

Preview

+
+ Rows: + + +
+
+ + + + {data?.length > 0 && + data[0].map((cell, cellIdx) => ( + + ))} + + + + {data.map( + (row, rowIdx) => + rowIdx > 0 && ( + + {row.map((cell, cellIdx) => ( + + ))} + + ) + )} + {data?.length >= preview && ( + + + + )} + +
+ {cell} +
+ {cell} +
+ (More rows clipped) +
+
+
+ ); +}; diff --git a/csm_web/frontend/src/components/data_export/ExportSelector.tsx b/csm_web/frontend/src/components/data_export/ExportSelector.tsx new file mode 100644 index 00000000..56cd5c6a --- /dev/null +++ b/csm_web/frontend/src/components/data_export/ExportSelector.tsx @@ -0,0 +1,52 @@ +import React, { useState } from "react"; + +import { ExportType, EXPORT_TYPE_DATA } from "./DataExportTypes"; + +import "../../css/data-export.scss"; + +interface ExportSelectorProps { + onContinue: (exportType: ExportType) => void; +} + +/** + * Component for selecting the courses to include in the export, + * along with the export data config selection. + */ +export const ExportSelector = ({ onContinue }: ExportSelectorProps) => { + const [dataExportType, setDataExportType] = useState(ExportType.ATTENDANCE_DATA); + + const handleContinue = () => { + onContinue(dataExportType); + }; + + return ( +
+
+

Select Export Data

+
+
+ {Array.from(EXPORT_TYPE_DATA.entries()) + .sort() + .map(([exportType, description]) => ( + + ))} +
+
+
+
+
+ Continue +
+
+
+ ); +}; diff --git a/csm_web/frontend/src/css/base/autogrid.scss b/csm_web/frontend/src/css/base/autogrid.scss new file mode 100644 index 00000000..910ccd6e --- /dev/null +++ b/csm_web/frontend/src/css/base/autogrid.scss @@ -0,0 +1,5 @@ +@use "variables" as *; + +.auto-grid-item { + padding: 0 8px; +} diff --git a/csm_web/frontend/src/css/data-export.scss b/csm_web/frontend/src/css/data-export.scss new file mode 100644 index 00000000..d457fdf5 --- /dev/null +++ b/csm_web/frontend/src/css/data-export.scss @@ -0,0 +1,175 @@ +/* Data export styles */ +@use "base/variables" as *; + +.data-export-container { + display: flex; + flex-direction: column; + gap: 40px; + align-items: stretch; + justify-content: center; +} + +.export-selector-container { + display: flex; + flex-direction: column; + + gap: 50px; +} + +.export-selector-footer { + display: flex; + align-items: center; + justify-content: center; +} + +.export-selector-section, +.export-page-section { + display: flex; + flex-direction: column; + justify-content: center; +} + +.export-selector-data-type-container { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + padding-top: 15px; +} + +.export-page-sidebar-container { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; +} + +.export-page-sidebar-title { + margin-bottom: 8px; +} + +.export-selector-data-type-options { + display: flex; + flex-direction: column; + gap: 8px; + align-items: flex-start; + justify-content: center; + + width: fit-content; +} + +.export-page-courses-options { + width: fit-content; + max-height: 25%; +} + +.export-selector-data-type-label, +.export-page-input-label { + display: block; + width: 100%; + white-space: nowrap; + user-select: none; +} + +.export-page-container { + display: flex; + flex-direction: column; + gap: 16px; +} + +.export-page-config { + display: flex; + flex-flow: row wrap; + gap: 8px; +} + +.export-page-sidebar { + min-width: 650px; + padding: 16px 0; + background-color: #f3f3f3; + border-radius: 12px; +} + +.export-page-sidebar.sidebar-left { + flex: 1; +} + +.export-page-sidebar.sidebar-right { + flex: 2; +} + +.export-page-header { + display: flex; + gap: 16px; + align-items: center; + justify-content: flex-start; +} + +.export-page-footer { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-start; +} + +.export-page-preview { + flex: 2; +} + +.export-page-preview-container { + display: flex; + flex-direction: column; + gap: 4px; +} + +.export-preview-header { + display: flex; + flex-direction: row; + gap: 8px; + align-items: center; + + margin-bottom: 8px; +} + +.export-preview-select { + width: 75px !important; +} + +.export-preview-icon { + color: black; + cursor: pointer; +} + +.export-preview-icon:hover { + color: #222; +} + +.export-preview-table-container { + max-width: 90vw; + overflow-x: auto; +} + +.export-preview-table { + border-collapse: collapse; +} + +.export-preview-table-item, +.export-preview-table-header-item { + padding: 4px; + + white-space: nowrap; + border: 1px #aaa solid; +} + +.export-preview-table-header-item { + text-align: left; +} + +.export-preview-table-more-row-item { + padding: 8px 0; + color: #888; + + column-span: all; + text-align: center; +} diff --git a/csm_web/frontend/src/css/home.scss b/csm_web/frontend/src/css/home.scss index 6fd58c0d..2b515ee8 100644 --- a/csm_web/frontend/src/css/home.scss +++ b/csm_web/frontend/src/css/home.scss @@ -70,11 +70,23 @@ flex-wrap: wrap; align-items: center; justify-content: space-between; - width: 50%; - max-width: 300px; margin-bottom: 40px; } +#home-courses-heading-left { + display: flex; + flex: 1; + flex-direction: row; + gap: 20px; + align-items: center; + justify-content: flex-start; +} + +#home-courses-heading-right { + display: flex; + flex-direction: row; +} + .section-link { padding: 10px; font-size: 14px; diff --git a/csm_web/frontend/src/utils/api.tsx b/csm_web/frontend/src/utils/api.tsx index e7530611..2e8fbbdc 100644 --- a/csm_web/frontend/src/utils/api.tsx +++ b/csm_web/frontend/src/utils/api.tsx @@ -21,13 +21,21 @@ export function normalizeEndpoint(endpoint: string) { return `/api/${endpoint}`; } -export function fetchWithMethod(endpoint: string, method: string, data: any = {}, isFormData = false) { +export function fetchWithMethod( + endpoint: string, + method: string, + data: any = {}, + isFormData = false, + queryParams: URLSearchParams | null = null +) { if (!Object.prototype.hasOwnProperty.call(HTTP_METHODS, method)) { // check that method choice is valid throw new Error("HTTP method must be one of: POST, GET, PUT, PATCH, or DELETE"); } + const normalizedEndpoint = endpointWithQueryParams(normalizeEndpoint(endpoint), queryParams); + if (isFormData) { - return fetch(normalizeEndpoint(endpoint), { + return fetch(normalizedEndpoint, { method: method, credentials: "same-origin", headers: { @@ -36,7 +44,7 @@ export function fetchWithMethod(endpoint: string, method: string, data: any = {} body: data }); } - return fetch(normalizeEndpoint(endpoint), { + return fetch(normalizedEndpoint, { method: method, credentials: "same-origin", headers: { @@ -51,11 +59,27 @@ export function fetchWithMethod(endpoint: string, method: string, data: any = {} /** * Fetch data from normalized endpoint. */ -export async function fetchNormalized(endpoint: string) { - return await fetch(normalizeEndpoint(endpoint), { credentials: "same-origin" }); +export async function fetchNormalized(endpoint: string, queryParams: URLSearchParams | null = null) { + const normalizedEndpoint = normalizeEndpoint(endpoint); + return await fetch(endpointWithQueryParams(normalizedEndpoint, queryParams), { credentials: "same-origin" }); } -export async function fetchJSON(endpoint: string) { - const response = await fetch(normalizeEndpoint(endpoint), { credentials: "same-origin" }); +export async function fetchJSON(endpoint: string, queryParams: URLSearchParams | null = null) { + const normalizedEndpoint = normalizeEndpoint(endpoint); + const response = await fetch(endpointWithQueryParams(normalizedEndpoint, queryParams), { + credentials: "same-origin" + }); return await response.json(); } + +/** + * Add query parameters to the endpoint, if necessary. + * If no query parameters, then the endpoint is returned unchanged. + */ +export function endpointWithQueryParams(endpoint: string, queryParams: URLSearchParams | null = null) { + if (queryParams !== null) { + return `${endpoint}?${queryParams}`; + } else { + return endpoint; + } +} diff --git a/csm_web/frontend/src/utils/queries/export.tsx b/csm_web/frontend/src/utils/queries/export.tsx new file mode 100644 index 00000000..a57e01ff --- /dev/null +++ b/csm_web/frontend/src/utils/queries/export.tsx @@ -0,0 +1,87 @@ +import { useMutation, UseMutationResult } from "@tanstack/react-query"; +import { parse as csv_parse } from "csv-parse/browser/esm/sync"; + +import { ExportType } from "../../components/data_export/DataExportTypes"; +import { endpointWithQueryParams, fetchNormalized, normalizeEndpoint } from "../api"; +import { handleError, handlePermissionsError, handleRetry, ServerError } from "./helpers"; + +interface DataExportPreviewMutationRequest { + courses: number[]; + fields: string[]; + type: ExportType; + preview: number; +} + +/** + * Mutation for fetching export data for preview. + * Returns a table containing the CSV contents. + */ +export const useDataExportPreviewMutation = (): UseMutationResult< + string[][], + ServerError, + DataExportPreviewMutationRequest +> => { + const mutationResult = useMutation( + async (body: DataExportPreviewMutationRequest) => { + const response = await fetchNormalized( + "/export", + new URLSearchParams({ + courses: body.courses.join(","), + fields: body.fields.join(","), + type: body.type, + preview: body.preview.toString() + }) + ); + + if (response.ok) { + const content = await response.text(); + // format content into a table + const table = csv_parse(content); + return table; + } else { + handlePermissionsError(response.status); + throw new ServerError( + `Failed to fetch preview; type ${body.type}, courses ${body.courses}, fields ${body.fields}` + ); + } + }, + { + retry: handleRetry + } + ); + + handleError(mutationResult); + return mutationResult; +}; + +interface DataExportMutationRequest { + courses: number[]; + fields: string[]; + type: ExportType; +} + +/** + * Mutation for fetching export data for download. + * Returns a table containing the CSV contents. + */ +export const useDataExportMutation = (): UseMutationResult => { + const mutationResult = useMutation( + async (body: DataExportMutationRequest) => { + const endpoint = endpointWithQueryParams( + normalizeEndpoint("/export"), + new URLSearchParams({ + courses: body.courses.join(","), + fields: body.fields.join(","), + type: body.type + }) + ); + + // open csv file endpoint + window.open(endpoint, "_blank"); + return; + } + ); + + handleError(mutationResult); + return mutationResult; +}; diff --git a/csm_web/frontend/static/frontend/img/file-export.svg b/csm_web/frontend/static/frontend/img/file-export.svg new file mode 100644 index 00000000..43ffa9fa --- /dev/null +++ b/csm_web/frontend/static/frontend/img/file-export.svg @@ -0,0 +1 @@ + diff --git a/csm_web/frontend/static/frontend/img/refresh.svg b/csm_web/frontend/static/frontend/img/refresh.svg new file mode 100644 index 00000000..22c0bf3d --- /dev/null +++ b/csm_web/frontend/static/frontend/img/refresh.svg @@ -0,0 +1 @@ + diff --git a/csm_web/scheduler/urls.py b/csm_web/scheduler/urls.py index b997222f..8c60cd33 100644 --- a/csm_web/scheduler/urls.py +++ b/csm_web/scheduler/urls.py @@ -23,4 +23,5 @@ path("matcher//mentors/", views.matcher.mentors), path("matcher//configure/", views.matcher.configure), path("matcher//create/", views.matcher.create), + path("export/", views.export_data), ] diff --git a/csm_web/scheduler/views/__init__.py b/csm_web/scheduler/views/__init__.py index 8002052a..55ed65f3 100644 --- a/csm_web/scheduler/views/__init__.py +++ b/csm_web/scheduler/views/__init__.py @@ -1,5 +1,6 @@ -from .course import CourseViewSet from . import matcher +from .course import CourseViewSet +from .export import export_data from .profile import ProfileViewSet from .resource import ResourceViewSet from .section import SectionViewSet diff --git a/csm_web/scheduler/views/export.py b/csm_web/scheduler/views/export.py new file mode 100644 index 00000000..1dc599d8 --- /dev/null +++ b/csm_web/scheduler/views/export.py @@ -0,0 +1,590 @@ +import csv +import datetime +import io +from typing import Generator, Iterable, List, Optional, Tuple + +from django.contrib.postgres.aggregates import ArrayAgg, JSONBAgg +from django.core.exceptions import BadRequest +from django.db.models import CharField, Count, Q, Value +from django.db.models.functions import Concat +from django.http.response import StreamingHttpResponse +from rest_framework.decorators import api_view +from scheduler.models import Attendance, Course, Section, Student + + +@api_view(["GET"]) +def export_data(request): + """ + Endpoint: /api/export + + GET: Returns a CSV file of exported data. + Query parameters: + preview: int or None + if int > 0, then returns only that many entries from the database + courses: int[] + comma-separated list of course ids + fields: str[] + comma-separated list of fields + type: str + type of data to export + """ + + export_type = request.query_params.get("type", None) + courses_str = request.query_params.get("courses", None) + fields_str = request.query_params.get("fields", "") + preview = request.query_params.get("preview", None) + + if courses_str is None or export_type is None: + raise BadRequest( + "Must include `courses` and `type` fields in the query parameters" + ) + + # convert courses query param into a list of ints + try: + courses = [int(course_id) for course_id in courses_str.split(",")] + except ValueError as exc: + raise BadRequest( + "`courses` query parameter must be a comma-separated list of integers" + ) from exc + fields = fields_str.split(",") + + # check course ids against the user's coordinator courses + coordinator_courses = set( + request.user.coordinator_set.values_list("course__id", flat=True) + ) + courses_set_diff = set(courses).difference(coordinator_courses) + if len(courses_set_diff) > 0: + raise PermissionError( + "You must be a coordinator for all of the courses in the request" + ) + + # convert preview query param into an int + if preview is not None: + try: + preview = int(preview) + except ValueError as exc: + raise BadRequest( + "`preview` query parameter must be an integer or excluded" + ) from exc + + if preview <= 0: + preview = None + + # create generator for the CSV file + csv_generator, filename = prepare_csv(export_type, courses, fields, preview=preview) + + # stream the response; this allows for more efficient data return + response = StreamingHttpResponse( + csv_generator, + content_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + return response + + +def get_section_times_dict(courses: List[int], section_ids: Iterable[int]): + """ + Query the database for section times data, restricting to the given iterable of section ids. + + Normally, all data fields are fetched at the same time in a single query, but + a second aggregate query on a different related field + causes two OUTER JOIN operations in the SQL; this means that + the table size increases multiplicatively, and creates many duplicate items. + An alternative considered is a subquery in the SELECT statement, + but this causes an extra subquery for every returned row in the database. + A last alternative is a subquery to fetch the aggregate data as an INNER JOIN, + but Django does not make it easy to take control of the INNER JOIN calls. + As such, we make a second query to individually fetch the aggregate section time data, + and combine the results in Python. + """ + section_time_queryset = Section.objects.filter( + # filter for courses + mentor__course__id__in=courses + ).annotate( + _location=ArrayAgg("spacetimes__location"), + _start_time=ArrayAgg("spacetimes__start_time"), + _duration=ArrayAgg("spacetimes__duration"), + # weird behavior where ArrayAgg with custom DayOfWeekField + # doesn't work (array of enums seems to be parsed by character + # and returned as a single string) + _day=JSONBAgg("spacetimes__day_of_week"), + ) + + # filter for only students in the earlier queryset + # (some may be omitted by the preview or if the section is empty) + section_time_queryset = section_time_queryset.filter(id__in=section_ids) + + section_time_values = section_time_queryset.values( + "id", + "_location", + "_start_time", + "_duration", + "_day", + ) + # format values in a dictionary for efficient lookup + return {d["id"]: d for d in section_time_values} + + +def format_section_times(section_info: dict): + """ + Format a dictionary of section time info. + """ + # format the section times + locations = section_info["_location"] + start_times: List[datetime.time] = section_info["_start_time"] + durations: List[datetime.timedelta] = section_info["_duration"] + days = section_info["_day"] + + time_list = [] + for loc, day, start, duration in zip(locations, days, start_times, durations): + start_formatted = start.strftime("%I:%M %p") + end_datetime = ( + datetime.datetime.combine(datetime.datetime.today(), start) + duration + ) + end_formatted = end_datetime.time().strftime("%I:%M %p") + time_list.append(f"{loc}, {day} {start_formatted}-{end_formatted}") + formatted_times = "; ".join(time_list) + + return formatted_times + + +def prepare_csv( + export_type: str, + courses: List[int], + fields: List[str], + preview: Optional[int] = None, +) -> Tuple[Generator, str]: + """ + Delegate CSV preparation to various other methods. + """ + + if export_type == "ATTENDANCE_DATA": + generator = prepare_attendance_data(courses, fields, preview=preview) + filename = "attendance_data.csv" + elif export_type == "COURSE_DATA": + generator = prepare_course_data(courses, fields, preview=preview) + filename = "course_data.csv" + elif export_type == "SECTION_DATA": + generator = prepare_section_data(courses, fields, preview=preview) + filename = "section_data.csv" + elif export_type == "STUDENT_DATA": + generator = prepare_student_data(courses, fields, preview=preview) + filename = "student_data.csv" + else: + raise BadRequest("Invalid export type") + + return generator, filename + + +def create_csv_dict_writer(fieldnames, **kwargs): + """ + Create a CSV DictWriter, wrapped around an in-memory buffer. + + All arguments are passed into the DictWriter constructor. + """ + buffer = io.StringIO() + writer = csv.DictWriter(f=buffer, fieldnames=fieldnames, **kwargs) + + def get_data(): + """ + Fetch the current data from the buffer, + and clear it for the next usage. + """ + buffer.seek(0) + data = buffer.read() + buffer.seek(0) + buffer.truncate() + + return data + + return writer, get_data + + +def prepare_attendance_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare attendance data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - student_email + - student_name + Optional: + - course_name + - active + - section_id + - mentor_email + - mentor_name + """ + student_queryset = Student.objects.filter(course__id__in=courses).annotate( + full_name=Concat( + "user__first_name", + Value(" "), + "user__last_name", + output_field=CharField(), + ), + attendance_ids=ArrayAgg("attendance"), + ) + + export_fields = ["user__email", "full_name"] + export_headers = ["Email", "Name"] + + if "course_name" in fields: + export_fields.append("course__name") + export_headers.append("Course") + if "active" in fields: + export_fields.append("active") + export_headers.append("Active") + if "section_id" in fields: + export_fields.append("section__id") + export_headers.append("Section ID") + if "mentor_email" in fields: + export_fields.append("section__mentor__user__email") + export_headers.append("Mentor email") + if "mentor_name" in fields: + student_queryset = student_queryset.annotate( + mentor_name=Concat( + "section__mentor__user__first_name", + Value(" "), + "section__mentor__user__last_name", + output_field=CharField(), + ) + ) + export_fields.append("mentor_name") + export_headers.append("Mentor name") + + if preview is not None and preview > 0: + # limit queryset + student_queryset = student_queryset[:preview] + + student_values = student_queryset.values(*export_fields, "attendance_ids") + + attendance_ids = set() + for student in student_values: + attendance_ids.update(student["attendance_ids"]) + + attendance_queryset = Attendance.objects.filter( + id__in=attendance_ids + ).select_related("sectionOccurrence") + + attendance_values = attendance_queryset.values( + "id", "presence", "sectionOccurrence__date" + ) + + # preprocess to get all possible columns + attendance_dict = {} + date_set = set() + for attendance in attendance_values: + attendance_dict[attendance["id"]] = attendance + date_set.add(attendance["sectionOccurrence__date"]) + + sorted_dates = sorted(date_set) + + sorted_iso_dates = [date.isoformat() for date in sorted_dates] + header_row = export_fields + sorted_iso_dates + header_desc = export_headers + sorted_iso_dates + csv_writer, get_formatted_row = create_csv_dict_writer(header_row) + + header_dict = dict(zip(header_row, header_desc)) + csv_writer.writerow(header_dict) + yield get_formatted_row() + + for student in student_values: + # initialize row + row = {k: v for k, v in student.items() if k in export_fields} + row.update({iso_date: "" for iso_date in sorted_iso_dates}) + + for attendance_id in student["attendance_ids"]: + if attendance_id is None: + continue + + attendance = attendance_dict[attendance_id] + att_date = attendance["sectionOccurrence__date"] + att_presence = attendance["presence"] + + row[att_date.isoformat()] = att_presence + + csv_writer.writerow(row) + yield get_formatted_row() + + +def prepare_course_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare course data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - course_name + Optional: + - course_id + - description + - num_sections + - num_students + - num_mentors + """ + + course_queryset = Course.objects.filter(id__in=courses) + + export_fields = ["name"] + export_headers = ["Name"] + if "course_id" in fields: + export_fields.append("id") + export_headers.append("Course ID") + if "description" in fields: + export_fields.append("title") + export_headers.append("Course title") + if "num_sections" in fields: + course_queryset = course_queryset.annotate( + num_sections=Count("mentor__section") + ) + export_fields.append("num_sections") + export_headers.append("Number of sections") + if "num_students" in fields: + course_queryset = course_queryset.annotate( + num_students=Count("mentor__section__students") + ) + export_fields.append("num_students") + export_headers.append("Number of students") + if "num_mentors" in fields: + course_queryset = course_queryset.annotate(num_mentors=Count("mentor")) + export_fields.append("num_mentors") + export_headers.append("Number of mentors") + + if preview is not None and preview > 0: + # limit queryset + course_queryset = course_queryset[:preview] + + values = course_queryset.values(*export_fields) + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + csv_writer.writerow(row) + yield get_formatted_row() + + +def prepare_section_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare section data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - mentor_email + - mentor_name + Optional: + - course_name + - section_id + - section_times + - section_description + - num_students + - capacity + """ + section_queryset = Section.objects.filter(mentor__course__id__in=courses).annotate( + mentor_name=Concat( + "mentor__user__first_name", + Value(" "), + "mentor__user__last_name", + output_field=CharField(), + ) + ) + + export_fields = ["mentor__user__email", "mentor_name"] + export_headers = ["Mentor email", "Mentor name"] + + if "course_name" in fields: + export_fields.append("mentor__course__name") + export_headers.append("Course") + if "section_id" in fields: + export_fields.append("id") + export_headers.append("Section ID") + if "section_description" in fields: + export_fields.append("description") + export_headers.append("Description") + if "num_students" in fields: + section_queryset = section_queryset.annotate(num_students=Count("students")) + export_fields.append("num_students") + export_headers.append("Student count") + if "capacity" in fields: + export_fields.append("capacity") + export_headers.append("Capacity") + + if preview is not None and preview > 0: + # limit queryset + section_queryset = section_queryset[:preview] + + # query database for values + values = section_queryset.values(*export_fields) + + section_time_dict = {} + if "section_times" in fields: + used_ids = set(d["id"] for d in values) + section_time_dict = get_section_times_dict(courses, used_ids) + + export_fields.append("section_times") + export_headers.append("Section times") + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + # filter out unwanted fields (id, etc.) + final_row = {k: v for k, v in row.items() if k in export_fields} + if "section_times" in fields: + # fetch section info from auxiliary query + section_info = section_time_dict[row["id"]] + formatted_times = format_section_times(section_info) + final_row["section_times"] = formatted_times + + csv_writer.writerow(final_row) + yield get_formatted_row() + + +def prepare_student_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare student data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - student_email + - student_name + Optional: + - course_name + - active + - mentor_email + - mentor_name + - section_id + - section_times + - num_present + - num_excused + - num_unexcused + """ + # include the full name in the student queryset by default + # (email is already included as user__email) + student_queryset = Student.objects.filter(course__id__in=courses).annotate( + full_name=Concat( + "user__first_name", Value(" "), "user__last_name", output_field=CharField() + ) + ) + + # fields to fetch from the database + export_qs_fields = ["user__email", "full_name", "section__id"] + # fields to use for the CSV file; must correspond exactly to export_headers + export_fields = ["user__email", "full_name"] + # headers to use for the CSV file; must correspond exactly to export_fields + export_headers = ["Email", "Name"] + + if "course_name" in fields: + export_fields.append("course__name") + export_qs_fields.append("course__name") + export_headers.append("Course") + if "active" in fields: + export_fields.append("active") + export_qs_fields.append("active") + export_headers.append("Active") + if "mentor_email" in fields: + export_fields.append("section__mentor__user__email") + export_qs_fields.append("section__mentor__user__email") + export_headers.append("Mentor email") + if "mentor_name" in fields: + student_queryset = student_queryset.annotate( + mentor_name=Concat( + "section__mentor__user__first_name", + Value(" "), + "section__mentor__user__last_name", + output_field=CharField(), + ) + ) + export_fields.append("mentor_name") + export_qs_fields.append("mentor_name") + export_headers.append("Mentor name") + if "section_id" in fields: + export_fields.append("section__id") + export_headers.append("Section ID") + + if "num_present" in fields: + student_queryset = student_queryset.annotate( + num_present=Count("attendance", filter=Q(attendance__presence="PR")) + ) + export_fields.append("num_present") + export_qs_fields.append("num_present") + export_headers.append("Present count") + if "num_unexcused" in fields: + student_queryset = student_queryset.annotate( + num_unexcused=Count("attendance", filter=Q(attendance__presence="UN")) + ) + export_fields.append("num_unexcused") + export_qs_fields.append("num_unexcused") + export_headers.append("Unexcused count") + if "num_excused" in fields: + student_queryset = student_queryset.annotate( + num_excused=Count("attendance", filter=Q(attendance__presence="EX")) + ) + export_fields.append("num_excused") + export_qs_fields.append("num_excused") + export_headers.append("Excused count") + + if preview is not None and preview > 0: + # limit queryset + student_queryset = student_queryset[:preview] + + # query database for values + values = student_queryset.values(*export_qs_fields) + + # default empty dict (not used if section_times is not specified) + section_time_dict = {} + if "section_times" in fields: + # A second aggregate query on a different related field + # causes two OUTER JOIN operations in the SQL; this means that + # the table size increases multiplicatively, and creates many duplicate items. + # An alternative considered is a subquery in the SELECT statement, + # but this causes an extra subquery for every returned row in the database. + # A last alternative is a subquery to fetch the aggregate data as an INNER JOIN, + # but Django does not make it easy to take control of the INNER JOIN calls. + # As such, we make a second query to individually fetch the aggregate section time data, + # and combine the results in Python. + + used_ids = set(d["section__id"] for d in values) + section_time_dict = get_section_times_dict(courses, used_ids) + + export_fields.append("section_times") + export_headers.append("Section times") + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + # filter out unwanted fields (id, etc.) + final_row = {k: v for k, v in row.items() if k in export_fields} + if "section_times" in fields: + # fetch section info from auxiliary query + section_info = section_time_dict[row["section__id"]] + formatted_times = format_section_times(section_info) + final_row["section_times"] = formatted_times + + csv_writer.writerow(final_row) + yield get_formatted_row() diff --git a/package-lock.json b/package-lock.json index 6ce9e501..974df6c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,6 +48,7 @@ "css-loader": "^6.8.1", "css-minimizer-webpack-plugin": "^5.0.1", "csso-cli": "^3.0.0", + "csv-parse": "^5.5.3", "cypress": "^13.6.1", "cypress-pipe": "^2.0.0", "eslint": "^8.50.0", @@ -6464,6 +6465,12 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.8.tgz", "integrity": "sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==" }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==", + "dev": true + }, "node_modules/cypress": { "version": "13.6.1", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.6.1.tgz", @@ -21247,6 +21254,12 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.8.tgz", "integrity": "sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==" }, + "csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==", + "dev": true + }, "cypress": { "version": "13.6.1", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.6.1.tgz", diff --git a/package.json b/package.json index 0312eab6..ab9d8732 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "css-loader": "^6.8.1", "css-minimizer-webpack-plugin": "^5.0.1", "csso-cli": "^3.0.0", + "csv-parse": "^5.5.3", "cypress": "^13.6.1", "cypress-pipe": "^2.0.0", "eslint": "^8.50.0", From 4d1732fa80a9450cf46cf888ef4d3f509f2b47c9 Mon Sep 17 00:00:00 2001 From: Alec Li Date: Sat, 27 Jan 2024 16:25:11 -0800 Subject: [PATCH 2/2] Restyle export page, fix various bugs Add automatic preview refresh, restyle and add tooltip Rearrange export page sections into one page Fix section id query bugs, rework cases where no courses are selected Add additional fields to export types, fix download data with no courses Add statuses for export page --- .../src/components/data_export/DataExport.tsx | 40 +++-- .../data_export/DataExportTypes.tsx | 16 +- .../src/components/data_export/ExportPage.tsx | 144 ++++++++++-------- .../components/data_export/ExportSelector.tsx | 48 +++--- csm_web/frontend/src/css/data-export.scss | 69 ++++++--- csm_web/frontend/src/utils/queries/export.tsx | 6 + csm_web/scheduler/views/export.py | 89 +++++++++-- 7 files changed, 275 insertions(+), 137 deletions(-) diff --git a/csm_web/frontend/src/components/data_export/DataExport.tsx b/csm_web/frontend/src/components/data_export/DataExport.tsx index 0e4a3d25..5cadff82 100644 --- a/csm_web/frontend/src/components/data_export/DataExport.tsx +++ b/csm_web/frontend/src/components/data_export/DataExport.tsx @@ -1,22 +1,44 @@ import React, { useState } from "react"; +import { useProfiles } from "../../utils/queries/base"; +import { Role } from "../../utils/types"; +import LoadingSpinner from "../LoadingSpinner"; import { ExportType } from "./DataExportTypes"; import { ExportPage } from "./ExportPage"; import { ExportSelector } from "./ExportSelector"; export const DataExport = () => { const [dataExportType, setDataExportType] = useState(null); + const { data: profiles, isSuccess: profilesLoaded, isError: profilesError } = useProfiles(); + + if (profilesError) { + return Error loading user profiles.; + } else if (!profilesLoaded) { + return ; + } else if (profilesLoaded && !profiles.some(profile => profile.role === Role.COORDINATOR)) { + return Permission denied; you are not a coordinator for any course.; + } return (
- {dataExportType === null ? ( - { - setDataExportType(exportType); - }} - /> - ) : ( - setDataExportType(null)} /> - )} +
+

Export Data

+
+
+
+ { + setDataExportType(exportType); + }} + /> +
+
+ {dataExportType === null ? ( +
Select export type to start.
+ ) : ( + + )} +
+
); }; diff --git a/csm_web/frontend/src/components/data_export/DataExportTypes.tsx b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx index 2562ef74..d117cd21 100644 --- a/csm_web/frontend/src/components/data_export/DataExportTypes.tsx +++ b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx @@ -12,10 +12,10 @@ export enum ExportType { * Object for displaying export types in the UI */ export const EXPORT_TYPE_DATA = new Map([ - [ExportType.STUDENT_DATA, "Student data"], - [ExportType.ATTENDANCE_DATA, "Attendance data"], - [ExportType.SECTION_DATA, "Section data"], - [ExportType.COURSE_DATA, "Course data"] + [ExportType.STUDENT_DATA, "Student"], + [ExportType.ATTENDANCE_DATA, "Attendance"], + [ExportType.SECTION_DATA, "Section"], + [ExportType.COURSE_DATA, "Course"] ]); export const EXPORT_COLUMNS: { @@ -27,14 +27,18 @@ export const EXPORT_COLUMNS: { [ExportType.ATTENDANCE_DATA]: { required: { student_email: "Student email", - student_name: "Student name" + student_name: "Student name", + attendance_data: "Attendance data" }, optional: { course_name: "Course name", active: "Active", section_id: "Section ID", mentor_name: "Mentor name", - mentor_email: "Mentor email" + mentor_email: "Mentor email", + num_present: "Present attendance count", + num_excused: "Excused absence count", + num_unexcused: "Unexcused absence count" } }, [ExportType.COURSE_DATA]: { diff --git a/csm_web/frontend/src/components/data_export/ExportPage.tsx b/csm_web/frontend/src/components/data_export/ExportPage.tsx index f1c14e49..808b6c85 100644 --- a/csm_web/frontend/src/components/data_export/ExportPage.tsx +++ b/csm_web/frontend/src/components/data_export/ExportPage.tsx @@ -4,16 +4,16 @@ import { useDataExportMutation, useDataExportPreviewMutation } from "../../utils import { Role } from "../../utils/types"; import { AutoGrid } from "../AutoGrid"; import LoadingSpinner from "../LoadingSpinner"; +import { Tooltip } from "../Tooltip"; import { ExportType, EXPORT_COLUMNS } from "./DataExportTypes"; import RefreshIcon from "../../../static/frontend/img/refresh.svg"; interface ExportPageProps { dataExportType: ExportType; - onBack: () => void; } -export const ExportPage = ({ dataExportType, onBack }: ExportPageProps) => { +export const ExportPage = ({ dataExportType }: ExportPageProps) => { const { data: profiles, isSuccess: profilesLoaded, isError: profilesError } = useProfiles(); const [includedCourses, setIncludedCourses] = useState([]); const [includedFields, setIncludedFields] = useState( @@ -29,6 +29,10 @@ export const ExportPage = ({ dataExportType, onBack }: ExportPageProps) => { } }, [profilesLoaded, profiles]); + useEffect(() => { + setIncludedFields(Array.from(Object.keys(EXPORT_COLUMNS[dataExportType].optional))); + }, [dataExportType]); + if (profilesError) { return

Profiles not found

; } else if (!profilesLoaded) { @@ -111,6 +115,11 @@ export const ExportPage = ({ dataExportType, onBack }: ExportPageProps) => { * Download the data; open a new page with the data */ const downloadData = () => { + if (includedCourses.length === 0) { + // no data to download + return; + } + dataExportMutation.mutate({ courses: includedCourses, fields: includedFields, @@ -120,19 +129,13 @@ export const ExportPage = ({ dataExportType, onBack }: ExportPageProps) => { return (
-
-

Export Data

- -
{courseSelection}
{columnSelection}
- +
-
@@ -146,7 +149,6 @@ interface ExportPagePreviewProps { courses: number[]; fields: string[]; exportType: ExportType; - preview: number; } /** @@ -157,11 +159,12 @@ const ExportPagePreview = ({ exportType, courses, fields }: ExportPagePreviewPro const [preview, setPreview] = useState(10); const [data, setData] = useState([]); - const refreshPreview = () => { - if (courses.length == 0) { - return; - } + // automatically refresh on change + useEffect(() => { + refreshPreview(); + }, [courses, fields, preview]); + const refreshPreview = () => { dataExportPreviewMutation.mutate( { courses: courses, @@ -182,54 +185,73 @@ const ExportPagePreview = ({ exportType, courses, fields }: ExportPagePreviewPro setPreview(selectedValue); }; + let dataTable = null; + + if (data?.length > 1) { + // if has header and at least one row of content, display it + dataTable = ( + + + + {data?.length > 0 && + data[0].map((cell, cellIdx) => ( + + ))} + + + + {data.map( + (row, rowIdx) => + rowIdx > 0 && ( + + {row.map((cell, cellIdx) => ( + + ))} + + ) + )} + {data?.length >= preview && ( + + + + )} + +
+ {cell} +
+ {cell} +
+ (More rows clipped) +
+ ); + } else { + // not enough data + dataTable = Preview query returned no data.; + } + return (
-

Preview

-
- Rows: - - -
-
- - - - {data?.length > 0 && - data[0].map((cell, cellIdx) => ( - - ))} - - - - {data.map( - (row, rowIdx) => - rowIdx > 0 && ( - - {row.map((cell, cellIdx) => ( - - ))} - - ) - )} - {data?.length >= preview && ( - - - - )} - -
- {cell} -
- {cell} -
- (More rows clipped) -
+

Preview

+
+
+ Rows: + +
+ } + placement="right" + > + Refresh + +
+
+
{dataTable}
); diff --git a/csm_web/frontend/src/components/data_export/ExportSelector.tsx b/csm_web/frontend/src/components/data_export/ExportSelector.tsx index 56cd5c6a..e05e215c 100644 --- a/csm_web/frontend/src/components/data_export/ExportSelector.tsx +++ b/csm_web/frontend/src/components/data_export/ExportSelector.tsx @@ -5,47 +5,35 @@ import { ExportType, EXPORT_TYPE_DATA } from "./DataExportTypes"; import "../../css/data-export.scss"; interface ExportSelectorProps { - onContinue: (exportType: ExportType) => void; + onSelect: (exportType: ExportType) => void; } /** * Component for selecting the courses to include in the export, * along with the export data config selection. */ -export const ExportSelector = ({ onContinue }: ExportSelectorProps) => { - const [dataExportType, setDataExportType] = useState(ExportType.ATTENDANCE_DATA); +export const ExportSelector = ({ onSelect }: ExportSelectorProps) => { + const [dataExportType, setDataExportType] = useState(null); - const handleContinue = () => { - onContinue(dataExportType); + const handleSelect = (exportType: ExportType) => { + onSelect(exportType); + setDataExportType(exportType); }; return (
-
-

Select Export Data

-
-
- {Array.from(EXPORT_TYPE_DATA.entries()) - .sort() - .map(([exportType, description]) => ( - - ))} -
-
-
-
-
- Continue -
+
+ {Array.from(EXPORT_TYPE_DATA.entries()) + .sort() + .map(([exportType, description]) => ( +
handleSelect(exportType)} + > + {description} +
+ ))}
); diff --git a/csm_web/frontend/src/css/data-export.scss b/csm_web/frontend/src/css/data-export.scss index d457fdf5..cf7dcb04 100644 --- a/csm_web/frontend/src/css/data-export.scss +++ b/csm_web/frontend/src/css/data-export.scss @@ -4,38 +4,40 @@ .data-export-container { display: flex; flex-direction: column; - gap: 40px; align-items: stretch; justify-content: center; + + padding-right: 16px; } -.export-selector-container { +.data-export-body { display: flex; - flex-direction: column; + flex-direction: row; - gap: 50px; + gap: 40px; } -.export-selector-footer { - display: flex; - align-items: center; - justify-content: center; +.data-export-sidebar { + min-width: 150px; } -.export-selector-section, -.export-page-section { +.data-export-content { + flex: 1; +} + +.export-selector-container { display: flex; flex-direction: column; - justify-content: center; + gap: 50px; + + padding-left: 24px; + margin-top: 16px; } -.export-selector-data-type-container { +.export-selector-footer { display: flex; - flex-direction: column; align-items: center; justify-content: center; - - padding-top: 15px; } .export-page-sidebar-container { @@ -52,7 +54,7 @@ .export-selector-data-type-options { display: flex; flex-direction: column; - gap: 8px; + gap: 24px; align-items: flex-start; justify-content: center; @@ -64,18 +66,27 @@ max-height: 25%; } -.export-selector-data-type-label, -.export-page-input-label { +.export-selector-data-type-label { display: block; width: 100%; + + font-size: 1.1rem; + white-space: nowrap; + + cursor: pointer; user-select: none; + + &.active { + color: $csm-green-darkened; + } } .export-page-container { display: flex; flex-direction: column; gap: 16px; + align-items: stretch; } .export-page-config { @@ -123,6 +134,11 @@ gap: 4px; } +.export-page-preview-title { + // make margin smaller + margin-bottom: 8px; +} + .export-preview-header { display: flex; flex-direction: row; @@ -145,11 +161,24 @@ color: #222; } +.export-preview-refresh-tooltip-container { + position: relative; + + .tooltip-body { + // offset tooltip slightly + margin-left: 8px; + } +} + .export-preview-table-container { - max-width: 90vw; + max-width: 80vw; overflow-x: auto; } +.export-page-preview-wrapper { + padding: 0 12px; +} + .export-preview-table { border-collapse: collapse; } @@ -171,5 +200,5 @@ color: #888; column-span: all; - text-align: center; + text-align: left; } diff --git a/csm_web/frontend/src/utils/queries/export.tsx b/csm_web/frontend/src/utils/queries/export.tsx index a57e01ff..1407d468 100644 --- a/csm_web/frontend/src/utils/queries/export.tsx +++ b/csm_web/frontend/src/utils/queries/export.tsx @@ -23,6 +23,12 @@ export const useDataExportPreviewMutation = (): UseMutationResult< > => { const mutationResult = useMutation( async (body: DataExportPreviewMutationRequest) => { + if (body.courses.length === 0) { + // if no courses specified, then return an empty table; + // no request is needed + return [[]]; + } + const response = await fetchNormalized( "/export", new URLSearchParams({ diff --git a/csm_web/scheduler/views/export.py b/csm_web/scheduler/views/export.py index 1dc599d8..ea055df2 100644 --- a/csm_web/scheduler/views/export.py +++ b/csm_web/scheduler/views/export.py @@ -102,6 +102,7 @@ def get_section_times_dict(courses: List[int], section_ids: Iterable[int]): # filter for courses mentor__course__id__in=courses ).annotate( + _num_spacetimes=Count("spacetimes"), _location=ArrayAgg("spacetimes__location"), _start_time=ArrayAgg("spacetimes__start_time"), _duration=ArrayAgg("spacetimes__duration"), @@ -121,14 +122,17 @@ def get_section_times_dict(courses: List[int], section_ids: Iterable[int]): "_start_time", "_duration", "_day", + "_num_spacetimes", ) # format values in a dictionary for efficient lookup return {d["id"]: d for d in section_time_values} -def format_section_times(section_info: dict): +def format_section_times(section_info: dict) -> list[str]: """ Format a dictionary of section time info. + + Returns a list of formatted section spacetimes. """ # format the section times locations = section_info["_location"] @@ -144,9 +148,8 @@ def format_section_times(section_info: dict): ) end_formatted = end_datetime.time().strftime("%I:%M %p") time_list.append(f"{loc}, {day} {start_formatted}-{end_formatted}") - formatted_times = "; ".join(time_list) - return formatted_times + return time_list def prepare_csv( @@ -218,6 +221,9 @@ def prepare_attendance_data( - section_id - mentor_email - mentor_name + - num_present + - num_excused + - num_unexcused """ student_queryset = Student.objects.filter(course__id__in=courses).annotate( full_name=Concat( @@ -255,6 +261,24 @@ def prepare_attendance_data( ) export_fields.append("mentor_name") export_headers.append("Mentor name") + if "num_present" in fields: + student_queryset = student_queryset.annotate( + num_present=Count("attendance", filter=Q(attendance__presence="PR")) + ) + export_fields.append("num_present") + export_headers.append("Present count") + if "num_unexcused" in fields: + student_queryset = student_queryset.annotate( + num_unexcused=Count("attendance", filter=Q(attendance__presence="UN")) + ) + export_fields.append("num_unexcused") + export_headers.append("Unexcused count") + if "num_excused" in fields: + student_queryset = student_queryset.annotate( + num_excused=Count("attendance", filter=Q(attendance__presence="EX")) + ) + export_fields.append("num_excused") + export_headers.append("Excused count") if preview is not None and preview > 0: # limit queryset @@ -426,16 +450,32 @@ def prepare_section_data( # limit queryset section_queryset = section_queryset[:preview] - # query database for values - values = section_queryset.values(*export_fields) + # query database for values; always fetch id + values = section_queryset.values("id", *export_fields) section_time_dict = {} + max_spacetime_count = 0 if "section_times" in fields: used_ids = set(d["id"] for d in values) section_time_dict = get_section_times_dict(courses, used_ids) - export_fields.append("section_times") - export_headers.append("Section times") + # get the maximum number of section spacetimes + if len(section_time_dict) > 0: + max_spacetime_count = max( + d["_num_spacetimes"] for d in section_time_dict.values() + ) + + # these appends are only for the csv writer + if max_spacetime_count > 1: + for spacetime_idx in range(1, max_spacetime_count + 1): + export_fields.append(f"section_times_{spacetime_idx}") + export_headers.append(f"Section times ({spacetime_idx})") + else: + # if there is zero or one spacetime, the header doesn't need to differentiate + # between indices; we still keep the index in the raw field, + # to simplify the code in writing to the csv + export_fields.append("section_times_1") + export_headers.append("Section times") csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) @@ -451,7 +491,13 @@ def prepare_section_data( # fetch section info from auxiliary query section_info = section_time_dict[row["id"]] formatted_times = format_section_times(section_info) - final_row["section_times"] = formatted_times + + # write formatted spacetimes in separate columns + for spacetime_idx in range(max_spacetime_count): + cur_formatted = "" # default to empty string to pad extras + if spacetime_idx < len(formatted_times): + cur_formatted = formatted_times[spacetime_idx] + final_row[f"section_times_{spacetime_idx + 1}"] = cur_formatted csv_writer.writerow(final_row) yield get_formatted_row() @@ -553,6 +599,7 @@ def prepare_student_data( # default empty dict (not used if section_times is not specified) section_time_dict = {} + max_spacetime_count = 0 if "section_times" in fields: # A second aggregate query on a different related field # causes two OUTER JOIN operations in the SQL; this means that @@ -567,8 +614,23 @@ def prepare_student_data( used_ids = set(d["section__id"] for d in values) section_time_dict = get_section_times_dict(courses, used_ids) - export_fields.append("section_times") - export_headers.append("Section times") + # get the maximum number of section spacetimes + if len(section_time_dict) > 0: + max_spacetime_count = max( + d["_num_spacetimes"] for d in section_time_dict.values() + ) + + # these appends are only for the csv writer + if max_spacetime_count > 1: + for spacetime_idx in range(max_spacetime_count): + export_fields.append(f"section_times_{spacetime_idx + 1}") + export_headers.append(f"Section times ({spacetime_idx + 1})") + else: + # if there is zero or one spacetime, the header doesn't need to differentiate + # between indices; we still keep the index in the raw field, + # to simplify the code in writing to the csv + export_fields.append("section_times_1") + export_headers.append("Section times") csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) @@ -584,7 +646,12 @@ def prepare_student_data( # fetch section info from auxiliary query section_info = section_time_dict[row["section__id"]] formatted_times = format_section_times(section_info) - final_row["section_times"] = formatted_times + # write formatted spacetimes in separate columns + for spacetime_idx in range(max_spacetime_count): + cur_formatted = "" # default to empty string to pad extras + if spacetime_idx < len(formatted_times): + cur_formatted = formatted_times[spacetime_idx] + final_row[f"section_times_{spacetime_idx + 1}"] = cur_formatted csv_writer.writerow(final_row) yield get_formatted_row()