From 39a9a2687ab6b6770cd96689d7b4ee5cb6a70945 Mon Sep 17 00:00:00 2001 From: bluescorpian Date: Tue, 4 Mar 2025 16:43:32 +0200 Subject: [PATCH] allow download specefic years/months and stream the csv file --- backend/main.ts | 146 ++++++++++++++++++++------ frontend/src/lib/DownloadModal.svelte | 65 ++++++++++++ frontend/src/lib/Stations.svelte | 26 +++-- types.d.ts | 3 +- 4 files changed, 198 insertions(+), 42 deletions(-) create mode 100644 frontend/src/lib/DownloadModal.svelte diff --git a/backend/main.ts b/backend/main.ts index ea89fd3..93ba24e 100644 --- a/backend/main.ts +++ b/backend/main.ts @@ -1,6 +1,6 @@ import { StationStats, type TableDataItem } from "../types.d.ts"; import { getWSData } from "./wind2speed.ts"; -import { Application, Router } from "@oak/oak"; +import { Application, Context, Router } from "@oak/oak"; import { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts"; import { stringify } from "jsr:@std/csv/stringify"; @@ -41,6 +41,7 @@ async function downloadWindData(stationId: number) { "station", data.station.id, ]); + const months = stationStats.value?.months || []; const latestEntryTimestamp = stationStats.value?.latestEntryTimestamp || 0; const newTableData = data.tableData.filter( @@ -53,18 +54,28 @@ async function downloadWindData(stationId: number) { const transaction = kv.atomic(); for (const entry of newTableData) { + const date = parseObsTimeLocal(entry.obsTimeLocal); + const year = date.getFullYear(); + const month = date.getMonth(); + + if (!months.some((m) => m.year === year && m.month === month)) { + months.push({ year, month }); + } + transaction.set( - ["windHistoryData", data.station.id, entry.id], + ["windHistoryData", data.station.id, year, month, entry.id], entry ); } transaction.set(["station", data.station.id], { ...data.station, - entries: (stationStats.value?.entries ?? 0) + newTableData.length, + totalEntries: + (stationStats.value?.totalEntries ?? 0) + newTableData.length, latestEntryTimestamp: parseObsTimeLocal( newTableData[0].obsTimeLocal ).getTime(), - }); + months, + } as StationStats); transaction.set(["latestUpdatedStation"], stationId); const result = await transaction.commit(); @@ -86,6 +97,20 @@ function parseObsTimeLocal(obsTimeLocal: string): Date { const router = new Router(); +const authMiddleware = async (ctx: Context, next: () => Promise) => { + if ( + (ctx.request.headers.get("Authorization") ?? "") !== + (Deno.env.get("PASSWORD") ?? "") + ) { + ctx.response.status = 401; + ctx.response.body = { + msg: "Unauthorized", + }; + return; + } + await next(); +}; + router.post("/login", async (ctx) => { const body = await ctx.request.body.text(); const formData = new URLSearchParams(body); @@ -104,18 +129,7 @@ router.get("/tracked-stations", async (ctx) => { ctx.response.body = trackedStations ?? []; }); -router.post("/tracked-stations", async (ctx) => { - if ( - (ctx.request.headers.get("Authorization") ?? "") !== - (Deno.env.get("PASSWORD") ?? "") - ) { - ctx.response.status = 401; - ctx.response.body = { - msg: "Unauthorized", - }; - return; - } - +router.post("/tracked-stations", authMiddleware, async (ctx) => { try { const body = await ctx.request.body.text(); const formData = new URLSearchParams(body); @@ -140,6 +154,35 @@ router.post("/tracked-stations", async (ctx) => { } }); +router.post( + "/update-station-months/:stationId", + authMiddleware, + async (ctx) => { + const stationId = Number(ctx.params.stationId); + const stationStats = await kv.get(["station", stationId]); + const tableDataEntries = await kv.list({ + prefix: ["windHistoryData", stationId], + }); + + for await (const entry of tableDataEntries) { + const date = parseObsTimeLocal(entry.value.obsTimeLocal); + const year = date.getFullYear(); + const month = date.getMonth(); + + if ( + !stationStats.value?.months.some( + (m) => m.year === year && m.month === month + ) + ) { + stationStats.value?.months.push({ year, month }); + } + } + await kv.set(["station", stationId], stationStats.value); + + ctx.response.status = 200; + } +); + router.get("/stations", async (ctx) => { const stationsEntries = await kv.list({ prefix: ["station"], @@ -152,20 +195,35 @@ router.get("/stations", async (ctx) => { ctx.response.body = stations; }); -router.get("/wind-history/:stationId/csv", async (ctx) => { +router.get("/wind-history/:stationId", async (ctx) => { const stationId = Number(ctx.params.stationId); + const fileformat = ctx.request.url.searchParams.get("fileformat") || "csv"; + const year = Number(ctx.request.url.searchParams.get("year")); + const month = Number(ctx.request.url.searchParams.get("month")); + let datePrefix = []; + if (year) { + datePrefix.push(year); + if (month) datePrefix.push(month); + } + + const filenameParts = ["wind-history", stationId.toString()]; + if (year) { + filenameParts.push(year.toString()); + if (month) { + filenameParts.push(month.toString()); + } + } + const filename = filenameParts.join("-") + ".csv"; + const entries = await kv.list({ - prefix: ["windHistoryData", stationId], + prefix: ["windHistoryData", stationId, ...datePrefix], }); - const data: TableDataItem[] = []; - for await (const entry of entries) { - data.push(entry.value); - } - const csv = await stringify( - data as unknown as readonly Record[], - { - headers: true, - columns: [ + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + // Write CSV headers + const headers = [ "id", "stationId", "obsTimeLocal", @@ -178,16 +236,38 @@ router.get("/wind-history/:stationId/csv", async (ctx) => { "humidityAvg", "tempAvg", "pressureAvg", - ], - } - ); + ]; + controller.enqueue(encoder.encode(headers.join(",") + "\n")); + + // Write CSV data rows + for await (const entry of entries) { + const row = [ + entry.value.id, + entry.value.stationId, + entry.value.obsTimeLocal, + entry.value.winddirHigh, + entry.value.winddirLow, + entry.value.winddirAvg, + entry.value.windspeedHigh, + entry.value.windspeedAvg, + entry.value.windspeedLow, + entry.value.humidityAvg, + entry.value.tempAvg, + entry.value.pressureAvg, + ]; + controller.enqueue(encoder.encode(row.join(",") + "\n")); + } + + controller.close(); + }, + }); + ctx.response.headers.set("Content-Type", "text/csv"); - const date = new Date().toISOString().split("T")[0]; ctx.response.headers.set( "Content-Disposition", - `attachment; filename="wind-history-${stationId}-${date}.csv"` + `attachment; filename="${filename}"` ); - ctx.response.body = csv; + ctx.response.body = stream; }); const app = new Application(); diff --git a/frontend/src/lib/DownloadModal.svelte b/frontend/src/lib/DownloadModal.svelte new file mode 100644 index 0000000..239875a --- /dev/null +++ b/frontend/src/lib/DownloadModal.svelte @@ -0,0 +1,65 @@ + + + (isOpen = !isOpen)}> + Download wind history + +

{station.nam} ({station.id})

+ + {#each years as year} + + {year} + + + {#each station.months + .filter((m) => m.year === year) + .map((m) => m.month) + .sort((a, b) => a - b) as month} + + + {year} / {moment().month(month).format("MMM")} + + + + + {/each} + {/each} + + + diff --git a/frontend/src/lib/Stations.svelte b/frontend/src/lib/Stations.svelte index 8e1f29c..be08245 100644 --- a/frontend/src/lib/Stations.svelte +++ b/frontend/src/lib/Stations.svelte @@ -9,10 +9,12 @@ ListGroupItem, } from "@sveltestrap/sveltestrap"; import { type StationStats } from "../../../types"; - import moment from "moment"; + import DownloadModal from "./DownloadModal.svelte"; let stations: StationStats[] = $state([]); + let downloadOpen: boolean = $state(false); + let downloadStation: StationStats | null = $state(null); $effect(() => { fetch(import.meta.env.VITE_API_URL + "/stations") .then(async (res) => { @@ -38,15 +40,23 @@ ).fromNow()} Total Entries: {station.entries}Total Entries: {station.totalEntries} - + + + {/each} +{#if downloadStation} + +{/if} diff --git a/types.d.ts b/types.d.ts index f46024c..d96f7ac 100644 --- a/types.d.ts +++ b/types.d.ts @@ -81,6 +81,7 @@ export interface WindData { } export interface StationStats extends Station { - entries: number; + totalEntries: number; + months: { year: number; month: number }[]; latestEntryTimestamp: number; }