Merge pull request #448 from akvaplan-niva/feat-pub-multiplicates

pub multiplicates feature v0

components/social_media_icons.tsx

@@ -51,7 +51,7 @@ const github = {
 const akvaplanSocial: Social[] = [
   // phone,
   // mail,
-  x,
+  //x,
   face,
   linked,
   github,

deno.json

@@ -19,17 +19,17 @@
     "@preact/signals-core": "https://esm.sh/*@preact/[email protected]",
     "@deno/kv-oauth": "jsr:@deno/[email protected]",
     "@hyperjump/json-pointer": "npm:@hyperjump/[email protected]",
-    "@orama/orama": "npm:@orama/orama@2.1.1",
-    "@orama/plugin-data-persistence": "npm:@orama/plugin-data-persistence@2.1.1",
-    "@orama/stemmers": "npm:@orama/stemmers@2",
-    "@std/async": "jsr:@std/[email protected].8",
-    "@std/encoding": "jsr:@std/[email protected].5",
-    "@std/http": "jsr:@std/[email protected].10",
+    "@orama/orama": "npm:@orama/orama@3.0.8",
+    "@orama/plugin-data-persistence": "npm:@orama/plugin-data-persistence@3.0.8",
+    "@orama/stemmers": "npm:@orama/stemmers@^3.0.8",
+    "@std/async": "jsr:@std/[email protected].10",
+    "@std/encoding": "jsr:@std/[email protected].7",
+    "@std/http": "jsr:@std/[email protected].13",
     "@std/ulid": "jsr:@std/[email protected]",
     "@valibot/valibot": "jsr:@valibot/[email protected]",
     "accept-language-parser": "npm:[email protected]",
     "akvaplan_fresh/": "./",
-    "open-props": "npm:open-props@1.5.9",
+    "open-props": "npm:open-props@1.7.12",
     "slug": "https://deno.land/x/[email protected]/mod.ts"
   },
   "scopes": {},

fresh.gen.ts

@@ -56,6 +56,7 @@ import * as $project from "./routes/project.tsx";
 import * as $projects from "./routes/projects.tsx";
 import * as $projects_nva from "./routes/projects_nva.tsx";
 import * as $pub from "./routes/pub.tsx";
+import * as $pub_multiplicates from "./routes/pub_multiplicates.tsx";
 import * as $pubs from "./routes/pubs.tsx";
 import * as $research from "./routes/research.tsx";
 import * as $research_topics from "./routes/research_topics.tsx";
@@ -147,6 +148,7 @@ const manifest = {
     "./routes/projects.tsx": $projects,
     "./routes/projects_nva.tsx": $projects_nva,
     "./routes/pub.tsx": $pub,
+    "./routes/pub_multiplicates.tsx": $pub_multiplicates,
     "./routes/pubs.tsx": $pubs,
     "./routes/research.tsx": $research,
     "./routes/research_topics.tsx": $research_topics,

routes/pub_multiplicates.tsx

@@ -0,0 +1,59 @@
+import { pubs } from "akvaplan_fresh/services/pub.ts";
+import { Page } from "akvaplan_fresh/components/mod.ts";
+import { defineRoute, type RouteConfig } from "$fresh/server.ts";
+import { SearchResultItem } from "akvaplan_fresh/components/search_result_item.tsx";
+import { atomizeSlimPublication } from "akvaplan_fresh/search/indexers/pubs.ts";
+
+export const config: RouteConfig = {
+  routeOverride: "/:lang(en|no)/:page(pubs|publications|publikasjoner)/multi",
+};
+
+const getMultiplicates = async () => {
+  const all = await Array.fromAsync(await pubs());
+  const atoms = await Array.fromAsync(all.map(async (
+    { value },
+  ) => await atomizeSlimPublication(await value)));
+
+  const grouped = Map
+    .groupBy(atoms, (pub) => pub.title);
+
+  return [...grouped]
+    .filter(([_title, list]) => list.length > 1)
+    .sort((a, b) => b[1].length - a[1].length);
+};
+
+export default defineRoute(async (_req, ctx) => {
+  const { lang } = ctx.params;
+  const multi = await getMultiplicates();
+
+  return (
+    <Page
+      title={"Multi"}
+      lang={lang}
+      style={{
+        fontSize: "1rem",
+      }}
+    >
+      {[...multi].map(([title, list]) => (
+        <>
+          <details>
+            <summary>
+              {title}
+              <span>{" "}[{list.length}]</span>
+            </summary>
+            <ol>
+              {list.map(({ title, ...rest }) => (
+                <SearchResultItem
+                  document={{ title, ...rest }}
+                  score={1}
+                  lang={lang}
+                  collection={"pub"}
+                />
+              ))}
+            </ol>
+          </details>
+        </>
+      ))}
+    </Page>
+  );
+});

services/pub.ts

@@ -1,12 +1,18 @@
 import { SlimPublication } from "akvaplan_fresh/@interfaces/mod.ts";
 import { isHandleUrl } from "akvaplan_fresh/services/handle.ts";
+import { fetchAndStreamNdjson } from "akvaplan_fresh/streams/ndjson_stream.ts";
 
-export const PUBS_BASE = globalThis.Deno && Deno.env.has("AKVAPLAN_PUBS")
+export const PUBS_BASE = globalThis?.Deno && Deno.env.has("AKVAPLAN_PUBS")
   ? Deno.env.get("AKVAPLAN_PUBS")
   : "https://pubs.deno.dev";
 
 const NVA_API_PROD = "https://api.nva.unit.no";
 
+export const pubs = (limit = -1) =>
+  fetchAndStreamNdjson<{ value: SlimPublication }>(
+    new URL(`/pub?limit=${Number(limit)}`, PUBS_BASE),
+  );
+
 export const NVA_API = globalThis?.Deno && Deno.env.has("NVA_API")
   ? Deno.env.get("NVA_API")
   : NVA_API_PROD;

streams/ndjson_stream.ts

@@ -0,0 +1,27 @@
+import { TextLineStream, toTransformStream } from "jsr:@std/[email protected]";
+export const ndjsonStream = <In>(
+  stream: ReadableStream,
+) =>
+  stream.pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TextLineStream())
+    .pipeThrough<In>(
+      toTransformStream(async function* (src: AsyncIterable<string>) {
+        for await (const chunk of src) {
+          if (chunk.trim().length > 0) {
+            const parsed: In = JSON.parse(chunk);
+            yield parsed;
+          }
+        }
+      }),
+    );
+
+export async function* fetchAndStreamJson<In>(url: URL | string) {
+  const r = await fetch(url);
+  if (r.ok) {
+    for await (const entry of await r.json() as { value: In }[]) {
+      yield entry?.value;
+    }
+  }
+}
+export const fetchAndStreamNdjson = async <In>(url: URL) =>
+  ndjsonStream<In>((await fetch(url))?.body!);