update md5 to cyrb, add stats to homepage

dev/hash.mts

@@ -0,0 +1,22 @@
+// Benchmark hashing algorithms
+// import { xxHash32 } from 'js-xxhash';
+import nodeCrypto from 'node:crypto';
+import { cyrb53 } from '../server/hash.ts';
+
+const test = 'test'.repeat(100000);
+
+// Note: due to VM optimization the later functions run faster
+// Need to execute in separate processes for accurate testing
+const jobs = {
+    cyrb: () => cyrb53(test).toString(16),
+    // xxhash: () => xxHash32(test).toString(16),
+    // md5js: () => MD5.hash(test),
+    // Works only in node
+    md5node: () => nodeCrypto.createHash('md5').update(test).digest('hex'),
+    // requires https in browser, also Buffer API to convert not available
+    sha1: async () => Buffer.from(await crypto.subtle.digest('sha-1', Buffer.from(test))).toString('hex'),
+}
+
+console.time(process.argv[2]);
+console.log(await jobs[process.argv[2]]());
+console.timeEnd(process.argv[2]);

dev/run_hash.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+tsx hash.mts cyrb
+tsx hash.mts xxhash
+tsx hash.mts md5js
+tsx hash.mts md5node
+tsx hash.mts sha1

server/aivoice.ts

@@ -1,4 +1,4 @@
-import nodeCrypto from 'node:crypto';
+import { cyrb53 } from './hash';
 
 // Given input text, gets back an mp3 file URL
 // We can send this to each client and have it be read
@@ -43,7 +43,7 @@ export async function genAITextToSpeech(
         0,
         44100,
         'mp3',
-        nodeCrypto.createHash('md5').update(text).digest('hex'),
+        cyrb53(text).toString(),
       ],
     }),
   });

server/jData.ts

@@ -0,0 +1,62 @@
+import { gunzipSync } from 'zlib';
+import fs from 'fs';
+import config from './config';
+
+let qs = 0;
+let eps = 0;
+// On boot, start with the initial data included in repo
+console.time('load');
+let jData = JSON.parse(gunzipSync(fs.readFileSync('./jeopardy.json.gz')).toString());
+updateJDataStats();
+console.timeEnd('load');
+console.log('loaded %d episodes', Object.keys(jData).length);
+let etag: string | null = null;
+
+// Periodically refetch the latest episode data and replace it in memory
+setInterval(refreshEpisodes, 24 * 60 * 60 * 1000);
+refreshEpisodes();
+
+async function refreshEpisodes() {
+  if (config.NODE_ENV === 'development') {
+    return;
+  }
+  console.time('reload');
+  try {
+    const response = await fetch(
+      'https://github.com/howardchung/j-archive-parser/raw/release/jeopardy.json.gz',
+    );
+    const newEtag = response.headers.get('etag');
+    console.log(newEtag, etag);
+    if (newEtag !== etag) {
+      const arrayBuf = await response.arrayBuffer();
+      const buf = Buffer.from(arrayBuf);
+      jData = JSON.parse(gunzipSync(buf).toString());
+      updateJDataStats();
+      etag = newEtag;
+    }
+  } catch (e) {
+    console.log(e);
+  }
+  console.timeEnd('reload');
+}
+
+function updateJDataStats() {
+  console.time('count');
+  qs = 0;
+  eps = 0;
+  Object.keys(jData).forEach(key => {
+    eps += 1;
+    qs += jData[key].jeopardy.length;
+    qs += jData[key].double.length;
+    qs + jData[key].final.length;
+  });
+  console.timeEnd('count');
+}
+
+export function getJData() {
+  return jData;
+}
+
+export function getJDataStats() {
+  return { qs, eps };
+}

server/room.ts

@@ -1,51 +1,11 @@
 import { Socket, Server } from 'socket.io';
 import Papa from 'papaparse';
-import { gunzipSync } from 'zlib';
 import { redis, redisCount } from './redis';
-import fs from 'fs';
-import nodeCrypto from 'node:crypto';
 import { genAITextToSpeech } from './aivoice';
 import { getOpenAIDecision, openai } from './openai';
 import config from './config';
 import { getGameState, getPerQuestionState } from './gamestate';
-
-// On boot, start with the initial data included in repo
-console.time('load');
-let fileData: Buffer | undefined = fs.readFileSync('./jeopardy.json.gz');
-let jData = JSON.parse(gunzipSync(fileData).toString());
-let hash = nodeCrypto.createHash('md5').update(fileData).digest('hex');
-fileData = undefined;
-console.timeEnd('load');
-console.log('loaded %d episodes', Object.keys(jData).length);
-
-async function refreshEpisodes() {
-  if (config.NODE_ENV === 'development') {
-    return;
-  }
-  console.time('reload');
-  try {
-    const response = await fetch(
-      'https://github.com/howardchung/j-archive-parser/raw/release/jeopardy.json.gz',
-    );
-    const arrayBuf = await response.arrayBuffer();
-    const buf = Buffer.from(arrayBuf);
-    const newHash = nodeCrypto.createHash('md5').update(buf).digest('hex');
-    // Check if new compressed data matches current
-    if (newHash !== hash) {
-      jData = JSON.parse(gunzipSync(buf).toString());
-      hash = newHash;
-      console.log('reloaded %d episodes', Object.keys(jData).length);
-    } else {
-      console.log('skipping reload since data is the same');
-    }
-  } catch (e) {
-    console.log(e);
-  }
-  console.timeEnd('reload');
-}
-// Periodically refetch the latest episode data and replace it in memory
-setInterval(refreshEpisodes, 24 * 60 * 60 * 1000);
-refreshEpisodes();
+import { getJData } from './jData';
 
 export class Room {
   // Serialized state
@@ -572,6 +532,7 @@ export class Room {
         console.warn(e);
       }
     } else {
+      const jData = getJData();
       // Load question data into game
       let nums = Object.keys(jData);
       if (filter) {

server/server.ts

@@ -11,6 +11,7 @@ import { Room } from './room';
 import { redis, getRedisCountDay } from './redis';
 import { makeRoomName, makeUserName } from './moniker';
 import config from './config';
+import { getJDataStats } from './jData';
 
 const app = express();
 let server = null as https.Server | http.Server | null;
@@ -55,6 +56,10 @@ app.get('/ping', (req, res) => {
   res.json('pong');
 });
 
+app.get('/metadata', (req, res) => {
+  res.json(getJDataStats());
+});
+
 app.get('/stats', async (req, res) => {
   if (req.query.key && req.query.key === config.STATS_KEY) {
     const roomData: any[] = [];

src/components/Home/Home.tsx

@@ -1,8 +1,10 @@
-import React from 'react';
+import React, { useEffect, useState } from 'react';
 import { Divider, Header, Icon } from 'semantic-ui-react';
+import CountUp from 'react-countup';
 
 import { NewRoomButton, JeopardyTopBar } from '../TopBar/TopBar';
 import styles from './Home.module.css';
+import { serverPath } from '../../utils';
 
 const Feature = ({
   icon,
@@ -33,23 +35,40 @@ const Feature = ({
 
 const Hero = ({
   heroText,
-  subText,
   action,
   image,
   color,
 }: {
   heroText?: string;
-  subText?: string;
   action?: React.ReactNode;
   image?: string;
   color?: string;
 }) => {
+  const [epCount, setEpCount] = useState(8000);
+  const [qCount, setQCount] = useState(500000);
+  useEffect(() => {
+    const update = async () => {
+      const response = await fetch(serverPath + '/metadata');
+      const json = await response.json();
+      setQCount(json.qs);
+      setEpCount(json.eps);
+    }
+    update();
+  }, []);
   return (
     <div className={`${styles.hero} ${color === 'green' ? styles.green : ''}`}>
       <div className={styles.heroInner}>
         <div style={{ padding: '30px', flex: '1 1 0' }}>
           <div className={styles.heroText}>{heroText}</div>
-          <div className={styles.subText}>{subText}</div>
+          <div className={styles.subText}>
+            <CountUp start={8000} end={epCount} delay={0} duration={3} />
+            {' '}
+            episodes featuring
+            {' '}
+            <CountUp start={500000} end={qCount} delay={0} duration={3} />
+            {' '}
+            clues
+          </div>
           {action}
         </div>
         <div
@@ -75,7 +94,6 @@ export const JeopardyHome = () => {
       <div className={styles.container}>
         <Hero
           heroText={'Play Jeopardy! online with friends.'}
-          subText={'Pick from 7,000+ episodes featuring 400,000+ clues.'}
           action={<NewRoomButton />}
           image={'/screenshot3.png'}
         />

src/components/Jeopardy/Jeopardy.tsx

@@ -30,7 +30,7 @@ import { io, type Socket } from 'socket.io-client';
 import { type AppState } from '../App/App';
 import ReactMarkdown from 'react-markdown';
 import { type PublicGameState } from '../../../server/gamestate';
-import { MD5 } from '../../md5';
+import { cyrb53 } from '../../../server/hash';
 
 const dailyDouble = new Audio('/jeopardy/jeopardy-daily-double.mp3');
 const boardFill = new Audio('/jeopardy/jeopardy-board-fill.mp3');
@@ -386,7 +386,7 @@ export class Jeopardy extends React.Component<{
     if (this.state.game?.enableAIVoices) {
       try {
         await new Promise(async (resolve, reject) => {
-          const hash = MD5.hash(text);
+          const hash = cyrb53(text).toString();
           const aiVoice = new Audio(
             this.state.game?.enableAIVoices +
               '/gradio_api/file=audio/output/{hash}.mp3'.replace(