Metaforecast

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: pnpm
       - name: Install dependencies
         run: pnpm install

.github/workflows/metaforecast-image.yaml

@@ -0,0 +1,68 @@
+name: Create and publish a Docker image for Metaforecast
+
+on:
+  push:
+    branches:
+      - master
+
+env:
+  REGISTRY: registry.k8s.quantifieduncertainty.org
+  REGISTRY_USERNAME: quri
+  IMAGE_NAME: metaforecast
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+
+    # Allow image push and artifact attestation
+    permissions:
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,format=long
+            type=ref,event=branch
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # TODO - `turbo prune`
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ops/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built.
+      # It increases supply chain security for people who consume the image.
+      # For more information, see https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.vscode/squiggle.code-workspace.default

@@ -1,22 +1,58 @@
 // Copy this file to squiggle.code-workspace if you want to use multi-root repo and to use Jest VS Code extension.
 {
   "folders": [
+    // root
     {
       "name": "squiggle",
       "path": "..",
     },
+    // packages
     {
-      "name": "vscode-ext",
-      "path": "../packages/vscode-ext",
+      "name": "prettier-plugin",
+      "path": "../packages/prettier-plugin",
+    },
+    {
+      "name": "serializer",
+      "path": "../packages/serializer",
     },
     {
       "name": "squiggle-lang",
       "path": "../packages/squiggle-lang",
     },
+    {
+      "name": "textmate-grammar",
+      "path": "../packages/textmate-grammar",
+    },
+    {
+      "name": "ui",
+      "path": "../packages/ui",
+    },
     {
       "name": "components",
       "path": "../packages/components",
     },
+    // internal packages
+    {
+      "name": "ai",
+      "path": "../internal-packages/ai",
+    },
+    {
+      "name": "configs",
+      "path": "../internal-packages/configs",
+    },
+    {
+      "name": "content",
+      "path": "../internal-packages/content",
+    },
+    {
+      "name": "ops",
+      "path": "../internal-packages/ops",
+    },
+    {
+      "name": "versioned-components",
+      "path": "../internal-packages/versioned-components",
+    },
+    // apps
     {
       "name": "website",
       "path": "../apps/website",
@@ -26,8 +62,8 @@
       "path": "../apps/hub",
     },
     {
-      "name": "content",
-      "path": "../internal-packages/content",
+      "name": "metaforecast",
+      "path": "../apps/metaforecast",
     },
   ],
   "settings": {

apps/metaforecast/.dockerignore

@@ -0,0 +1,4 @@
+/node_modules
+/.git
+/.github
+/ops

apps/metaforecast/.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  "extends": "next/core-web-vitals",
+  "rules": {
+    "next/no-document-import-in-page": "off",
+    "import/no-anonymous-default-export": "off"
+  }
+}

apps/metaforecast/.gitignore

@@ -0,0 +1,25 @@
+# Security
+**/betfaircertificates/
+**/secrets.json
+
+# Build artifacts
+*.swp
+
+# next.js
+/.next/
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+
+# vercel
+.vercel
+
+/.env*
+
+/src/**/*.generated.tsx
+/src/**/*.generated.ts
+/prisma/generated

apps/metaforecast/.prettierignore

@@ -0,0 +1,3 @@
+ops
+src/graphql/introspection.json
+schema.graphql

apps/metaforecast/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "typescript.tsdk": "node_modules/typescript/lib"
+}

apps/metaforecast/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2023 Quantified Uncertainty Research Institute.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

apps/metaforecast/README.md

@@ -0,0 +1,115 @@
+## What this is
+
+[Metaforecast](https://metaforecast.org) is a search engine for probabilities from various prediction markes and forecasting platforms. Try searching "Trump", "China" or "Semiconductors".
+
+This repository includes the source code for both the website and the library that fetches forecasts needed to replace them. We also aim to provide tooling to integrate metaforecast with other services.
+
+[![](./public/screenshot-frontpage.png)](https://metaforecast.org)
+
+## How to run
+
+### 1. Download this repository
+
+```
+$ git clone https://github.com/quantified-uncertainty/squiggle
+$ cd apps/metaforecast
+$ pnpm install
+```
+
+### 2. Set up a database and environment variables
+
+You'll need a PostgreSQL instance, either local (see https://www.postgresql.org/download/) or in the cloud (for example, you can spin one up on https://www.digitalocean.com/products/managed-databases-postgresql or https://supabase.com/).
+
+Environment can be set up with an `.env` file. You'll need to configure at least `DIGITALOCEAN_POSTGRES`.
+
+See [./docs/configuration.md](./docs/configuration.md) for details.
+
+### 3. Actually run
+
+After installing and building (`pnpm run build`) the application, `pnpm run cli` starts a local CLI which presents the user with choices. If you would like to skip that step, use the option name instead, e.g., `pnpm run cli wildeford`.
+
+![](./public/screenshot-cli.png)
+
+`pnpm run dev` starts a Next.js dev server with the website on `http://localhost:3000`.
+
+So overall this would look like
+
+```
+$ git clone https://github.com/quantified-uncertainty/squiggle
+$ cd apps/metaforecast
+$ pnpm install
+$ pnpm run build
+$ pnpm run cli
+$ pnpm run dev
+```
+
+### 4. Example: download the metaforecasts database
+
+```
+$ git clone https://github.com/quantified-uncertainty/squiggle
+$ cd apps/metaforecast
+$ pnpm install
+$ node src/backend/manual/manualDownload.js
+```
+
+## Integrations
+
+Metaforecast has been integrated into:
+
+- Twitter, using our [@metaforecast](https://twitter.com/metaforecast) bot
+- [Global Guessing](https://globalguessing.com/russia-ukraine-forecasts/), which integrates our dashboards
+- [Fletcher](https://fletcher.fun/), a popular Discord bot. You can invoke metaforecast with `!metaforecast search-term`
+- [Elicit](https://elicit.org/), which uses GPT-3 to deliver vastly superior semantic search (as opposed to fuzzy word matching). If you have access to the Elicit IDE, you can use the action "Search Metaforecast database. This is not being updated regularly.
+
+You can use our [GraphQL API](https://metaforecast.org/api/graphql) to build your own integration.
+
+We are also open to integrating our Elasticsearch instance with other trusted services (in addition to Fletcher.)
+
+In general, if you want to integrate metaforecast into your service, we want to hear from you.
+
+## Code layout
+
+- frontend code is in [src/pages/](./src/pages/), [src/web/](./src/web/) and in a few other places which are required by Next.js (e.g. root-level configs in postcss.config.js and tailwind.config.js)
+- various backend code is in [src/backend/](./src/backend/)
+- fetching libraries for various platforms is in [src/backend/platforms/](./src/backend/platforms/)
+- rudimentary documentation is in [docs/](./docs)
+
+## What are "stars" and how are they computed
+
+Star ratings—e.g. ★★★☆☆—are an indicator of the quality of an aggregate forecast for a question. These ratings currently try to reflect my own best judgment and the best judgment of forecasting experts I've asked, based on our collective experience forecasting on these platforms. Thus, stars have a strong subjective component which could be formalized and refined in the future. You can see the code used to decide how many stars a forecast should get by looking at the function `calculateStars()` in the files for every platform [here](./src/backend/platforms).
+
+With regards the quality, I am most uncertain about Smarkets, Hypermind, Ladbrokes and WilliamHill, as I haven't used them as much. Also note that, whatever other redeeming features they might have, prediction markets rarely go above 95% or below 5%.
+
+## Tech stack
+
+Overall, the services which we use are:
+
+- Elasticsearch for search
+- Vercel for website deployment
+- Heroku for background jobs, e.g. fetching new forecasts
+- Postgres on DigitalOcean for database
+
+## Various notes
+
+- This repository is released under the [MIT license](https://opensource.org/licenses/MIT). See `LICENSE.md`
+- Commits follow [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/#summary)
+- For elicit and metaculus, this library currently filters out questions with <10 predictions.
+- The database is updated once a day, at 3:00 AM UTC, with the command `tsx src/backend/flow/doEverythingForScheduler.ts`. The frontpage is updated after that, at 6:00 AM UTC with the command `tsx src/backend/cli/index.ts frontpage`. It's possible that either of these two operations makes the webpage briefly go down.
+
+## To do
+
+- [x] Update Metaculus and Manifold Markets fetchers
+- [x] Add markets from [Insight Prediction](https://insightprediction.com/).
+- [ ] Update broken fetchers:
+  - [x] For Good Judgment
+  - [ ] Kalshi: Requires a US person to create an account to access their v2 api.
+- [ ] Use <https://news.manifold.markets/p/above-the-fold-midterms-special> to update stars calculation for Manifold.
+- [ ] Add a few more snippets, with fetching individual questions, questions with histories, questions added within the last 24h to the /contrib folder (good first issue)
+- [ ] Refactor code so that users can capture and push the question history chart to imgur (good first issue)
+- [ ] Upgrade to [React 18](https://reactjs.org/blog/2022/03/08/react-18-upgrade-guide.html). This will require dealing with the workaround we used for [this issue](https://github.com/vercel/next.js/issues/36019#issuecomment-1103266481)
+- [ ] Add database of resolutions
+- [ ] Allow users to embed predictions in the EA Forum/LessWrong (in progress)
+- [ ] Find a long-term mantainer for this project
+- [ ] Allow users to record their own predictions
+- [ ] Release snapshots (I think @niplav is working on this)
+- [ ] ...

apps/metaforecast/codegen.yml

@@ -0,0 +1,32 @@
+schema: schema.graphql
+documents:
+  - "src/**/*.graphql"
+
+# This should be updated to match your client files
+# documents: 'client/**/!(*.d).{ts,tsx}'
+generates:
+  # This will take your schema and print an SDL schema.
+  schema.graphql:
+    plugins:
+      - schema-ast
+
+  src/graphql/types.generated.ts:
+    plugins:
+      - typescript
+
+  src/graphql/introspection.json:
+    plugins:
+      - introspection:
+          minify: true
+
+  src/:
+    preset: near-operation-file
+    presetConfig:
+      extension: .generated.tsx
+      baseTypesPath: graphql/types.generated.ts
+    plugins:
+      - typescript-operations:
+          strictScalars: true
+          scalars:
+            Date: number
+      - typed-document-node

apps/metaforecast/docs/coding-style.md

@@ -0,0 +1,29 @@
+# TypeScript
+
+- avoid `any`; get rid of any existing `any` whenever you can so that we can enable `"strict": true` later on in `tsconfig.json`
+- define custom types for common data structures
+  - don't worry about `interface` vs `type`, [both are fine](https://www.typescriptlang.org/docs/handbook/2/everyday-types.html#differences-between-type-aliases-and-interfaces)
+
+## Typescript and React/Next
+
+- use `React.FC<Props>` type for React components, e.g. `const MyComponent: React.FC<Props> = ({ ... }) => { ... };`
+- use `NextPage<Props>` for typing stuff in `src/pages/`
+- use generic versions of `GetServerSideProps<Props>` and `GetStaticProps<Props>`
+
+# React
+
+- create one file per one component (tiny helper components in the same file are fine)
+- name file identically to the component it describes (e.g. `const DisplayQuestions: React.FC<Props> = ...` in `DisplayQuestions.ts`)
+- use named export instead of default export for all React components
+  - it's better for refactoring
+  - and it plays well with `React.FC` typing
+
+# Styles
+
+- use [Tailwind](https://tailwindcss.com/)
+- avoid positioning styles in components, position elements from the outside (e.g. with [space-\*](https://tailwindcss.com/docs/space) or grid/flexbox)
+
+# General notes
+
+- use `const` instead of `let` whenever possible
+- set up [prettier](https://prettier.io/) to format code on save