chore: update contributing guidelines, improve environment setup, and refactor search functionality

CONTRIBUTING.md

@@ -12,13 +12,13 @@ First, start by installing dependencies:
 2. pnpm [instructions](https://pnpm.io/installation)
 3. redis [instructions](https://redis.io/docs/latest/operate/oss_and_stack/install/install-redis/)
 
-Set environment variables in a .env in the /apps/api/ directory you can copy over the template in .env.example.
+Set environment variables in a .env in the `apps/api/` directory you can copy over the template in .env.example.
 
 To start, we wont set up authentication, or any optional sub services (pdf parsing, JS blocking support, AI features )
 
 .env:
 
-```
+```bash
 # ===== Required ENVS ======
 NUM_WORKERS_PER_QUEUE=8
 PORT=3002
@@ -46,16 +46,14 @@ LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse
 SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messages
 POSTHOG_API_KEY= # set if you'd like to send posthog events like job logs
 POSTHOG_HOST= # set if you'd like to send posthog events like job logs
-
-
 ```
 
 ### Installing dependencies
 
 First, install the dependencies using pnpm.
 
 ```bash
-# cd apps/api # to make sure you're in the right folder
+# cd apps/api/ # to make sure you're in the right folder
 pnpm install # make sure you have pnpm version 9+!
 ```
 
@@ -73,7 +71,7 @@ redis-server
 
 ### Terminal 2 - setting up workers
 
-Now, navigate to the apps/api/ directory and run:
+Now, navigate to the `apps/api/` directory and run:
 
 ```bash
 pnpm run workers
@@ -84,7 +82,7 @@ This will start the workers who are responsible for processing crawl jobs.
 
 ### Terminal 3 - setting up the main server
 
-To do this, navigate to the apps/api/ directory and run if you don’t have this already, install pnpm here: https://pnpm.io/installation
+To do this, navigate to the `apps/api/` directory and run if you don’t have this already, install pnpm here: https://pnpm.io/installation
 Next, run your server with:
 
 ```bash
@@ -95,15 +93,15 @@ pnpm run start
 
 Alright: now let’s send our first request.
 
-```curl
+```bash
 curl -X GET http://localhost:3002/test
 ```
 
 This should return the response Hello, world!
 
 If you’d like to test the crawl endpoint, you can run this
 
-```curl
+```bash
 curl -X POST http://localhost:3002/v1/crawl \
     -H 'Content-Type: application/json' \
     -d '{
@@ -113,6 +111,6 @@ curl -X POST http://localhost:3002/v1/crawl \
 
 ## Tests:
 
-The best way to do this is run the test with `npm run test:local-no-auth` if you'd like to run the tests without authentication.
+The best way to do this is run the test with `pnpm run test:local-no-auth` if you'd like to run the tests without authentication.
 
-If you'd like to run the tests with authentication, run `npm run test:prod`
+If you'd like to run the tests with authentication, run `pnpm run test:prod`

apps/api/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "nodemon --exec ts-node src/index.ts",
     "start:production": "tsc && node dist/src/index.js",
-    "format": "prettier --write \"src/**/*.(js|ts)\"",
+    "format": "npx prettier --write \"src/**/*.(js|ts)\"",
     "flyio": "node dist/src/index.js",
     "start:dev": "nodemon --exec ts-node src/index.ts",
     "build": "tsc && pnpm sentry:sourcemaps",

apps/api/src/control.ts

@@ -1,2 +1,2 @@
 // ! IN CASE OPENAI goes down, then activate the fallback -> true
-export const is_fallback = false;
+export const IS_FALLBACK = false;

apps/api/src/controllers/v0/search.ts

@@ -6,22 +6,17 @@ import {
 import { authenticateUser } from "../auth";
 import { PlanType, RateLimiterMode } from "../../types";
 import { logJob } from "../../services/logging/log_job";
-import { PageOptions, SearchOptions } from "../../lib/entities";
+import { PageOptions } from "../../lib/entities";
+import { SearchOptions } from "../../search/types";
 import { search } from "../../search";
 import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
 import { v4 as uuidv4 } from "uuid";
 import { logger } from "../../lib/logger";
 import { getScrapeQueue } from "../../services/queue-service";
 import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
 import * as Sentry from "@sentry/node";
-import { getJobPriority } from "../../lib/job-priority";
-import { Job } from "bullmq";
-import {
-  Document,
-  fromLegacyCombo,
-  fromLegacyScrapeOptions,
-  toLegacyDocument,
-} from "../v1/types";
+import { getJobPriority, BASE_TEAM_ID } from "../../lib/job-priority";
+import { Document, fromLegacyCombo, toLegacyDocument } from "../v1/types";
 
 export async function searchHelper(
   jobId: string,
@@ -39,24 +34,17 @@ export async function searchHelper(
   returnCode: number;
 }> {
   const query = req.body.query;
-  const advanced = false;
   if (!query) {
     return { success: false, error: "Query is required", returnCode: 400 };
   }
 
   const tbs = searchOptions.tbs ?? undefined;
   const filter = searchOptions.filter ?? undefined;
-  let num_results = Math.min(searchOptions.limit ?? 7, 10);
-
-  if (team_id === "d97c4ceb-290b-4957-8432-2b2a02727d95") {
-    num_results = 1;
-  }
-
+  const num_results = team_id === BASE_TEAM_ID ? 1 : 10;
   const num_results_buffer = Math.floor(num_results * 1.5);
 
   let res = await search({
     query: query,
-    advanced: advanced,
     num_results: num_results_buffer,
     tbs: tbs,
     filter: filter,

apps/api/src/lib/entities.ts

@@ -170,7 +170,7 @@ export class Document {
   }
 }
 
-export class SearchResult {
+export class SearchResult implements SearchResult {
   url: string;
   title: string;
   description: string;

apps/api/src/lib/job-priority.ts

@@ -28,6 +28,8 @@ export async function deleteJobPriority(team_id, job_id) {
   }
 }
 
+export const BASE_TEAM_ID = "d97c4ceb-290b-4957-8432-2b2a02727d95";
+
 export async function getJobPriority({
   plan,
   team_id,
@@ -37,7 +39,7 @@ export async function getJobPriority({
   team_id: string;
   basePriority?: number;
 }): Promise<number> {
-  if (team_id === "d97c4ceb-290b-4957-8432-2b2a02727d95") {
+  if (team_id === BASE_TEAM_ID) {
     return 50;
   }
 

apps/api/src/lib/withAuth.ts

@@ -1,6 +1,4 @@
-import { AuthResponse } from "../../src/types";
 import { logger } from "./logger";
-import * as Sentry from "@sentry/node";
 import { configDotenv } from "dotenv";
 configDotenv();