Merge pull request #106 from n4ze3m/next

v1.0.4
n4ze3m · Oct 7, 2023 · 1b8a5e5 · 1b8a5e5
2 parents 4d21331 + d9f2d94
commit 1b8a5e5
Show file tree

Hide file tree

Showing 12 changed files with 320 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -122,6 +122,8 @@ Dialoqbase is a side project and is not ready for production. It is still in the
 - [x] Github repository
 - [x] mp3
 - [x] mp4
+- [X] Sitemap
+- [x] Youtube
 - [ ] Notion
 
 and more...

diff --git a/app/ui/package.json b/app/ui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "app",
   "private": true,
-  "version": "1.0.3",
+  "version": "1.0.4",
   "type": "module",
   "scripts": {
     "dev": "vite",

diff --git a/app/ui/src/components/Bot/Playground/HistoryCard.tsx b/app/ui/src/components/Bot/Playground/HistoryCard.tsx
@@ -29,7 +29,7 @@ export const PlaygroundHistoryCard = ({
       <ChatBubbleLeftIcon className="w-5 h-5 text-gray-400 dark:text-gray-600 group-hover:text-gray-500 dark:group-hover:text-gray-400 transition-colors" />
       <div className="flex-1 overflow-hidden break-all">
         <span
-          className="text-gray-500 dark:text-gray-400 text-sm font-semibold"
+          className="text-gray-500 dark:text-gray-400 text-sm font-semibold truncate"
           title={item.title}
         >
           {item.title.length > 20

diff --git a/app/ui/src/components/Common/BotForm.tsx b/app/ui/src/components/Common/BotForm.tsx
@@ -23,6 +23,7 @@ import { SpiderIcon } from "../Icons/SpiderIcon";
 import { GithubIcon } from "../Icons/GithubIcon";
 import { YoutubeIcon } from "../Icons/YoutubeIcon";
 import { ApiIcon } from "../Icons/ApiIcon";
+import { SitemapIcon } from "../Icons/SitemapIcon";
 
 type Props = {
   createBot: (values: any) => void;
@@ -59,6 +60,7 @@ export const BotForm = ({
               required: true,
               message: "Please enter the webpage URL",
             },
+
           ]}
         >
           <input
@@ -426,6 +428,29 @@ export const BotForm = ({
         </>
       ),
     },
+    {
+      id: 9,
+      value: "sitemap",
+      title: "Sitemap",
+      icon: SitemapIcon,
+      formComponent: (
+        <Form.Item
+          name="content"
+          rules={[
+            {
+              required: true,
+              message: "Please enter the sitemap URL",
+            }
+          ]}
+        >
+          <input
+            type="url"
+            placeholder="Enter the sitemap URL"
+            className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"
+          />
+        </Form.Item>
+      ),
+    },
   ]);
 
   const [selectedSource, _setSelectedSource] = React.useState<any>(
@@ -511,10 +536,7 @@ export const BotForm = ({
       {selectedSource && selectedSource.value === "rest" && (
         <Row gutter={24}>
           <Col span={12}>
-            <Form.Item
-              name={["options", "headers"]}
-              label="Headers"
-            >
+            <Form.Item name={["options", "headers"]} label="Headers">
               <textarea
                 placeholder="Enter the headers"
                 className=" block w-full shadow-sm sm:text-sm focus:ring-indigo-500 focus:border-indigo-500 border-gray-300 rounded-md"

diff --git a/app/ui/src/components/Icons/SitemapIcon.tsx b/app/ui/src/components/Icons/SitemapIcon.tsx
@@ -0,0 +1,24 @@
+import React from "react";
+
+export const SitemapIcon = React.forwardRef<
+  SVGSVGElement,
+  React.SVGProps<SVGSVGElement>
+>((props, ref) => {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      x="0"
+      y="0"
+      enableBackground="new 0 0 512 512"
+      version="1.1"
+      viewBox="0 0 512 512"
+      xmlSpace="preserve"
+      ref={ref}
+      {...props}
+    >
+      <path d="M76.8 332.8v25.6H128c14.1 0 25.6 11.5 25.6 25.6v51.2c0 14.1-11.5 25.6-25.6 25.6H76.8c-14.1 0-25.6-11.5-25.6-25.6V384c0-14.1 11.5-25.6 25.6-25.6v-51.2C34.4 307.2 0 341.6 0 384v51.2C0 477.6 34.4 512 76.8 512H128c42.4 0 76.8-34.4 76.8-76.8V384c0-42.4-34.4-76.8-76.8-76.8H76.8v25.6zM384 332.8v25.6h51.2c14.1 0 25.6 11.5 25.6 25.6v51.2c0 14.1-11.5 25.6-25.6 25.6H384c-14.1 0-25.6-11.5-25.6-25.6V384c0-14.1 11.5-25.6 25.6-25.6v-51.2c-42.4 0-76.8 34.4-76.8 76.8v51.2c0 42.4 34.4 76.8 76.8 76.8h51.2c42.4 0 76.8-34.4 76.8-76.8V384c0-42.4-34.4-76.8-76.8-76.8H384v25.6zM230.4 25.6v25.6h51.2c14.1 0 25.6 11.5 25.6 25.6V128c0 14.1-11.5 25.6-25.6 25.6h-51.2c-14.1 0-25.6-11.5-25.6-25.6V76.8c0-14.1 11.5-25.6 25.6-25.6V0c-42.4 0-76.8 34.4-76.8 76.8V128c0 42.4 34.4 76.8 76.8 76.8h51.2c42.4 0 76.8-34.4 76.8-76.8V76.8C358.4 34.4 324 0 281.6 0h-51.2v25.6z"></path>
+      <path d="M128 332.8v-25.6c0-14.1 11.5-25.6 25.6-25.6h204.8c14.1 0 25.6 11.5 25.6 25.6v25.6c0 14.1 11.5 25.6 25.6 25.6 14.1 0 25.6-11.5 25.6-25.6v-25.6c0-42.4-34.4-76.8-76.8-76.8H153.6c-42.4 0-76.8 34.4-76.8 76.8v25.6c0 14.1 11.5 25.6 25.6 25.6 14.1 0 25.6-11.5 25.6-25.6z"></path>
+      <path d="M230.4 179.2V256c0 14.1 11.5 25.6 25.6 25.6s25.6-11.5 25.6-25.6v-76.8c0-14.1-11.5-25.6-25.6-25.6s-25.6 11.5-25.6 25.6"></path>
+    </svg>
+  );
+});
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dialoqbase",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "Create chatbots with ease",
   "scripts": {
     "ui:dev": "pnpm run --filter ui dev",

diff --git a/server/package.json b/server/package.json
@@ -27,6 +27,7 @@
     "@fastify/jwt": "^7.0.0",
     "@fastify/multipart": "^7.6.0",
     "@fastify/sensible": "^5.0.0",
+    "@fastify/session": "^10.5.0",
     "@fastify/static": "^6.10.2",
     "@ffmpeg.wasm/core-mt": "^0.12.0",
     "@ffmpeg.wasm/main": "^0.12.0",
@@ -64,6 +65,7 @@
     "pdf-parse": "^1.1.1",
     "pdfjs-dist": "^3.7.107",
     "pubsub-js": "^1.9.4",
+    "sitemapper": "^3.2.6",
     "unique-names-generator": "^4.7.1",
     "wavefile": "^11.0.0",
     "ytdl-core": "^4.11.5"

diff --git a/server/src/queue/controllers/sitemap.controller.ts b/server/src/queue/controllers/sitemap.controller.ts
@@ -0,0 +1,62 @@
+import { PrismaClient } from "@prisma/client";
+import { QSource } from "../type";
+import { websiteQueueController } from "./website.controller";
+const prisma = new PrismaClient();
+import Sitemapper from "sitemapper";
+
+export const sitemapQueueController = async (
+  source: QSource,
+) => {
+  const url = source.content!;
+
+  const sitemapper = new Sitemapper({
+    url,
+    timeout: 300000,
+    concurrency: process.env.SITEMAPPER_CONCURRENCY
+      ? parseInt(process.env.SITEMAPPER_CONCURRENCY)
+      : 2,
+    retries: process.env.SITEMAPPER_MAX_RETRIES
+      ? parseInt(process.env.SITEMAPPER_MAX_RETRIES)
+      : 1,
+    requestHeaders: {
+      "User-Agent": process.env.SITEMAPPER_USER_AGENT ||
+        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0'",
+    },
+  });
+
+  const data = await sitemapper.fetch();
+
+  if (data.errors.length > 0) {
+    console.log(data.errors);
+    throw new Error("Sitemapper failed to fetch sitemap");
+  }
+
+  const links = data.sites;
+
+  for (const link of links) {
+    const newSource = await prisma.botSource.create({
+      data: {
+        botId: source.botId,
+        content: link,
+        isPending: true,
+        status: "PENDING",
+        type: "website",
+      },
+    });
+
+    await websiteQueueController({
+      ...newSource,
+      embedding: source.embedding,
+    });
+
+    await prisma.botSource.update({
+      where: {
+        id: newSource.id,
+      },
+      data: {
+        status: "FINISHED",
+        isPending: false,
+      },
+    });
+  }
+};
diff --git a/server/src/queue/index.ts b/server/src/queue/index.ts
@@ -13,6 +13,7 @@ import { audioQueueController } from "./controllers/audio.controller";
 import { videoQueueController } from "./controllers/video.controller";
 import { youtubeQueueController } from "./controllers/youtube.controller";
 import { restQueueController } from "./controllers/rest.controller";
+import { sitemapQueueController } from "./controllers/sitemap.controller";
 
 const prisma = new PrismaClient();
 
@@ -92,6 +93,9 @@ export default async function queueHandler(job: Job, done: DoneCallback) {
           case "rest":
             await restQueueController(source);
             break;
+          case "sitemap":
+            await sitemapQueueController(source);
+            break;
           default:
             break;
         }
@@ -110,7 +114,6 @@ export default async function queueHandler(job: Job, done: DoneCallback) {
         await prisma.$disconnect();
       } catch (e) {
         console.log(e);
-
         await prisma.botSource.update({
           where: {
             id: source.id,
@@ -121,6 +124,7 @@ export default async function queueHandler(job: Job, done: DoneCallback) {
           },
         });
         await prisma.$disconnect();
+        done();
       }
     }
   } catch (e) {

diff --git a/server/src/routes/api/v1/bot/handlers/get.handler.ts b/server/src/routes/api/v1/bot/handlers/get.handler.ts
@@ -59,7 +59,7 @@ export const getBotByIdAllSourcesHandler = async (
     where: {
       botId: id,
       type: {
-        not: "crawl",
+        notIn: ["crawl", "sitemap"],
       },
     },
   });

diff --git a/server/src/utils/datasource.ts b/server/src/utils/datasource.ts
@@ -5,4 +5,5 @@ export const SUPPORTED_SOURCE_TYPES = [
   "github",
   "youtube",
   "rest",
+  "sitemap",
 ];