From 4ffe0cfe96eec0bad23d5c81842e754bc5de1f35 Mon Sep 17 00:00:00 2001 From: Mohammed Taher Ghazal <50504183+MTG2000@users.noreply.github.com> Date: Thu, 23 Nov 2023 14:49:11 +0400 Subject: [PATCH] feat: generate story excerpt using AI service (#279) * feat: generate story excerpt using AI service * deploy: allow empty hygraph creds * chore: remove incorrect default values from hygraph env vars * fix: remove title & desc og overrides in story page --- api/functions/graphql/types/post.js | 10 +++ .../on-queue-callback/on-queue-callback.js | 84 +++++++++++++++++++ .../test-something/test-something.js | 14 +++- api/services/queue-service/ai-service.js | 26 ++++++ api/services/queue-service/index.js | 2 + api/utils/consts/index.js | 2 + package-lock.json | 43 ++-------- serverless.yml | 7 ++ .../StoryPageContent/StoryPageContent.tsx | 6 +- .../pages/PostDetailsPage/PostDetailsPage.tsx | 2 +- .../pages/PostDetailsPage/postDetails.graphql | 3 + src/graphql/index.tsx | 5 +- 12 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 api/functions/on-queue-callback/on-queue-callback.js create mode 100644 api/services/queue-service/ai-service.js diff --git a/api/functions/graphql/types/post.js b/api/functions/graphql/types/post.js index 2fcfa368..66eb614e 100644 --- a/api/functions/graphql/types/post.js +++ b/api/functions/graphql/types/post.js @@ -753,6 +753,16 @@ const createStory = extendType({ author_name: createdStory.user.name, tags, }) + .catch((err) => { + console.log("Error happened while posting to queue service:"); + console.log(err); + }), + queueService.aiService + .generateStoryOgSummary({ + id: createdStory.id, + title: createdStory.title, + body: createdStory.body, + }) .catch((err) => { console.log("Error happened while posting to queue service:"); console.log(err); diff --git a/api/functions/on-queue-callback/on-queue-callback.js b/api/functions/on-queue-callback/on-queue-callback.js new file mode 100644 index 00000000..1aa9f81c --- /dev/null +++ b/api/functions/on-queue-callback/on-queue-callback.js @@ -0,0 +1,84 @@ +const serverless = require("serverless-http"); +const { createExpressApp } = require("../../modules"); +const express = require("express"); +const CONSTS = require("../../utils/consts"); +const { prisma } = require("../../prisma"); +const cacheService = require("../../services/cache.service"); + +const onQueueCallback = async (req, res) => { + const base64Token = Buffer.from( + `${CONSTS.BF_QUEUES_SERVICE_USERNAME}:${CONSTS.BF_QUEUES_SERVICE_PASS}` + ).toString("base64"); + const authToken = req.headers.authorization?.split(" ")[1]; + + if (authToken !== base64Token) + return res.status(401).send("Unauthorized Access"); + + const type = req.body.type; + + const handler = jobHandlers[type]; + + if (!handler) return res.status(400).send("Unknown job type: ", type); + + try { + await handler(req.body); + } catch (error) { + console.log(error); + return res + .status(400) + .send(error.message ?? "An unexpected error happened"); + } + + // Exec job type specific logic + return res.status(200).send("OK"); +}; + +let app; + +if (process.env.LOCAL) { + app = createExpressApp(); + app.post("/on-queue-callback", onQueueCallback); +} else { + const router = express.Router(); + router.post("/on-queue-callback", onQueueCallback); + app = createExpressApp(router); +} + +const handler = serverless(app); +exports.handler = async (event, context) => { + return await handler(event, context); +}; + +const jobHandlers = { + "create-story-root-event": async (data) => { + const { story_id, root_event_id } = data; + if (!story_id || !root_event_id) + throw new Error("story_id or root_event_id are not provided"); + + await Promise.all([ + prisma.story.update({ + where: { id: Number(story_id) }, + data: { + nostr_event_id: root_event_id, + }, + }), + cacheService.invalidateStoryById(story_id), + ]); + }, + + "generate-story-og-summary": async (data) => { + const { story_id, summary } = data; + if (!story_id || !summary) + throw new Error("story_id or summary are not provided"); + + await Promise.all([ + prisma.story.update({ + where: { id: Number(story_id) }, + data: { + excerpt: summary, + }, + }), + cacheService.invalidateStoryById(story_id), + ]); + }, +}; diff --git a/api/functions/test-something/test-something.js b/api/functions/test-something/test-something.js index bb3a6139..04a0d715 100644 --- a/api/functions/test-something/test-something.js +++ b/api/functions/test-something/test-something.js @@ -1,6 +1,7 @@ const serverless = require("serverless-http"); const { createExpressApp } = require("../../modules"); const express = require("express"); +const { queueService } = require("../../services/queue-service"); const testSomething = async (req, res) => { // first, do some validation to make sure the function has been invoked internally @@ -10,8 +11,19 @@ const testSomething = async (req, res) => { // return res.status(401).json({ status: "ERROR", message: "Unauthorized" }); // } - const {} = req.body; + const story = req.body; try { + queueService.aiService + .generateStoryOgSummary({ + id: story.id, + title: story.title, + body: story.body, + }) + .catch((err) => { + console.log("Error happened while posting to queue service:"); + console.log(err); + }); + return res.status(200).json({ status: "OK", message: "Done" }); } catch (error) { console.log(error); diff --git a/api/services/queue-service/ai-service.js b/api/services/queue-service/ai-service.js new file mode 100644 index 00000000..359c340a --- /dev/null +++ b/api/services/queue-service/ai-service.js @@ -0,0 +1,26 @@ +const { marked } = require("marked"); +const env = require("../../utils/consts"); +const { callQueueApi } = require("./helpers"); + +const aiService = { + generateStoryOgSummary: ({ id, title, body }) => { + const htmlBody = marked.parse(body); + + const bodyAsText = htmlBody + .replace(/<[^>]+>/g, "") + .replace(/&/g, "&") + .replace(/'/g, "'") + .replace(/"/g, '"'); + + return callQueueApi("/add-job/ai/generate-story-og-summary", { + story: { + id, + title, + body: bodyAsText, + }, + callback_url: env.FUNCTIONS_URL + "/on-queue-callback", + }); + }, +}; + +module.exports = aiService; diff --git a/api/services/queue-service/index.js b/api/services/queue-service/index.js index a20a1ac9..fdcca527 100644 --- a/api/services/queue-service/index.js +++ b/api/services/queue-service/index.js @@ -1,11 +1,13 @@ const emailService = require("./emails-service"); const nostrService = require("./nostr-service"); const searchIndexService = require("./search-index-service"); +const aiService = require("./ai-service"); const queueService = { nostrService, emailService, searchIndexService, + aiService, }; module.exports = { queueService }; diff --git a/api/utils/consts/index.js b/api/utils/consts/index.js index 7fa4813f..ccf59b00 100644 --- a/api/utils/consts/index.js +++ b/api/utils/consts/index.js @@ -56,6 +56,8 @@ const env = envsafe( devDefault: "http://localhost:3001", }), HYGRAPH_WEBHOOKS_SECRET: str({ + allowEmpty: true, + default: "", devDefault: "SUPER_SECRET", }), }, diff --git a/package-lock.json b/package-lock.json index 706b23ec..30cf59d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16728,27 +16728,13 @@ "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, "node_modules/@types/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==", "dev": true, "dependencies": { "@types/node": "*", - "form-data": "^3.0.0" - } - }, - "node_modules/@types/node-fetch/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" + "form-data": "^4.0.0" } }, "node_modules/@types/normalize-package-data": { @@ -82808,26 +82794,13 @@ "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, "@types/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==", "dev": true, "requires": { "@types/node": "*", - "form-data": "^3.0.0" - }, - "dependencies": { - "form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - } + "form-data": "^4.0.0" } }, "@types/normalize-package-data": { diff --git a/serverless.yml b/serverless.yml index cf387b95..583fbf3b 100644 --- a/serverless.yml +++ b/serverless.yml @@ -122,6 +122,13 @@ functions: path: on-job-success method: post + on-queue-callback: + handler: api/functions/on-queue-callback/on-queue-callback.handler + events: + - http: + path: on-queue-callback + method: post + upload-image-url: handler: api/functions/upload-image-url/upload-image-url.handler events: diff --git a/src/features/Posts/pages/PostDetailsPage/Components/StoryPageContent/StoryPageContent.tsx b/src/features/Posts/pages/PostDetailsPage/Components/StoryPageContent/StoryPageContent.tsx index 1c647c6f..dc225c80 100644 --- a/src/features/Posts/pages/PostDetailsPage/Components/StoryPageContent/StoryPageContent.tsx +++ b/src/features/Posts/pages/PostDetailsPage/Components/StoryPageContent/StoryPageContent.tsx @@ -51,11 +51,7 @@ function StoryPageContent({ story }: Props) { return ( <> - +
- +
{isLargeScreen ? ( diff --git a/src/features/Posts/pages/PostDetailsPage/postDetails.graphql b/src/features/Posts/pages/PostDetailsPage/postDetails.graphql index cfdffc33..5053f635 100644 --- a/src/features/Posts/pages/PostDetailsPage/postDetails.graphql +++ b/src/features/Posts/pages/PostDetailsPage/postDetails.graphql @@ -3,6 +3,7 @@ query PostDetails($id: Int!, $type: POST_TYPE!) { ... on Story { id title + excerpt createdAt author { id @@ -42,6 +43,7 @@ query PostDetails($id: Int!, $type: POST_TYPE!) { ... on Bounty { id title + excerpt createdAt author { id @@ -86,6 +88,7 @@ query PostDetails($id: Int!, $type: POST_TYPE!) { ... on Question { id title + excerpt createdAt author { id diff --git a/src/graphql/index.tsx b/src/graphql/index.tsx index 9a65d142..efd444bf 100644 --- a/src/graphql/index.tsx +++ b/src/graphql/index.tsx @@ -1377,7 +1377,7 @@ export type PostDetailsQueryVariables = Exact<{ }>; -export type PostDetailsQuery = { __typename?: 'Query', getPostById: { __typename?: 'Bounty', id: number, title: string, createdAt: any, body: string, type: string, cover_image: string | null, deadline: string, reward_amount: number, applicants_count: number, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> }, applications: Array<{ __typename?: 'BountyApplication', id: number, date: string, workplan: string, author: { __typename?: 'User', id: number, name: string, avatar: string } }> } | { __typename?: 'Question', id: number, title: string, createdAt: any, body: string, type: string, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> } } | { __typename?: 'Story', id: number, title: string, createdAt: any, body: string, type: string, cover_image: string | null, is_published: boolean | null, nostr_event_id: string | null, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> }, project: { __typename?: 'Project', id: number, title: string, thumbnail_image: string | null, hashtag: string } | null } }; +export type PostDetailsQuery = { __typename?: 'Query', getPostById: { __typename?: 'Bounty', id: number, title: string, excerpt: string, createdAt: any, body: string, type: string, cover_image: string | null, deadline: string, reward_amount: number, applicants_count: number, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> }, applications: Array<{ __typename?: 'BountyApplication', id: number, date: string, workplan: string, author: { __typename?: 'User', id: number, name: string, avatar: string } }> } | { __typename?: 'Question', id: number, title: string, excerpt: string, createdAt: any, body: string, type: string, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> } } | { __typename?: 'Story', id: number, title: string, excerpt: string, createdAt: any, body: string, type: string, cover_image: string | null, is_published: boolean | null, nostr_event_id: string | null, author: { __typename?: 'User', id: number, name: string, avatar: string, join_date: any, primary_nostr_key: string | null }, tags: Array<{ __typename?: 'Tag', id: number, title: string }>, votes: { __typename?: 'Votes', total: number, total_anonymous_votes: number, voters: Array<{ __typename?: 'Voter', amount_voted: number, user: { __typename?: 'User', id: number, name: string, avatar: string } }> }, project: { __typename?: 'Project', id: number, title: string, thumbnail_image: string | null, hashtag: string } | null } }; export type GetTagInfoQueryVariables = Exact<{ tag: InputMaybe; @@ -2629,6 +2629,7 @@ export const PostDetailsDocument = gql` ... on Story { id title + excerpt createdAt author { id @@ -2668,6 +2669,7 @@ export const PostDetailsDocument = gql` ... on Bounty { id title + excerpt createdAt author { id @@ -2712,6 +2714,7 @@ export const PostDetailsDocument = gql` ... on Question { id title + excerpt createdAt author { id