Merge pull request #53 from anzharip/feat/separate-stopwords

feat: separate list of custom stopwords into it's own module
anzharip · Nov 30, 2020 · 6fd0f76 · 6fd0f76
2 parents 1a97fa7 + 4e67f37
commit 6fd0f76
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/src/utility/twitter/custom-stopwords.ts b/src/utility/twitter/custom-stopwords.ts
@@ -0,0 +1,15 @@
+export const customStopwords = [
+  "ga",
+  "gw",
+  "lu",
+  "yg",
+  "bgt",
+  "ya",
+  "tp",
+  "engga",
+  "gak",
+  "yah",
+  "tau",
+  "pas",
+  "udh",
+];
diff --git a/src/utility/twitter/tweet-utilities.ts b/src/utility/twitter/tweet-utilities.ts
@@ -8,6 +8,7 @@ import { analyseSentiment } from "../gcp/sentiment";
 import { translateText } from "../gcp/translate";
 import { logger } from "../logger";
 import { regexTwitterHandle, regexURL } from "../regex";
+import { customStopwords } from "./custom-stopwords";
 
 const token = process.env.TWITTER_BEARER_TOKEN || "";
 const recentSearchURL = "https://api.twitter.com/2/tweets/search/recent";
@@ -85,6 +86,7 @@ export async function generateWordsArray(tweets: {
   return sw.removeStopwords(recentTweetsConcatClean.split(" "), [
     ...sw.en,
     ...sw.id,
+    ...customStopwords,
   ]);
 }