From ed59e2b869503312ff1035adf48c8f5629c38340 Mon Sep 17 00:00:00 2001 From: Rjnishant530 Date: Tue, 7 Jan 2025 15:48:18 +0530 Subject: [PATCH 1/2] init --- lib/routes/30secondsofcode/utils.ts | 1 - lib/routes/css-tricks/articles.ts | 48 ++++++++++++++ lib/routes/css-tricks/fresh.ts | 50 +++++++++++++++ lib/routes/css-tricks/guides.ts | 39 ++++++++++++ lib/routes/css-tricks/namespace.ts | 8 +++ lib/routes/css-tricks/popular.ts | 50 +++++++++++++++ lib/routes/css-tricks/utils.ts | 98 +++++++++++++++++++++++++++++ 7 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 lib/routes/css-tricks/articles.ts create mode 100644 lib/routes/css-tricks/fresh.ts create mode 100644 lib/routes/css-tricks/guides.ts create mode 100644 lib/routes/css-tricks/namespace.ts create mode 100644 lib/routes/css-tricks/popular.ts create mode 100644 lib/routes/css-tricks/utils.ts diff --git a/lib/routes/30secondsofcode/utils.ts b/lib/routes/30secondsofcode/utils.ts index 9ffbab8b5045b9..c3e88d9705366b 100644 --- a/lib/routes/30secondsofcode/utils.ts +++ b/lib/routes/30secondsofcode/utils.ts @@ -48,7 +48,6 @@ async function processItem({ link: articleLink, date }) { category: tags, image: `${rootUrl}${image}`, banner: `${rootUrl}${image}`, - language: 'en-us', } as DataItem; }); } diff --git a/lib/routes/css-tricks/articles.ts b/lib/routes/css-tricks/articles.ts new file mode 100644 index 00000000000000..5e4b7f956e6026 --- /dev/null +++ b/lib/routes/css-tricks/articles.ts @@ -0,0 +1,48 @@ +import { Data, Route, ViewType } from '@/types'; +import { load } from 'cheerio'; +import ofetch from '@/utils/ofetch'; +import { processCards } from './utils'; +export const route: Route = { + path: '/articles', + view: ViewType.Articles, + categories: ['programming'], + example: '/articles', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['css-tricks.com/category/articles/'], + target: '/articles', + }, + ], + name: 'Articles', + maintainers: ['Rjnishant530'], + handler, +}; + +async function handler() { + // const category = ctx.req.param('category') ?? ''; + // const subCategory = ctx.req.param('subCategory') ?? ''; + + const rootUrl = 'https://css-tricks.com'; + const currentUrl = `${rootUrl}/category/articles/`; + const response = await ofetch(currentUrl); + const $ = load(response); + const articleCards = $('article.article-card').toArray(); + const items = await processCards(articleCards); + return { + title: 'Articles - CSS-Tricks', + description: 'Latest Articles - CSS-Tricks', + link: currentUrl, + item: items, + language: 'en', + logo: `${rootUrl}/favicon.ico`, + icon: `${rootUrl}/favicon.ico`, + } as Data; +} diff --git a/lib/routes/css-tricks/fresh.ts b/lib/routes/css-tricks/fresh.ts new file mode 100644 index 00000000000000..e26228f15deee1 --- /dev/null +++ b/lib/routes/css-tricks/fresh.ts @@ -0,0 +1,50 @@ +import { Data, Route, ViewType } from '@/types'; +import { extractMiniCards, processCards, rootUrl } from './utils'; +export const route: Route = { + path: '/fresh/:dateSort?', + view: ViewType.Articles, + categories: ['programming'], + example: '/fresh', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + parameters: { + dateSort: { + description: 'Sort posts by publication date instead of popularity', + default: 'true', + options: [ + { value: 'false', label: 'False' }, + { value: 'true', label: 'True' }, + ], + }, + }, + radar: [ + { + source: ['css-tricks.com'], + target: '/fresh', + }, + ], + name: 'Fresh From the Almanac', + maintainers: ['Rjnishant530'], + handler, +}; + +async function handler(ctx) { + const dateSort = ctx.req.param('dateSort') ? JSON.parse(ctx.req.param('dateSort')) : true; + const popularCards = await extractMiniCards('body > div.page-wrap > section.post-sliders > div:nth-child(4) article.mini-card.module.module-article'); + const items = await processCards(popularCards, true, dateSort); + return { + title: 'Fresh From the Almanac', + description: 'Properties, selectors, rules, and functions!', + link: rootUrl, + item: items, + language: 'en', + logo: `${rootUrl}/favicon.ico`, + icon: `${rootUrl}/favicon.ico`, + } as Data; +} diff --git a/lib/routes/css-tricks/guides.ts b/lib/routes/css-tricks/guides.ts new file mode 100644 index 00000000000000..9b85c63f25a6a2 --- /dev/null +++ b/lib/routes/css-tricks/guides.ts @@ -0,0 +1,39 @@ +import { Data, Route, ViewType } from '@/types'; +import { extractMiniCards, processCards, rootUrl } from './utils'; +export const route: Route = { + path: '/guides', + view: ViewType.Articles, + categories: ['programming'], + example: '/guides', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['css-tricks.com'], + target: '/guides', + }, + ], + name: 'CSS Guides', + maintainers: ['Rjnishant530'], + handler, +}; + +async function handler() { + const guideCards = await extractMiniCards('body > div.page-wrap > section.post-sliders > div:nth-child(3) article.mini-card.module.module-article'); + const items = await processCards(guideCards, true); + return { + title: 'Latest CSS Guides', + description: 'Dive deep into CSS features and concepts', + link: rootUrl, + item: items, + language: 'en', + logo: `${rootUrl}/favicon.ico`, + icon: `${rootUrl}/favicon.ico`, + } as Data; +} diff --git a/lib/routes/css-tricks/namespace.ts b/lib/routes/css-tricks/namespace.ts new file mode 100644 index 00000000000000..248fe9ee439632 --- /dev/null +++ b/lib/routes/css-tricks/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'CSS-Tricks', + url: 'css-tricks.com', + categories: ['programming'], + lang: 'en', +}; diff --git a/lib/routes/css-tricks/popular.ts b/lib/routes/css-tricks/popular.ts new file mode 100644 index 00000000000000..940507d3ad2cb3 --- /dev/null +++ b/lib/routes/css-tricks/popular.ts @@ -0,0 +1,50 @@ +import { Data, Route, ViewType } from '@/types'; +import { extractMiniCards, processCards, rootUrl } from './utils'; +export const route: Route = { + path: '/popular/:dateSort?', + view: ViewType.Articles, + categories: ['programming'], + example: '/popular', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + parameters: { + dateSort: { + description: 'Sort posts by publication date instead of popularity', + default: 'true', + options: [ + { value: 'false', label: 'False' }, + { value: 'true', label: 'True' }, + ], + }, + }, + radar: [ + { + source: ['css-tricks.com'], + target: '/popular', + }, + ], + name: 'Popular this month', + maintainers: ['Rjnishant530'], + handler, +}; + +async function handler(ctx) { + const dateSort = ctx.req.param('dateSort') ? JSON.parse(ctx.req.param('dateSort')) : true; + const popularCards = await extractMiniCards('div.popular-articles > div.mini-card-grid article.mini-card.module.module-article'); + const items = await processCards(popularCards, true, dateSort); + return { + title: 'Popular this month', + description: 'Popular CSS articles this month', + link: rootUrl, + item: items, + language: 'en', + logo: `${rootUrl}/favicon.ico`, + icon: `${rootUrl}/favicon.ico`, + } as Data; +} diff --git a/lib/routes/css-tricks/utils.ts b/lib/routes/css-tricks/utils.ts new file mode 100644 index 00000000000000..4e8b601b80cc9b --- /dev/null +++ b/lib/routes/css-tricks/utils.ts @@ -0,0 +1,98 @@ +import { load } from 'cheerio'; +import ofetch from '@/utils/ofetch'; +import { DataItem } from '@/types'; +import { parseDate } from '@/utils/parse-date'; +import cache from '@/utils/cache'; + +export const rootUrl = 'https://css-tricks.com'; +type Card = { + id: string; + title: string; + link: string; + thumbnail: string; +}; + +export async function extractMiniCards(selector) { + const response = await ofetch(rootUrl); + const $ = load(response); + return $(selector).toArray(); +} + +function extractCardsInfo(cards) { + return cards.map((card) => { + const $ = load(card); + const id = $(card).attr('id'); + const thumbnail = $(card).find('div.article-thumbnail-wrap > a >img').attr('src'); + const article = $('div.article-article'); + const title = article.find('h2 > a').text(); + const link = article.find('h2 > a').attr('href'); + return { + id, + title, + link, + thumbnail, + } as Card; + }); +} + +function extractMiniCardsInfo(cards) { + return cards.map((card) => { + const $ = load(card); + const id = $(card).attr('id')?.replace('mini-', ''); + const thumbnail = ''; + const title = $('h3.mini-card-title').find('a:not(.aal_anchor)').text(); + const link = $('h3.mini-card-title').find('a:not(.aal_anchor)').attr('href'); + return { + id, + title, + link, + thumbnail, + } as Card; + }); +} + +export async function processCards(cards, mini: boolean = false, dateSort: boolean = true) { + const cardsWithInfo = mini ? extractMiniCardsInfo(cards) : extractCardsInfo(cards); + const cardsPromise = await Promise.allSettled( + cardsWithInfo.map(async (card: Card) => await fetchCardDetails(card, dateSort)) + ); + return cardsPromise.filter((card) => card.status === 'fulfilled').map((card) => card.value as DataItem); +} + +export async function fetchCardDetails(card: Card, dateSort: boolean) { + return await cache.tryGet(`css-tricks:${card.id}`, async () => { + const response = await ofetch(card.link); + const $ = load(response); + const tags = $('meta[property="article:tag"]') + ?.toArray() + .map((tag) => $(tag).attr('content')); + const date = $('meta[property="article:published_time"]').attr('content') || ''; + const updateDate = $('meta[property="article:modified_time"]').attr('content') || ''; + const summary = $('meta[property="description"]').attr('content') || ''; + const authorUrl = $('header.mega-header').find('div.author-row > a').attr('href'); + const authorAvatar = $('header.mega-header').find('header.mega-header div.author-row > a >img').attr('src'); + const authorName = $('header.mega-header').find('header.mega-header div.author-row > div > a.author-name').text(); + const content = $('div.article-content').html() || ''; + return { + title: card.title, + link: card.link, + description: content, + banner: card.thumbnail, + image: card.thumbnail, + pubDate: dateSort ? parseDate(date) : '', + updated: dateSort ? parseDate(updateDate) : '', + author: [ + { + name: authorName || '', + url: authorUrl || '', + avatar: authorAvatar || '', + }, + ], + content: { + html: content, + text: summary, + }, + category: tags, + } as DataItem; + }); +} From 08cb0a4f48ac8ec0e9d4b3a400d0585a62dc97e4 Mon Sep 17 00:00:00 2001 From: Rjnishant530 Date: Sat, 18 Jan 2025 11:42:05 +0530 Subject: [PATCH 2/2] use wordpress api --- lib/routes/css-tricks/articles.ts | 4 +-- lib/routes/css-tricks/fresh.ts | 18 +++------- lib/routes/css-tricks/guides.ts | 4 +-- lib/routes/css-tricks/popular.ts | 19 +++-------- lib/routes/css-tricks/utils.ts | 57 +++++++++++++++++++++++++++---- 5 files changed, 62 insertions(+), 40 deletions(-) diff --git a/lib/routes/css-tricks/articles.ts b/lib/routes/css-tricks/articles.ts index 5e4b7f956e6026..f40c93c2569a76 100644 --- a/lib/routes/css-tricks/articles.ts +++ b/lib/routes/css-tricks/articles.ts @@ -1,7 +1,7 @@ import { Data, Route, ViewType } from '@/types'; import { load } from 'cheerio'; import ofetch from '@/utils/ofetch'; -import { processCards } from './utils'; +import { processWithWp } from './utils'; export const route: Route = { path: '/articles', view: ViewType.Articles, @@ -35,7 +35,7 @@ async function handler() { const response = await ofetch(currentUrl); const $ = load(response); const articleCards = $('article.article-card').toArray(); - const items = await processCards(articleCards); + const items = await processWithWp(articleCards); return { title: 'Articles - CSS-Tricks', description: 'Latest Articles - CSS-Tricks', diff --git a/lib/routes/css-tricks/fresh.ts b/lib/routes/css-tricks/fresh.ts index e26228f15deee1..4ddf7786b02f6e 100644 --- a/lib/routes/css-tricks/fresh.ts +++ b/lib/routes/css-tricks/fresh.ts @@ -1,7 +1,7 @@ import { Data, Route, ViewType } from '@/types'; import { extractMiniCards, processCards, rootUrl } from './utils'; export const route: Route = { - path: '/fresh/:dateSort?', + path: '/fresh', view: ViewType.Articles, categories: ['programming'], example: '/fresh', @@ -13,16 +13,6 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, - parameters: { - dateSort: { - description: 'Sort posts by publication date instead of popularity', - default: 'true', - options: [ - { value: 'false', label: 'False' }, - { value: 'true', label: 'True' }, - ], - }, - }, radar: [ { source: ['css-tricks.com'], @@ -34,10 +24,10 @@ export const route: Route = { handler, }; -async function handler(ctx) { - const dateSort = ctx.req.param('dateSort') ? JSON.parse(ctx.req.param('dateSort')) : true; +async function handler() { const popularCards = await extractMiniCards('body > div.page-wrap > section.post-sliders > div:nth-child(4) article.mini-card.module.module-article'); - const items = await processCards(popularCards, true, dateSort); + // Can't use wordPress API, these post Id's aren't available in the response + const items = await processCards(popularCards, true); return { title: 'Fresh From the Almanac', description: 'Properties, selectors, rules, and functions!', diff --git a/lib/routes/css-tricks/guides.ts b/lib/routes/css-tricks/guides.ts index 9b85c63f25a6a2..79a85c350e3d26 100644 --- a/lib/routes/css-tricks/guides.ts +++ b/lib/routes/css-tricks/guides.ts @@ -1,5 +1,5 @@ import { Data, Route, ViewType } from '@/types'; -import { extractMiniCards, processCards, rootUrl } from './utils'; +import { extractMiniCards, processWithWp, rootUrl } from './utils'; export const route: Route = { path: '/guides', view: ViewType.Articles, @@ -26,7 +26,7 @@ export const route: Route = { async function handler() { const guideCards = await extractMiniCards('body > div.page-wrap > section.post-sliders > div:nth-child(3) article.mini-card.module.module-article'); - const items = await processCards(guideCards, true); + const items = await processWithWp(guideCards, true); return { title: 'Latest CSS Guides', description: 'Dive deep into CSS features and concepts', diff --git a/lib/routes/css-tricks/popular.ts b/lib/routes/css-tricks/popular.ts index 940507d3ad2cb3..af4209872561cb 100644 --- a/lib/routes/css-tricks/popular.ts +++ b/lib/routes/css-tricks/popular.ts @@ -1,7 +1,7 @@ import { Data, Route, ViewType } from '@/types'; -import { extractMiniCards, processCards, rootUrl } from './utils'; +import { extractMiniCards, processWithWp, rootUrl } from './utils'; export const route: Route = { - path: '/popular/:dateSort?', + path: '/popular', view: ViewType.Articles, categories: ['programming'], example: '/popular', @@ -13,16 +13,6 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, - parameters: { - dateSort: { - description: 'Sort posts by publication date instead of popularity', - default: 'true', - options: [ - { value: 'false', label: 'False' }, - { value: 'true', label: 'True' }, - ], - }, - }, radar: [ { source: ['css-tricks.com'], @@ -34,10 +24,9 @@ export const route: Route = { handler, }; -async function handler(ctx) { - const dateSort = ctx.req.param('dateSort') ? JSON.parse(ctx.req.param('dateSort')) : true; +async function handler() { const popularCards = await extractMiniCards('div.popular-articles > div.mini-card-grid article.mini-card.module.module-article'); - const items = await processCards(popularCards, true, dateSort); + const items = await processWithWp(popularCards, true); return { title: 'Popular this month', description: 'Popular CSS articles this month', diff --git a/lib/routes/css-tricks/utils.ts b/lib/routes/css-tricks/utils.ts index 4e8b601b80cc9b..22f06442bc40d8 100644 --- a/lib/routes/css-tricks/utils.ts +++ b/lib/routes/css-tricks/utils.ts @@ -51,15 +51,22 @@ function extractMiniCardsInfo(cards) { }); } -export async function processCards(cards, mini: boolean = false, dateSort: boolean = true) { +export async function processWithWp(cards, mini: boolean = false) { const cardsWithInfo = mini ? extractMiniCardsInfo(cards) : extractCardsInfo(cards); - const cardsPromise = await Promise.allSettled( - cardsWithInfo.map(async (card: Card) => await fetchCardDetails(card, dateSort)) - ); + const ids = cardsWithInfo.map((card: Card) => card.id.replace('post-', '')); + const allPosts = await ofetch(`${rootUrl}/wp-json/wp/v2/posts?include=${ids.join(',')}&_embed&per_page=${ids.length}`); + // To maintain the ID/post Order + const idMappedPost = Object.fromEntries(allPosts.map((post) => [post.id, post])); + return ids.map((id) => extractPostDetails(idMappedPost[id])); +} + +export async function processCards(cards, mini: boolean = false) { + const cardsWithInfo = mini ? extractMiniCardsInfo(cards) : extractCardsInfo(cards); + const cardsPromise = await Promise.allSettled(cardsWithInfo.map(async (card: Card) => await fetchCardDetails(card))); return cardsPromise.filter((card) => card.status === 'fulfilled').map((card) => card.value as DataItem); } -export async function fetchCardDetails(card: Card, dateSort: boolean) { +export async function fetchCardDetails(card: Card) { return await cache.tryGet(`css-tricks:${card.id}`, async () => { const response = await ofetch(card.link); const $ = load(response); @@ -79,8 +86,8 @@ export async function fetchCardDetails(card: Card, dateSort: boolean) { description: content, banner: card.thumbnail, image: card.thumbnail, - pubDate: dateSort ? parseDate(date) : '', - updated: dateSort ? parseDate(updateDate) : '', + pubDate: parseDate(date), + updated: parseDate(updateDate), author: [ { name: authorName || '', @@ -96,3 +103,39 @@ export async function fetchCardDetails(card: Card, dateSort: boolean) { } as DataItem; }); } + +function extractPostDetails(data) { + const title = data.title.rendered; + const link = data.link; + const content = data.content.rendered; + const summary = data.excerpt.rendered; + const date = data.date_gmt; + const updateDate = data.modified_gmt; + const author = data._embedded.author; + const authorName = author[0]?.name; + const authorUrl = author[0]?.link; + const authorAvatar = author[0]?.avatar_urls['48']; + const thumbnail = data._embedded['wp:featuredmedia']?.[0]?.source_url; + const tags = data._embedded['wp:term']?.[1]?.map((tag) => tag.name); + return { + title, + link, + description: content, + banner: thumbnail, + image: thumbnail, + pubDate: parseDate(date), + updated: parseDate(updateDate), + author: [ + { + name: authorName || '', + url: authorUrl || '', + avatar: authorAvatar || '', + }, + ], + content: { + html: content, + text: summary, + }, + category: tags, + } as DataItem; +}