Skip to content

Commit

Permalink
database: jsonb for links
Browse files Browse the repository at this point in the history
  • Loading branch information
h11 committed Apr 5, 2020
1 parent 6719382 commit f2d89cd
Show file tree
Hide file tree
Showing 12 changed files with 753 additions and 1,115 deletions.
18 changes: 15 additions & 3 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,18 @@ on:
jobs:
heroku:
runs-on: ubuntu-18.04
services:
postgres:
image: postgres
env:
POSTGRES_USER: user
POSTGRES_PASSWORD: password
POSTGRES_DB: feedbox
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/[email protected]
- uses: actions/[email protected]
Expand All @@ -26,15 +38,15 @@ jobs:
env:
NODE_ENV: 'development'
SERVER: 'http://localhost:8000'
COOKIE_SECRET: "crypto.randomBytes(2).toString('hex')"
GITHUB_AUTH_SECRET: "crypto.randomBytes(2).toString('hex')"
COOKIE_SECRET: 'crypto.randomBytes(2).toString("hex")'
GITHUB_AUTH_SECRET: 'crypto.randomBytes(2).toString("hex")'
GITHUB_CLIENT_ID: 'https://github.com/settings/developers'
GITHUB_CLIENT_SECRET: 'https://github.com/settings/developers'
MAILGUN_API_KEY: 'key-example'
MAILGUN_DOMAIN: 'feedbox.example.com'
MAILGUN_FROM: 'feedbox <[email protected]>'
ROLLBAR_TOKEN: 'https://rollbar.com'
DATABASE_URL: 'postgres://example'
DATABASE_URL: 'postgres://user:password@localhost:5432/feedbox'
- uses: codecov/[email protected]
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down
16 changes: 0 additions & 16 deletions dotenv
Original file line number Diff line number Diff line change
@@ -1,16 +0,0 @@
NODE_ENV="development"
SERVER="http://localhost:8000"

COOKIE_SECRET="crypto.randomBytes(32).toString('hex')"

GITHUB_AUTH_SECRET="crypto.randomBytes(32).toString('hex')"
GITHUB_CLIENT_ID="70f278a95aa587dc6fd6"
GITHUB_CLIENT_SECRET="d302f69ea8020c101c3c08032a7f411840c5979f"

MAILGUN_API_KEY="key-example"
MAILGUN_DOMAIN="feedbox.example.com"
MAILGUN_FROM="feedbox <[email protected]>"

ROLLBAR_TOKEN="https://rollbar.com"

DATABASE_URL="postgres://example"
2 changes: 1 addition & 1 deletion dotenv.example
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@ MAILGUN_FROM="feedbox <[email protected]>"

ROLLBAR_TOKEN="https://rollbar.com"

DATABASE_URL="postgres://example"
DATABASE_URL="postgres://user:password@localhost:5432/feedbox"
13 changes: 13 additions & 0 deletions migrations/20200405135736_r1.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import * as Knex from 'knex'

export const up = (knex: Knex) => {
return knex.schema.table('feedbox_feed', (table) => {
table.jsonb('links').defaultTo('[]')
})
}

export const down = (knex: Knex) => {
return knex.schema.alterTable('feedbox_feed', (table) => {
table.dropColumn('links')
})
}
21 changes: 10 additions & 11 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"build": "make release"
},
"engines": {
"node": "13.*",
"node": "12.*",
"yarn": "1.*"
},
"jest": {
Expand All @@ -30,15 +30,14 @@
"dotenv-safe": "8.2.0",
"fast-xml-parser": "3.16.0",
"feedparser": "2.2.9",
"hapi-pino": "6.5.0",
"hapi-pino": "7.0.0",
"iconv-lite": "0.5.1",
"jschardet": "2.1.1",
"knex": "0.20.11",
"knex": "0.20.13",
"mailgun-js": "0.22.0",
"node-fetch": "2.6.0",
"pg": "7.18.2",
"pg": "8.0.0",
"rollbar": "2.15.0",
"sqlite3": "4.1.1",
"tslib": "1.11.1"
},
"devDependencies": {
Expand All @@ -48,17 +47,17 @@
"@rollup/plugin-replace": "2.3.1",
"@rollup/plugin-typescript": "4.0.0",
"@types/feedparser": "2.2.3",
"@types/jest": "25.1.4",
"@types/jest": "25.2.1",
"@types/mailgun-js": "0.22.4",
"@types/node": "13.9.2",
"@types/node": "13.11.0",
"@types/node-fetch": "2.5.5",
"dotenv-safe": "8.2.0",
"form-data": "3.0.0",
"jest": "25.1.0",
"jest": "25.2.7",
"nodemon": "2.0.2",
"prettier": "2.0.1",
"rollup": "2.1.0",
"rollup-plugin-svelte": "5.1.1",
"prettier": "2.0.2",
"rollup": "2.3.3",
"rollup-plugin-svelte": "5.2.1",
"rollup-plugin-terser": "5.3.0",
"svelte": "3.20.1",
"typescript": "3.8.3"
Expand Down
23 changes: 4 additions & 19 deletions server/models/config.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,15 @@
import * as path from 'path'

const prod = process.env.NODE_ENV === 'production'

const common = {
export const config = {
debug: process.env.DEBUG_SQL === 'true',
migrations: {
directory: path.resolve(__dirname, '../../migrations'),
tableName: 'knex_migrations',
},
}

const sqlite3 = {
client: 'sqlite3',
connection: {
filename: path.resolve(__dirname, './feedbox.sqlite'),
},
useNullAsDefault: true,
}

const pg = {
client: 'pg',
connection: `${process.env.DATABASE_URL}?ssl=true`,
connection: process.env.DATABASE_URL,
pool: { min: 2, max: 20 },
}

export const production = Object.assign({}, common, pg)
export const development = Object.assign({}, common, sqlite3)
export const config = prod ? production : development
export default config
export const production = config
export const development = config
73 changes: 25 additions & 48 deletions server/models/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,54 +85,44 @@ export const model = {
return r
},

async prepareFeedForUpdate(): Promise<FeedDoc[]> {
async getActiveFeeds(): Promise<Feed[]> {
const feeds = await conn()
.select(
'feedbox_feed.id as id',
'feedbox_feed.url as url',
'feedbox_feed.links as links',
'feedbox_feed.updated as updated',
'feedbox_user.email as email',
)
.from('feedbox_r_user_feed')
.from('feedbox_feed')
.innerJoin(
'feedbox_feed',
'feedbox_feed.id',
'feedbox_r_user_feed',
'feedbox_r_user_feed.feed_id',
'feedbox_feed.id',
)
return feeds
},

async getSubscribers(id: number): Promise<User[]> {
const users = await conn()
.select(
'feedbox_user.id as id',
'feedbox_user.github_id as githubId',
'feedbox_user.email as email',
)
.from('feedbox_user')
.innerJoin(
'feedbox_user',
'feedbox_user.id',
'feedbox_r_user_feed',
'feedbox_r_user_feed.user_id',
'feedbox_user.id',
)
const map = new Map<number, FeedDoc>()
feeds.forEach(({ id, url, updated, email }) => {
const v = map.get(id) ?? {
id,
url,
updated,
emails: [] as string[],
links: new Set(),
}
v.emails.push(email)
map.set(id, v)
})
const links = await conn()
.select('feed_id', 'url')
.from('feedbox_link')
.whereIn('feed_id', Array.from(map.keys()))
links.forEach(({ feed_id, url }) => {
const v = map.get(feed_id)!
v.links.add(url)
})
return Array.from(map.values())
.where({ 'feedbox_r_user_feed.feed_id': id })
return users
},

async addLink(feed_id: number, url: string) {
await conn().insert({ feed_id, url }).into('feedbox_link')
},

async updateFeedUpdated(id: number, updated: Date) {
await conn()('feedbox_feed').where({ id }).update({ updated })
async updateFeed(id: number, links: string[], updated: Date) {
await conn()('feedbox_feed')
.where({ id })
.update({ links: JSON.stringify(links), updated })
},

async subscribe(user_id: number, feed_id: number) {
Expand Down Expand Up @@ -184,23 +174,10 @@ export interface Feed {
id: number
url: string
updated: number | null
}

export interface Link {
id: number
url: string
feed_id: number
links: string[]
}

export interface RUserFeed {
user_id: number
feed_id: number
}

export interface FeedDoc {
id: number
url: string
updated: number | null
emails: string[]
links: Set<string>
}
Loading

0 comments on commit f2d89cd

Please sign in to comment.