Skip to content

Commit

Permalink
feat: crag exporter job (#379)
Browse files Browse the repository at this point in the history
* feat: crag exporter
  • Loading branch information
vnugent authored Jan 14, 2024
1 parent d649c6d commit e25c856
Show file tree
Hide file tree
Showing 11 changed files with 288 additions and 47 deletions.
28 changes: 24 additions & 4 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,19 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Crag Geojson",
"program": "${workspaceFolder}/src/db/export/CragGeojson/index.ts",
"preLaunchTask": "tsc: build - tsconfig.json",
"outFiles": [
"${workspaceFolder}/build/**/*.js"
],
"skipFiles": [
"<node_internals>/**"
],
},
{
"type": "node",
"request": "launch",
Expand Down Expand Up @@ -48,14 +61,21 @@
"type": "node",
"request": "launch",
"name": "Launch API Server (serve-dev)",
"skipFiles": ["<node_internals>/**"],
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/src/main.ts",
"preLaunchTask": "tsc: build - tsconfig.json",
"outFiles": ["${workspaceFolder}/build/**/*.js"],
"outFiles": [
"${workspaceFolder}/build/**/*.js"
],
"runtimeExecutable": "yarn",
"runtimeArgs": ["run", "serve-dev"],
"runtimeArgs": [
"run",
"serve-dev"
],
"console": "integratedTerminal"
},
},
{
"name": "Debug Jest Tests",
"type": "node",
Expand Down
22 changes: 22 additions & 0 deletions export-crag-data.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

if [ -z ${GITHUB_ACCESS_TOKEN} ]
then
echo "GITHUB_ACCESS_TOKEN not defined."
exit 1
fi

echo "cloning openbeta-export repository"
git clone --depth 1 --branch production https://ob-bot-user:${GITHUB_ACCESS_TOKEN}@github.com/OpenBeta/openbeta-export || exit 1
git config user.name "db-export-bot"
git config user.email "db-export-bot@noreply"
cd ..

echo "start exporting CRAG data..."
yarn export-crags

echo "... finished export. Committing data..."

git add -A
git commit -am "export crag data"
git push origin production
8 changes: 5 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@
"@turf/area": "^6.5.0",
"@turf/bbox": "^6.5.0",
"@turf/bbox-polygon": "^6.5.0",
"@turf/centroid": "^6.5.0",
"@turf/circle": "^6.5.0",
"@turf/convex": "^6.5.0",
"@turf/helpers": "^6.5.0",
"@types/uuid": "^8.3.3",
"apollo-datasource-mongodb": "^0.5.4",
"apollo-server": "^3.9.0",
Expand Down Expand Up @@ -83,7 +84,8 @@
"export:json:full": "yarn build && node build/db/export/json/index.js",
"export-prod": "./export.sh",
"prepare": "husky install",
"import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js"
"import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
"export-crags": "tsc ; node build/db/utils/jobs/CragGeojson/index.js"
},
"standard": {
"plugins": [
Expand All @@ -103,4 +105,4 @@
"engines": {
"node": ">=16.14.0"
}
}
}
20 changes: 20 additions & 0 deletions src/db/AreaSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,20 @@ import { GradeContexts } from '../GradeUtils.js'

const { Schema, connection } = mongoose

const polygonSchema = new mongoose.Schema({
type: {
type: String,
enum: ['Polygon'],
required: true
},
coordinates: {
type: [[[Number]]], // Array of arrays of arrays of numbers
required: true
}
}, {
_id: false
})

const ChangeRecordMetadata = new Schema<ChangeRecordMetadataType>({
user: {
type: 'object',
Expand All @@ -32,6 +46,7 @@ const MetadataSchema = new Schema<IAreaMetadata>({
type: PointSchema,
index: '2dsphere'
},
polygon: polygonSchema,
bbox: [{ type: Number, required: true }],
leftRightIndex: { type: Number, required: false },
ext_id: { type: String, required: false, index: true },
Expand Down Expand Up @@ -121,13 +136,18 @@ AreaSchema.index({ _deleting: 1 }, { expireAfterSeconds: 0 })
AreaSchema.index({
'metadata.leftRightIndex': 1
}, {
name: 'leftRightIndex',
partialFilterExpression: {
'metadata.leftRightIndex': {
$gt: -1
}
}
})

AreaSchema.index({
children: 1
})

export const createAreaModel = (name: string = 'areas'): mongoose.Model<AreaType> => {
return connection.model(name, AreaSchema)
}
Expand Down
10 changes: 7 additions & 3 deletions src/db/AreaTypes.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import mongoose from 'mongoose'
import { MUUID } from 'uuid-mongodb'

import { BBox, Point } from '@turf/helpers'
import { BBox, Point, Polygon } from '@turf/helpers'
import { ClimbType } from './ClimbTypes.js'
import { ChangeRecordMetadataType } from './ChangeLogType.js'
import { GradeContexts } from '../GradeUtils.js'
Expand Down Expand Up @@ -118,8 +118,7 @@ export interface IAreaMetadata {
*/
isBoulder?: boolean
/**
* Areas may be very large, and this point may represent the centroid of the area's bounds
* or a spec point chosen by users.
* Location of a wall or a boulder aka leaf node. Use `bbox` or `polygon` non-leaf areas.
* */
lnglat: Point
/**
Expand All @@ -143,6 +142,11 @@ export interface IAreaMetadata {
* GQL layer use these values for querying and identification of areas.
*/
area_id: MUUID

/**
* A polygon (created by convex hull) containing all child areas.
*/
polygon?: Polygon
}
export interface IAreaContent {
/** longform to mediumform description of this area.
Expand Down
2 changes: 1 addition & 1 deletion src/db/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export const connectDB = async (onConnected: () => any = defaultFn): Promise<voi

await mongoose.connect(
`${scheme}://${user}:${pass}@${server}/${dbName}?authSource=${authDb}&tls=${tlsFlag}&replicaSet=${rsName}`,
{ autoIndex: false }
{ autoIndex: true }
)
} catch (e) {
logger.error("Can't connect to db")
Expand Down
140 changes: 140 additions & 0 deletions src/db/utils/jobs/CragGeojson/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import { createWriteStream } from 'node:fs'
import { point, feature, featureCollection, Feature, Point, Polygon } from '@turf/helpers'
import os from 'node:os'
import { MUUID } from 'uuid-mongodb'

import { connectDB, gracefulExit, getAreaModel } from '../../../index.js'
import { logger } from '../../../../logger.js'

/**
* Export leaf areas as Geojson. Leaf areas are crags/boulders that have climbs.
*/
async function exportLeafCrags (): Promise<void> {
const model = getAreaModel()

const stream = createWriteStream('crags.geojson', { encoding: 'utf-8' })

const features: Array<Feature<Point, {
name: string
id: string
}>> = []

for await (const doc of model.find({ 'metadata.leaf': true }).lean()) {
const { metadata, area_name: areaName, pathTokens, ancestors } = doc

const ancestorArray = ancestors.split(',')
const pointFeature = point(doc.metadata.lnglat.coordinates, {
id: metadata.area_id.toUUID().toString(),
name: areaName,
type: 'crag',
parent: {
id: ancestorArray[ancestorArray.length - 2],
name: pathTokens[doc.pathTokens.length - 2]
}
})
features.push(pointFeature)
}
stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
stream.close()
}

/**
* Export crag groups as Geojson. Crag groups are immediate parent of leaf areas (crags/boulders).
*/
async function exportCragGroups (): Promise<void> {
const model = getAreaModel()
const stream = createWriteStream('crag-groups.geojson', { encoding: 'utf-8' })

interface CragGroup {
uuid: MUUID
name: string
polygon: Polygon
childAreaList: Array<{
name: string
uuid: MUUID
leftRightIndex: number
}>
}

const rs: CragGroup[] = await model.aggregate([
{ $match: { 'metadata.leaf': true } },
{
$lookup: {
from: 'areas',
localField: '_id',
foreignField: 'children',
as: 'parentCrags'
}
},
{
$match: {
$and: [
{ parentCrags: { $type: 'array', $ne: [] } }
]
}
},
{
$unwind: '$parentCrags'
},
{
$addFields: {
parentCrags: {
childId: '$metadata.area_id'
}
}
},
{
$group: {
_id: {
uuid: '$parentCrags.metadata.area_id',
name: '$parentCrags.area_name',
polygon: '$parentCrags.metadata.polygon'
},
childAreaList: {
$push: {
leftRightIndex: '$metadata.leftRightIndex',
uuid: '$metadata.area_id',
name: '$area_name'
}
}
}
},
{
$project: {
_id: 0,
uuid: '$_id.uuid',
name: '$_id.name',
polygon: '$_id.polygon',
childAreaList: 1
}
}
])

const features: Array<Feature<Polygon, {
name: string
id: string
}>> = []

for await (const doc of rs) {
const polygonFeature = feature(doc.polygon, {
type: 'crag-group',
name: doc.name,
id: doc.uuid.toUUID().toString(),
children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => (
{ id: uuid.toUUID().toString(), name, lr: leftRightIndex }))
})
features.push(polygonFeature)
}

stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
stream.close()
}

async function onDBConnected (): Promise<void> {
logger.info('Start exporting crag data as Geojson')
await exportLeafCrags()
await exportCragGroups()
await gracefulExit()
}

void connectDB(onDBConnected)
Loading

0 comments on commit e25c856

Please sign in to comment.