diff --git a/app/controllers/api.controller.ts b/app/controllers/api.controller.ts index 02a275f..5c41168 100644 --- a/app/controllers/api.controller.ts +++ b/app/controllers/api.controller.ts @@ -273,7 +273,7 @@ router.post('/postVoyage', async (req: Request, res: Response, next) => { try { let dbid = req.body.dbid; let voyage = req.body.voyage as ITrackedVoyage; - let apiResult = await DataCoreAPI.mongoPostTrackedVoyage(dbid, voyage, getLogDataFromReq(req)); + let apiResult = await DataCoreAPI.sqlitePostTrackedVoyage(dbid, voyage, getLogDataFromReq(req)); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); @@ -289,7 +289,7 @@ router.get('/getVoyages', async (req: Request, res: Response, next) => { try { let dbid = req.query?.dbid ? Number.parseInt(req.query.dbid.toString()) : undefined; let trackerId = req.query?.trackerId ? Number.parseInt(req.query.trackerId.toString()) : undefined; - let apiResult = await DataCoreAPI.mongoGetTrackedVoyages(dbid, trackerId); + let apiResult = await DataCoreAPI.sqliteGetTrackedVoyages(dbid, trackerId); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); @@ -303,10 +303,10 @@ router.post('/postAssignment', async (req: Request, res: Response, next) => { } try { - let dbid = req.body.dbid; + let dbid = Number.parseInt(req.body.dbid); let crew = req.body.crew; let assignment = req.body.assignment as ITrackedAssignment; - let apiResult = await DataCoreAPI.mongoPostTrackedAssignment(dbid, crew, assignment, getLogDataFromReq(req)); + let apiResult = await DataCoreAPI.sqlitePostTrackedAssignment(dbid, crew, assignment, getLogDataFromReq(req)); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); @@ -320,7 +320,7 @@ router.post('/postAssignments', async (req: Request, res: Response, next) => { } try { - let dbid = req.body.dbid; + let dbid = Number.parseInt(req.body.dbid); let assign = req.body.assignments as { [key: string]: ITrackedAssignment[] }; let crew = Object.keys(assign); let assignmap = Object.values(assign); @@ -337,7 +337,7 @@ router.post('/postAssignments', async (req: Request, res: Response, next) => { x++; } - let apiResult = await DataCoreAPI.mongoPostTrackedAssignmentsMany(dbid, finalcrew, assignments, getLogDataFromReq(req)); + let apiResult = await DataCoreAPI.sqlitePostTrackedAssignmentsMany(dbid, finalcrew, assignments, getLogDataFromReq(req)); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); @@ -353,7 +353,7 @@ router.get('/getAssignments', async (req: Request, res: Response, next) => { try { let dbid = req.query?.dbid ? Number.parseInt(req.query.dbid.toString()) : undefined; let trackerId = req.query?.trackerId ? Number.parseInt(req.query.trackerId.toString()) : undefined; - let apiResult = await DataCoreAPI.mongoGetTrackedVoyages(dbid, trackerId); + let apiResult = await DataCoreAPI.sqliteGetTrackedVoyages(dbid, trackerId); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); @@ -369,7 +369,7 @@ router.get('/getTrackedData', async (req: Request, res: Response, next) => { try { let dbid = req.query?.dbid ? Number.parseInt(req.query.dbid.toString()) : undefined; let trackerId = req.query?.trackerId ? Number.parseInt(req.query.trackerId.toString()) : undefined; - let apiResult = await DataCoreAPI.mongoGetTrackedData(dbid, trackerId); + let apiResult = await DataCoreAPI.sqliteGetTrackedData(dbid, trackerId); res.status(apiResult.Status).send(apiResult.Body); } catch (e) { next(e); diff --git a/app/logic/api.ts b/app/logic/api.ts index 5a9b866..c56e163 100644 --- a/app/logic/api.ts +++ b/app/logic/api.ts @@ -14,8 +14,9 @@ import { PlayerData } from '../datacore/player'; import { ITrackedAssignment, ITrackedVoyage } from '../datacore/voyage'; import { TrackedCrew, TrackedVoyage } from '../mongoModels/voyageHistory'; import { connectToMongo } from '../mongo'; -import { IFBB_BossBattle_Document, IFBB_Solve_Document, IFBB_Trial_Document } from '../mongoModels/playerCollab'; +import { IFBB_BossBattle_Document } from '../mongoModels/playerCollab'; import { CrewTrial, Solve } from '../datacore/boss'; +import { postOrPutVoyage_sqlite, getVoyagesByDbid_sqlite, getVoyagesByTrackerId_sqlite, postOrPutAssignmentsMany_sqlite, getAssignmentsByDbid_sqlite, getAssignmentsByTrackerId_sqlite, postOrPutAssignment_sqlite } from './voyagetracker'; require('dotenv').config(); @@ -841,6 +842,313 @@ export class ApiClass { } + + + async sqlitePostTrackedVoyage(dbid: number, voyage: ITrackedVoyage, logData: LogData): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Tracked Voyage data', { dbid, voyage, logData }); + + const timeStamp = new Date(); + + try { + let res = await postOrPutVoyage_sqlite(dbid, voyage, timeStamp); + if (res >= 300) { + return { + Status: res, + Body: { + 'dbid': dbid, + 'error': 'Unable to insert record.', + 'timeStamp': timeStamp.toISOString() + } + }; + } + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + return { + Status: 201, + Body: { + 'dbid': dbid, + 'trackerId': voyage.tracker_id, + timeStamp: timeStamp.toISOString() + } + }; + } + + async sqliteGetTrackedVoyages(dbid?: number, trackerId?: number): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Get voyage data', { dbid, trackerId }); + let voyages: TrackedVoyage[] | null = null; + + if (!dbid && !trackerId) return { + Status: 400, + Body: { result: "bad input" } + } + + if (!dbid) return { + Status: 400, + Body: { result: "bad input" } + } + + try { + if (trackerId) { + voyages = (trackerId ? await getVoyagesByTrackerId_sqlite(dbid, trackerId) : null); + } + else { + voyages = await getVoyagesByDbid_sqlite(dbid) + } + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + if (voyages) { + return { + Status: 200, + Body: voyages + }; + } + else { + return { + Status: 200, // 204 + Body: [] + }; + } + + } + + + + async sqlitePostTrackedAssignment(dbid: number, crew: string, assignment: ITrackedAssignment, logData: LogData): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Tracked Voyage data', { dbid, voyage: assignment, logData }); + + const timeStamp = new Date(); + + try { + let res = await postOrPutAssignment_sqlite(dbid, crew, assignment, timeStamp); + if (res >= 300) { + return { + Status: res, + Body: { + 'dbid': dbid, + 'error': 'Unable to insert record.', + 'timeStamp': timeStamp.toISOString() + } + }; + } + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + return { + Status: 201, + Body: { + 'dbid': dbid, + 'trackerId': assignment.tracker_id, + timeStamp: timeStamp.toISOString() + } + }; + } + + + async sqlitePostTrackedAssignmentsMany(dbid: number, crew: string[], assignments: ITrackedAssignment[], logData: LogData): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Tracked Voyage data', { dbid, voyage: assignments, logData }); + + const timeStamp = new Date(); + + try { + let res = await postOrPutAssignmentsMany_sqlite(dbid, crew, assignments, timeStamp); + if (res >= 300) { + return { + Status: res, + Body: { + 'dbid': dbid, + 'error': 'Unable to insert record.', + 'timeStamp': timeStamp.toISOString() + } + }; + } + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + return { + Status: 201, + Body: { + 'dbid': dbid, + 'trackerIds': assignments.map(a => a.tracker_id), + timeStamp: timeStamp.toISOString() + } + }; + } + + + async sqliteGetTrackedAssignments(dbid?: number, trackerId?: number): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Get voyage data', { dbid, trackerId }); + let assignments: TrackedCrew[] | null = null; + + if (!dbid && !trackerId) return { + Status: 400, + Body: { result: "bad input" } + } + + if (!dbid) return { + Status: 400, + Body: { result: "bad input" } + } + + try { + if (trackerId) { + assignments = (trackerId ? await getAssignmentsByTrackerId_sqlite(dbid, trackerId) : null); + } + else { + assignments = await getAssignmentsByDbid_sqlite(dbid); + } + + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + if (assignments) { + return { + Status: 200, + Body: assignments + }; + } + else { + return { + Status: 200, // 204 + Body: [] + }; + } + + } + + + async sqliteGetTrackedData(dbid?: number, trackerId?: number): Promise { + if (!this.mongoAvailable) return { Status: 500, Body: 'Database is down' }; + + Logger.info('Get tracked data', { dbid, trackerId }); + let voyages: TrackedVoyage[] | null = null; + let assignments: TrackedCrew[] | null = null; + + if (!dbid && !trackerId) return { + Status: 400, + Body: { result: "bad input" } + } + + if (!dbid) return { + Status: 400, + Body: { result: "bad input" } + } + + try { + if (trackerId) { + voyages = (trackerId ? await getVoyagesByTrackerId_sqlite(dbid, trackerId) : null); + assignments = (trackerId ? await getAssignmentsByTrackerId_sqlite(dbid, trackerId) : null); + } + else { + voyages = await getVoyagesByDbid_sqlite(dbid); + assignments = await getAssignmentsByDbid_sqlite(dbid); + } + } catch (err) { + if (typeof err === 'string') { + return { + Status: 500, + Body: err + }; + } + else if (err instanceof Error) { + return { + Status: 500, + Body: err.toString() + }; + } + } + + if (voyages || assignments) { + return { + Status: 200, + Body: { + voyages, + assignments + } + }; + } + else { + return { + Status: 200, // 204 + Body: { voyages: [], assignments: [] } + }; + } + + } + + + } export let DataCoreAPI = new ApiClass(); diff --git a/app/logic/mongotools.ts b/app/logic/mongotools.ts index 947e1c9..4d6364b 100644 --- a/app/logic/mongotools.ts +++ b/app/logic/mongotools.ts @@ -3,13 +3,13 @@ import { WithId } from "mongodb"; import { collections } from "../mongo"; import { PlayerProfile } from "../mongoModels/playerProfile"; import { PlayerData } from "../datacore/player"; -import { ITrackedAssignment, ITrackedVoyage, IVoyageHistory } from "../datacore/voyage"; +import { ITrackedAssignment, ITrackedVoyage } from "../datacore/voyage"; import { ITelemetryVoyage, TelemetryVoyage, TrackedCrew, TrackedVoyage } from "../mongoModels/voyageHistory"; import { BossBattleDocument, IFBB_BossBattle_Document, SolveDocument, TrialDocument } from "../mongoModels/playerCollab"; import * as seedrandom from 'seedrandom'; import { Collaboration, CrewTrial, Solve } from "../datacore/boss"; import { createProfileObject } from "./profiletools"; -import { createHash } from 'node:crypto' +import { createHash } from 'node:crypto'; export async function getProfile(dbid: number) { let res: PlayerProfile | null = null; diff --git a/app/logic/telemetry.ts b/app/logic/telemetry.ts index 53471fa..dd443ec 100644 --- a/app/logic/telemetry.ts +++ b/app/logic/telemetry.ts @@ -1,6 +1,4 @@ import { VoyageRecord } from '../models/VoyageRecord'; -import { Op } from 'sequelize'; -import { Sequelize } from 'sequelize-typescript'; export async function recordTelemetryDB(type: string, data: any) { switch (type) { diff --git a/app/logic/voyagetracker.ts b/app/logic/voyagetracker.ts new file mode 100644 index 0000000..77207fc --- /dev/null +++ b/app/logic/voyagetracker.ts @@ -0,0 +1,138 @@ + +import { ITrackedAssignment, ITrackedVoyage } from "../datacore/voyage"; +import { TrackedCrew, TrackedVoyage } from "../models/Tracked"; +import { makeSql } from "../sequelize"; + +export async function getVoyagesByDbid_sqlite(dbid: number) { + let res: TrackedVoyage[] | null = null; + + const sql = await makeSql(dbid); + if (sql) { + res = await TrackedVoyage.findAll({ where: { dbid }}); + // sql?.close(); + } + + return res; +} + +export async function getVoyagesByTrackerId_sqlite(dbid: number, trackerId: number) { + let res: TrackedVoyage[] | null = null; + + const sql = await makeSql(dbid); + if (sql) { + res = await TrackedVoyage.findAll({ where: { trackerId }}); + // sql?.close(); + } + + return res; +} + +export async function postOrPutVoyage_sqlite( + dbid: number, + voyage: ITrackedVoyage, + timeStamp: Date = new Date()) { + + const sql = await makeSql(dbid); + + if (sql) { + + let result = await TrackedVoyage.create({ + dbid, + trackerId: voyage.tracker_id, + voyage, + timeStamp + }) + // sql?.close(); + + return !!(result?.id) ? 201 : 400; + } + + return 500; +} + +export async function getAssignmentsByDbid_sqlite(dbid: number) { + let res: TrackedCrew[] | null = null; + + const sql = await makeSql(dbid); + + if (sql) { + res = await TrackedCrew.findAll({ where: { dbid }}); + // sql?.close(); + } + + return res; +} + +export async function getAssignmentsByTrackerId_sqlite(dbid: number, trackerId: number) { + let res: TrackedCrew[] | null = null; + + const sql = await makeSql(dbid); + if (sql) { + res = await TrackedCrew.findAll({ where: { trackerId }}); + // sql?.close(); + } + + return res; +} + +export async function postOrPutAssignment_sqlite( + dbid: number, + crew: string, + assignment: ITrackedAssignment, + timeStamp: Date = new Date()) { + + const sql = await makeSql(dbid); + if (sql) { + let result = await TrackedCrew.create({ + dbid, + crew, + trackerId: assignment.tracker_id, + assignment, + timeStamp + }) + // sql?.close(); + + return !!(result?.id) ? 201 : 400; + } + + return 500; +} + +export async function postOrPutAssignmentsMany_sqlite( + dbid: number, + crew: string[], + assignments: ITrackedAssignment[], + timeStamp: Date = new Date()) { + + let result = true; + const newdata = [] as any[]; + let x = 0; + + const sql = await makeSql(dbid); + if (sql) { + for (let crewMember of crew) { + let assignment = assignments[x++]; + newdata.push({ + dbid, + crew: crewMember, + trackerId: assignment.tracker_id, + assignment, + timeStamp + }); + + } + + try { + result = !!await TrackedCrew.bulkCreate(newdata); + } + catch (err: any) { + console.log(err); + } + + // sql?.close(); + result &&= !!newdata; + return result ? 201 : 400; + } + + return 500; +} diff --git a/app/sequelize.ts b/app/sequelize.ts index 29a203c..a6cfe89 100644 --- a/app/sequelize.ts +++ b/app/sequelize.ts @@ -4,6 +4,8 @@ import { User } from './models/User'; import { Profile } from './models/Profile'; import { Comment } from './models/Comment'; import { VoyageRecord } from './models/VoyageRecord'; +import fs from 'fs'; +import { TrackedVoyage, TrackedCrew } from './models/Tracked'; require('dotenv').config(); @@ -11,3 +13,38 @@ export const sequelize = new Sequelize(process.env.DB_CONNECTION_STRING!, { models: [User, Profile, Comment, VoyageRecord], logging: false }); + +export async function makeSql(dbid: number) { + var dpath = process.env.PROFILE_DATA_PATH; + if (!dpath) return null; + if (!fs.existsSync(dpath)) { + fs.mkdirSync(dpath); + } + + if (fs.existsSync(dpath)) { + if (dpath[dpath.length - 1] === '/') { + dpath += "database"; + } + else { + dpath += "/database"; + } + if (!fs.existsSync(dpath)) { + fs.mkdirSync(dpath); + } + dpath += "/" + dbid.toString() + ".sqlite"; + } + + dpath = "sqlite:" + dpath; + + const newdb = new Sequelize(dpath, { + models: [TrackedVoyage, TrackedCrew], + logging: false + }); + + if (newdb) { + await TrackedVoyage.sync(); + await TrackedCrew.sync(); + } + + return newdb; +} \ No newline at end of file diff --git a/app/server.ts b/app/server.ts index d333008..ed94b77 100644 --- a/app/server.ts +++ b/app/server.ts @@ -8,12 +8,7 @@ import expressWinston from 'express-winston'; import { ApiController } from './controllers'; import { Logger, DataCoreAPI } from './logic'; import { sequelize } from './sequelize'; -import { collections, connectToMongo } from './mongo'; -import { VoyageRecord } from './models/VoyageRecord'; -import { Op, Sequelize } from 'sequelize'; -import { ITelemetryVoyage, TelemetryVoyage } from './mongoModels/voyageHistory'; -import fs from 'fs' -import { CrewMember } from './datacore/crew'; +import { connectToMongo } from './mongo'; require('dotenv').config(); // Create a new express application instance @@ -22,6 +17,8 @@ const app: express.Application = express(); // When used with nginx reverse proxy, pick a rerouting port let port: number = 4420; +// let port: number = 4421; + if (process.argv.length > 2) { port = parseInt(process.argv[2]); } @@ -92,77 +89,6 @@ const cycleInitMongo = async (force?: boolean) => { setTimeout(async () => { await cycleInitMongo(); - // if (DataCoreAPI.mongoAvailable && collections.telemetry) { - // let response = collections.telemetry.aggregate([ - // {$group : {_id: "$crewSymbol", count: { $sum: 1 } } } - // ]); - // if (response) { - // let result = await response.toArray(); - // console.log(`${result.length} records queried.`); - // console.log(result); - // } - // // let response = collections.telemetry.find({ crewSymbol: 'mariner_mirror_crew' }); - // // if (response) { - // // console.log(await response.toArray()); - // // } - // } - - // if (DataCoreAPI.mongoAvailable) { - // console.log("Connection Established, Querying Old Telemetry Data..."); - // const baseFilter = { - // group: ['crewSymbol'], - // attributes: ['crewSymbol', [Sequelize.fn('COUNT', Sequelize.col('crewSymbol')), 'crewCount'], [Sequelize.fn('AVG', Sequelize.col('estimatedDuration')), 'averageDuration']], - // } as any; - - // if (collections.telemetry){ - // console.log("Wiping current MongoDB telemetry collection..."); - // await collections.telemetry.deleteMany({}); - // console.log("Done."); - // } - - // const crews = JSON.parse(fs.readFileSync("../website/static/structured/crew.json", 'utf8')) as CrewMember[]; - - // for (let crew of crews) { - - // console.log(`Reading old data from '${crew.name}' ... `); - - // let data: VoyageRecord[] | null; - - // data = await VoyageRecord.findAll({ where: { crewSymbol: crew.symbol } }); - - // if (!data?.length) { - // console.log("No data, skipping..."); - // continue; - // } - - // console.log(`Old data from '${crew.name}': ${data.length} Records...`); - - // if (collections.telemetry) { - - // let mapped = data.map(item => { return { - // crewSymbol: item.crewSymbol, - // voyageDate: item.voyageDate, - // estimatedDuration: item.estimatedDuration ?? 0 - // } as ITelemetryVoyage }); - // data.length = 0; - // data = null; - - // console.log(`Inserting records from crew '${mapped[0].crewSymbol}' into Mongo ...`); - // collections.telemetry.insertMany(mapped); - // console.log("Done. Moving on to next set..."); - // } - // else { - // data.length = 0; - // data = null; - - // console.log("Mongo is not found!"); - // break; - // } - // } - - // console.log("Populating MongoDB completed. You may quit."); - - // } }) // Serve the application at the given port diff --git a/tsconfig.json b/tsconfig.json index 1aa082e..529591f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "strict": true, "esModuleInterop": true, "experimentalDecorators": true, + "sourceMap": true, "emitDecoratorMetadata": true } }