From 232333c368a0b54740700b4b9a43305aae04288b Mon Sep 17 00:00:00 2001 From: George Shramko Date: Mon, 4 Sep 2023 19:00:35 -0700 Subject: [PATCH 1/5] More unit tests + log muting in mocha --- .../LeetcodeUpdatesCollectorService.spec.ts | 134 ++++++++++++++++++ .../VjudgeUpdatesCollectorService.spec.ts | 10 +- backend/test/testUtils/LogMuter.ts | 35 +++++ 3 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 backend/test/services/updates-collection/LeetcodeUpdatesCollectorService.spec.ts create mode 100644 backend/test/testUtils/LogMuter.ts diff --git a/backend/test/services/updates-collection/LeetcodeUpdatesCollectorService.spec.ts b/backend/test/services/updates-collection/LeetcodeUpdatesCollectorService.spec.ts new file mode 100644 index 0000000..a41f209 --- /dev/null +++ b/backend/test/services/updates-collection/LeetcodeUpdatesCollectorService.spec.ts @@ -0,0 +1,134 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { User } from '../../../src/model/schemas/userSchema'; +import { userUpdateEventEmitter } from '../../../src/events/UserUpdateEventEmitter'; + +import { leetcodeUpdatesCollectorService as sut } from '../../../src/services/updates-collection/LeetcodeUpdatesCollectorService' +import { leetcodeApi } from '../../../src/api/leetcode'; +import { refreshLeetcodeDataEventEmitter } from '../../../src/events/RefreshLeetcodeDataEventEmitter'; +import { logMuter } from '../../testUtils/LogMuter'; + +describe('leetcodeUpdatesCollectorService', function () { + let findOneStub: sinon.SinonStub; + let getLeetcodeSubmitsStub: sinon.SinonStub; + let getLatestAcSubmitsStub: sinon.SinonStub; + let emitUserUpdateStub: sinon.SinonStub; + let emitRefreshLeetcodeDataEvent: sinon.SinonStub; + + beforeEach(() => { + // mute console + logMuter.muteLogs(); + + //stub the methods + findOneStub = sinon.stub(User, 'findOne'); + getLeetcodeSubmitsStub = sinon.stub(leetcodeApi, 'getSubmitStats'); + getLatestAcSubmitsStub = sinon.stub(leetcodeApi, 'getLatestAcceptedSubmits') + emitUserUpdateStub = sinon.stub(userUpdateEventEmitter, 'emit'); + emitRefreshLeetcodeDataEvent = sinon.stub(refreshLeetcodeDataEventEmitter, 'emit'); + }); + + afterEach(function () { + logMuter.unmuteLogs(); + // Restore the stubbed methods after each test + sinon.restore(); + }); + + it('should handle user not found scenario for leetcode', async () => { + findOneStub.resolves(null); // User not found in MongoDB + + const result = await sut.getAndStoreUpdates('testUserId'); + + expect(result).to.deep.equal([]); // No updates should be returned + }); + + it('should handle user found but no Leetcode username scenario', async () => { + findOneStub.resolves({ username: 'testUser', leetcode: null }); // User found but no LeetCode username + + const result = await sut.getAndStoreUpdates('testUserId'); + + expect(result).to.deep.equal([]); // No updates should be returned + }); + + it('should handle new submissions scenario', async () => { + + // User has solved two problems on leetcode + findOneStub.resolves({ + leetcode: { + username: 'testUser', + submitStats: { + acSubmissionNum: [ + { + difficulty: "All", + count: 2, + submissions: 5 + }, + { + difficulty: "Easy", + count: 1, + submissions: 2 + }, + { + difficulty: "Medium", + count: 0, + submissions: 0 + }, + { + difficulty: "Hard", + count: 1, + submissions: 5 + } + ] + } + } + }); + + getLeetcodeSubmitsStub.resolves({ + username: 'testUser', + submitStats: { + acSubmissionNum: [ + { + difficulty: "All", + count: 6, + submissions: 15 + }, + { + difficulty: "Easy", + count: 3, + submissions: 6 + }, + { + difficulty: "Medium", + count: 1, + submissions: 1 + }, + { + difficulty: "Hard", + count: 2, + submissions: 8 + } + ] + } + }) + const problemNames = ['A', 'B', 'C', 'D']; + const latestSubmitsMock = problemNames.map(name => { + return { + title: name, + titleSlug: name, + timestamp: 12345, + lang: "cpp" + }}); + getLatestAcSubmitsStub.resolves(latestSubmitsMock); + + + const result = await sut.getAndStoreUpdates('testUserId'); + const newProblemsSolvedNames = result.map(data => data.problemTitle); + + expect(emitUserUpdateStub.callCount).to.equal(4); + expect(emitRefreshLeetcodeDataEvent.calledOnce).to.be.true; + + + expect(result.length).to.equal(4); + expect(newProblemsSolvedNames).to.deep.equal(problemNames); + }); +}); + diff --git a/backend/test/services/updates-collection/VjudgeUpdatesCollectorService.spec.ts b/backend/test/services/updates-collection/VjudgeUpdatesCollectorService.spec.ts index ad82b6f..10c1939 100644 --- a/backend/test/services/updates-collection/VjudgeUpdatesCollectorService.spec.ts +++ b/backend/test/services/updates-collection/VjudgeUpdatesCollectorService.spec.ts @@ -7,14 +7,18 @@ import { storeVjudgeSubmissionEventEmitter } from '../../../src/events/StoreVjud import { vjudgeUpdatesCollectorService as sut } from '../../../src/services/updates-collection/VjudgeUpdatesCollectorService'; +import { logMuter } from '../../testUtils/LogMuter'; -describe('getAndStoreVjudgeUpdates', function () { +describe('vjudgeUpdatesCollectorService', function () { let findOneStub: sinon.SinonStub; let getVjudgeSubmissionStatsStub: sinon.SinonStub; let emitUserUpdateStub: sinon.SinonStub; let emitStoreVjudgeEventStub: sinon.SinonStub; beforeEach(() => { + // mute console + logMuter.muteLogs(); + sinon.stub(console, 'error'); //stub the methods findOneStub = sinon.stub(User, 'findOne'); getVjudgeSubmissionStatsStub = sinon.stub(vjudgeApi, 'getSubmissionStats'); @@ -22,7 +26,9 @@ describe('getAndStoreVjudgeUpdates', function () { emitStoreVjudgeEventStub = sinon.stub(storeVjudgeSubmissionEventEmitter, 'emit'); }); - afterEach(() => { + afterEach(function() { + logMuter.unmuteLogs(); + // Restore the stubbed methods after each test sinon.restore(); }); diff --git a/backend/test/testUtils/LogMuter.ts b/backend/test/testUtils/LogMuter.ts new file mode 100644 index 0000000..83330bd --- /dev/null +++ b/backend/test/testUtils/LogMuter.ts @@ -0,0 +1,35 @@ +export class LogMuter { + private readonly logger = { + log: console.log, + error: console.error + }; + private logBuffer = ''; + private isMuted = false; + + muteLogs() { + this.isMuted = true; + console.log = (msg) => { + this.logBuffer += JSON.stringify(msg) + '\n'; + } + console.error = (msg) => { + this.logBuffer += "ERROR: " + JSON.stringify(msg) + '\n'; + } + } + flushLogs() { + this.logger.log(this.logBuffer); + this.logBuffer=''; + } + + unmuteLogs() { + this.isMuted = false; + console.log = this.logger.log; + console.error = this.logger.error; + this.logBuffer = ''; + } + + checkIfIsMuted() { + this.logger.log(this.isMuted); + } +} + +export const logMuter = new LogMuter(); \ No newline at end of file From 49ba3b8dd3e03eaa22f7c01a6e650d4b2027ed0e Mon Sep 17 00:00:00 2001 From: George Shramko Date: Mon, 4 Sep 2023 19:02:32 -0700 Subject: [PATCH 2/5] GCP Pub/Sub integration WIP --- backend/package-lock.json | 268 ++++++++++++++++++ backend/package.json | 3 +- backend/src/events/UserUpdateEventEmitter.ts | 28 +- .../LeetcodeUpdatesCollectorService.ts | 32 ++- .../VjudgeUpdatesCollectorService.ts | 7 +- 5 files changed, 313 insertions(+), 25 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 06ad4f2..d945d5d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@google-cloud/connect-firestore": "^2.0.2", "@google-cloud/firestore": "^4.15.1", + "@google-cloud/pubsub": "^4.0.2", "@google-cloud/storage": "^6.12.0", "@types/cors": "^2.8.13", "@types/cron": "^2.0.1", @@ -819,6 +820,14 @@ "node": ">=10" } }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@google-cloud/projectify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", @@ -835,6 +844,220 @@ "node": ">=12" } }, + "node_modules/@google-cloud/pubsub": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/pubsub/-/pubsub-4.0.2.tgz", + "integrity": "sha512-66StFux7jJzytVUTAjXnsiuHJYTXimqTocqLVorY26389KF9yO8KTI/74DeLhcKWVaUYroRvauRlvKWxXVIbHQ==", + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "@opentelemetry/api": "^1.0.0", + "@opentelemetry/semantic-conventions": "~1.3.0", + "@types/duplexify": "^3.6.0", + "@types/long": "^4.0.0", + "arrify": "^2.0.0", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "google-gax": "^4.0.2", + "heap-js": "^2.2.0", + "is-stream-ended": "^0.1.4", + "lodash.snakecase": "^4.1.1", + "p-defer": "^3.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/@grpc/grpc-js": { + "version": "1.8.21", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.21.tgz", + "integrity": "sha512-KeyQeZpxeEBSqFVTi3q2K7PiPXmgBfECc4updA1ejCLjYmoAlvvM3ZMp5ztTDUCUQmoY3CpDxvchjO1+rFkoHg==", + "dependencies": { + "@grpc/proto-loader": "^0.7.0", + "@types/node": ">=12.12.47" + }, + "engines": { + "node": "^8.13.0 || >=10.10.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/@grpc/proto-loader": { + "version": "0.7.9", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.9.tgz", + "integrity": "sha512-YJsOehVXzgurc+lLAxYnlSMc1p/Gu6VAvnfx0ATi2nzvr0YZcjhmZDeY8SeAKv1M7zE3aEJH0Xo9mK1iZ8GYoQ==", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.4", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@google-cloud/pubsub/node_modules/google-gax": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.0.3.tgz", + "integrity": "sha512-gllHYRhZvpz0LcVN+xtyzBeUa/ZYiLGF4JNBECrvL/LxDkaJc09hHoQ+KzRBI2Ewqgrjj7V3QrOC2pGno5ropw==", + "dependencies": { + "@grpc/grpc-js": "~1.8.0", + "@grpc/proto-loader": "^0.7.0", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "google-auth-library": "^9.0.0", + "node-fetch": "^2.6.1", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^1.1.1", + "protobufjs": "7.2.4", + "retry-request": "^6.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, + "node_modules/@google-cloud/pubsub/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/@google-cloud/pubsub/node_modules/proto3-json-serializer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-1.1.1.tgz", + "integrity": "sha512-AwAuY4g9nxx0u52DnSMkqqgyLHaW/XaPLtaAo3y/ZCfeaQB/g4YDH4kb8Wc/mWzWvu0YjOznVnfn373MVZZrgw==", + "dependencies": { + "protobufjs": "^7.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/retry-request": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-6.0.0.tgz", + "integrity": "sha512-24kaFMd3wCnT3n4uPnsQh90ZSV8OISpfTFXJ00Wi+/oD2OPrp63EQ8hznk6rhxdlpwx2QBhQSDz2Fg46ki852g==", + "dependencies": { + "debug": "^4.1.1", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@google-cloud/pubsub/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/@google-cloud/storage": { "version": "6.12.0", "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-6.12.0.tgz", @@ -1380,6 +1603,22 @@ "sparse-bitfield": "^3.0.3" } }, + "node_modules/@opentelemetry/api": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", + "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.3.1.tgz", + "integrity": "sha512-wU5J8rUoo32oSef/rFpOT1HIjLjAv3qIDHkw1QIhODV3OpAVHi5oVzlouozg9obUmZKtbZ0qUe/m7FP0y0yBzA==", + "engines": { + "node": ">=8.12.0" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1601,6 +1840,14 @@ "cron": "*" } }, + "node_modules/@types/duplexify": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.1.tgz", + "integrity": "sha512-n0zoEj/fMdMOvqbHxmqnza/kXyoGgJmEpsXjpP+gEqE1Ye4yNqc7xWipKnUoMpWhMuzJQSfK2gMrwlElly7OGQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/express": { "version": "4.17.17", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz", @@ -3765,6 +4012,14 @@ "he": "bin/he" } }, + "node_modules/heap-js": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/heap-js/-/heap-js-2.3.0.tgz", + "integrity": "sha512-E5303mzwQ+4j/n2J0rDvEPBN7GKjhis10oHiYOgjxsmxYgqG++hz9NyLLOXttzH8as/DyiBHYpUrJTZWYaMo8Q==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -4913,6 +5168,11 @@ "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", "dev": true }, + "node_modules/lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==" + }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -5797,6 +6057,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "engines": { + "node": ">=8" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", diff --git a/backend/package.json b/backend/package.json index 12c6c22..5ef8d0a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,7 +10,7 @@ "gcp-build": "npm install --save-dev && npm run build", "copy-sql": "cp ./src/db/db_init.sql ./dist/db/db_init.sql", "tsconfig": "tsc --init", - "test": "mocha" + "test": "mocha -R spec" }, "keywords": [], "author": "", @@ -18,6 +18,7 @@ "dependencies": { "@google-cloud/connect-firestore": "^2.0.2", "@google-cloud/firestore": "^4.15.1", + "@google-cloud/pubsub": "^4.0.2", "@google-cloud/storage": "^6.12.0", "@types/cors": "^2.8.13", "@types/cron": "^2.0.1", diff --git a/backend/src/events/UserUpdateEventEmitter.ts b/backend/src/events/UserUpdateEventEmitter.ts index 82950b0..45a3d5e 100644 --- a/backend/src/events/UserUpdateEventEmitter.ts +++ b/backend/src/events/UserUpdateEventEmitter.ts @@ -1,28 +1,40 @@ import EventEmitter from "events"; -import { EventRepository } from "../repository/EventRepository"; +import { eventRepository } from "../repository/EventRepository"; import { UpdateEventData } from "../model/UpdateEventData"; +import { PubSub } from '@google-cloud/pubsub'; export class UserUpdateEventEmitter { private eventEmitter: EventEmitter; private readonly EVENT_NAME = "user-update"; - private eventRepository: EventRepository; + private readonly pubSubClient = new PubSub(); constructor() { this.eventEmitter = new EventEmitter(); - this.eventRepository = new EventRepository(); - //TODO: setup things for GCP Pub/Sub - this.eventEmitter.on( this.EVENT_NAME, async (updateData: UpdateEventData) => { - await this.eventRepository.saveEvent(updateData); + await eventRepository.saveEvent(updateData); } ); } - emit(data: UpdateEventData) { + async emit(data: UpdateEventData) { this.eventEmitter.emit(this.EVENT_NAME, data); - // TODO: send to pub sub? + + // send update to PubSub for other integrations + const topicName = `update_events_user_${data.id}`; + const topic = this.pubSubClient.topic(topicName); + + const [exists] = await topic.exists(); + + if(!exists) { + await topic.create(); + } + + const messageId = await topic.publishMessage({ + data:JSON.stringify(data) + }); + console.log(`Message ${messageId} published to ${topicName}`); } } diff --git a/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts b/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts index 68c392b..6d98895 100644 --- a/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts +++ b/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts @@ -1,4 +1,4 @@ -import { leetcodeApi} from "../../api/leetcode"; +import { leetcodeApi } from "../../api/leetcode"; import { userUpdateEventEmitter } from "../../events/UserUpdateEventEmitter"; import { User } from "../../model/schemas/userSchema"; import { UpdateEventData } from "../../model/UpdateEventData"; @@ -10,7 +10,7 @@ export class LeetcodeUpdatesCollectorService implements IUpdatesCollectorService getPlatformName(): string { return "Leetcode"; } - async getAndStoreUpdates(userId: string) : Promise { + async getAndStoreUpdates(userId: string): Promise { const u = await User.findOne({ _id: userId }); if (!u) { @@ -39,20 +39,22 @@ export class LeetcodeUpdatesCollectorService implements IUpdatesCollectorService ); // once we have a list of problems that were solved since last check, emit an event for each of them - const updateEvents = updatesData.map((upd: any) => { - const updateEvent: UpdateEventData = { - id: userId, - username: username, - platform: "leetcode", - problemTitle: upd.title, - problemTitleSlug: upd.titleSlug, - // leetcode updates are with 1 second precision, but js are 1 millisecond - timestamp: parseInt(upd.timestamp) * 1000 - } - userUpdateEventEmitter.emit(updateEvent); + const updateEvents = await Promise.all(updatesData.map( + async (upd: any) => { + const updateEvent: UpdateEventData = { + id: userId, + username: username, + platform: "leetcode", + problemTitle: upd.title, + problemTitleSlug: upd.titleSlug, + // leetcode updates are with 1 second precision, but js are 1 millisecond + timestamp: parseInt(upd.timestamp) * 1000 + } + console.log("emitting actuawwy"); + await userUpdateEventEmitter.emit(updateEvent); - return updateEvent; - }); + return updateEvent; + })); refreshLeetcodeDataEventEmitter.emit({ codegramUsername: username, leetcodeData: newData }); return updateEvents; diff --git a/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts b/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts index 0159069..646cf3e 100644 --- a/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts +++ b/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts @@ -11,6 +11,7 @@ export class VjudgeUpdatesCollectorService implements IUpdatesCollectorService { getPlatformName(): string { return "VJudge"; } + private generateVjudgeUpdateEvent(userId: string, username: string, platform: string, problemName: string): UpdateEventData { const updateData: UpdateEventData = { id: userId, @@ -55,6 +56,8 @@ export class VjudgeUpdatesCollectorService implements IUpdatesCollectorService { console.log( `User ${userId} has solved something new on vjudge. (username: ${u.vjudge.username})` ); + + const emittionPromises: Promise[] = []; // 2-pointer approach to finding the different elements in 2 lists for (var i = 0, j = i; j < newValues.length; ++i, ++j) { @@ -65,7 +68,7 @@ export class VjudgeUpdatesCollectorService implements IUpdatesCollectorService { updates.push(update); // store update to be displayable in feed - userUpdateEventEmitter.emit(update); + emittionPromises.push(userUpdateEventEmitter.emit(update)); // update submits data to know about this problem storeVjudgeSubmissionEventEmitter.emit({ platform: platform, @@ -76,6 +79,8 @@ export class VjudgeUpdatesCollectorService implements IUpdatesCollectorService { ++j; } } + + await Promise.all(emittionPromises); } } if (!updates.length) { From 9725dac000e04b62a7606e561a5660c3738a7bfa Mon Sep 17 00:00:00 2001 From: George Shramko Date: Mon, 4 Sep 2023 19:30:14 -0700 Subject: [PATCH 3/5] Endcoding pubsub message to UTF-8 buffer --- backend/src/events/UserUpdateEventEmitter.ts | 25 +++++++++++-------- .../LeetcodeUpdatesCollectorService.ts | 1 - 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/backend/src/events/UserUpdateEventEmitter.ts b/backend/src/events/UserUpdateEventEmitter.ts index 45a3d5e..8e371e3 100644 --- a/backend/src/events/UserUpdateEventEmitter.ts +++ b/backend/src/events/UserUpdateEventEmitter.ts @@ -22,19 +22,24 @@ export class UserUpdateEventEmitter { this.eventEmitter.emit(this.EVENT_NAME, data); // send update to PubSub for other integrations - const topicName = `update_events_user_${data.id}`; - const topic = this.pubSubClient.topic(topicName); + try { + const topicName = `update_events_user_${data.id}`; + const topic = this.pubSubClient.topic(topicName); - const [exists] = await topic.exists(); + const [exists] = await topic.exists(); - if(!exists) { - await topic.create(); - } + if (!exists) { + await topic.create(); + } - const messageId = await topic.publishMessage({ - data:JSON.stringify(data) - }); - console.log(`Message ${messageId} published to ${topicName}`); + const message = Buffer.from(JSON.stringify(data)); + const messageId = await topic.publishMessage({ + data: message + }); + console.log(`Message ${messageId} published to ${topicName}`); + } catch (e) { + console.error(e); + } } } diff --git a/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts b/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts index 6d98895..6e90a04 100644 --- a/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts +++ b/backend/src/services/updates-collection/LeetcodeUpdatesCollectorService.ts @@ -50,7 +50,6 @@ export class LeetcodeUpdatesCollectorService implements IUpdatesCollectorService // leetcode updates are with 1 second precision, but js are 1 millisecond timestamp: parseInt(upd.timestamp) * 1000 } - console.log("emitting actuawwy"); await userUpdateEventEmitter.emit(updateEvent); return updateEvent; From 48ae1df0d7c83449af6607f691d8fc174ee39c11 Mon Sep 17 00:00:00 2001 From: George Shramko Date: Mon, 4 Sep 2023 20:55:11 -0700 Subject: [PATCH 4/5] Remove some logs pollution --- backend/src/config/corsConfig.ts | 1 - .../services/updates-collection/VjudgeUpdatesCollectorService.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/src/config/corsConfig.ts b/backend/src/config/corsConfig.ts index 5662555..60bf229 100644 --- a/backend/src/config/corsConfig.ts +++ b/backend/src/config/corsConfig.ts @@ -11,7 +11,6 @@ const whitelist = [ export const corsOptions: CorsOptions = { origin: (origin, callback) => { - console.log(origin); if (whitelist.includes(origin || '')) { callback(null, true); } else { diff --git a/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts b/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts index 646cf3e..22e7b07 100644 --- a/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts +++ b/backend/src/services/updates-collection/VjudgeUpdatesCollectorService.ts @@ -94,7 +94,6 @@ export class VjudgeUpdatesCollectorService implements IUpdatesCollectorService { return []; } })(); - console.log(updates); return updates; } catch (e) { console.error(`Failed to load vjudge updates for user ${userId}. Error: ${e}`); From f883f7cdbb7c90e3690f0a34e068cffe41e1a549 Mon Sep 17 00:00:00 2001 From: George Shramko Date: Mon, 4 Sep 2023 21:30:33 -0700 Subject: [PATCH 5/5] Use one topic for all user events in Pub/Sub --- backend/src/events/UserUpdateEventEmitter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/events/UserUpdateEventEmitter.ts b/backend/src/events/UserUpdateEventEmitter.ts index 8e371e3..41b5025 100644 --- a/backend/src/events/UserUpdateEventEmitter.ts +++ b/backend/src/events/UserUpdateEventEmitter.ts @@ -23,7 +23,7 @@ export class UserUpdateEventEmitter { // send update to PubSub for other integrations try { - const topicName = `update_events_user_${data.id}`; + const topicName = `UPDATE_EVENTS`; const topic = this.pubSubClient.topic(topicName); const [exists] = await topic.exists();