From 89d238f78debd56431bb8adc5ab7ac65eced985f Mon Sep 17 00:00:00 2001 From: FLYBYME Date: Sun, 1 Oct 2023 16:29:56 -0400 Subject: [PATCH 1/4] work --- services/github.service.js | 45 ++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/services/github.service.js b/services/github.service.js index 03d12ed..c1448c0 100644 --- a/services/github.service.js +++ b/services/github.service.js @@ -315,23 +315,34 @@ module.exports = { const keys = Object.keys(payload); const action = this.getActionName(payload); - // loop over payload keys - for (const key of keys) { - - this.logger.info(`action ${action} ${key}`) - if (key == 'action') { - events.push({ - key: key, - action: action, - payload: this.compactActionPayload(payload[key]) - }); - } else if (key == 'workflow_run') { - events.push({ - key: key, - action: action, - payload: this.compactWorkflowRunPayload(payload[key]) - }); - } + + if (actions == 'synchronize') { + events.push({ + key: 'pull_request', + action: action, + payload: payload.pull_request + }); + return events; + } + + // if first key is action pick second key as object + if (keys[0] == 'action') { + const object = keys[1]; + const objectPayload = payload[object]; + events.push({ + key: object, + action: action, + payload: objectPayload + }); + } else { + // if first key is not action pick first key as object + const object = keys[0]; + const objectPayload = payload[object]; + events.push({ + key: object, + action: action, + payload: objectPayload + }); } return events; From c3900e30403356d68024411dda4820ca560b1640 Mon Sep 17 00:00:00 2001 From: FLYBYME Date: Sun, 1 Oct 2023 16:31:26 -0400 Subject: [PATCH 2/4] work --- services/github.service.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/github.service.js b/services/github.service.js index c1448c0..6a19b27 100644 --- a/services/github.service.js +++ b/services/github.service.js @@ -59,6 +59,10 @@ module.exports = { // payload events const events = this.compactPayload(payload); + if (events.length == 0) { + console.log(payload) + return; + } // loop over events for (const event of events) { // emit event From a9cafd5af1265b91624730a5e3897275c908a3b6 Mon Sep 17 00:00:00 2001 From: FLYBYME Date: Sun, 1 Oct 2023 16:37:20 -0400 Subject: [PATCH 3/4] work --- services/github.service.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/github.service.js b/services/github.service.js index 6a19b27..9918794 100644 --- a/services/github.service.js +++ b/services/github.service.js @@ -317,10 +317,10 @@ module.exports = { const events = []; const keys = Object.keys(payload); - + console.log(keys) const action = this.getActionName(payload); - if (actions == 'synchronize') { + if (action == 'synchronize') { events.push({ key: 'pull_request', action: action, @@ -364,7 +364,7 @@ module.exports = { let actionName = ''; if (!action) { - actionsName = 'unknown'; + actionName = 'unknown'; } else { actionName = action } From fb5ac5fa6eb98df8e6917d974bfcf97f274f50a2 Mon Sep 17 00:00:00 2001 From: FLYBYME Date: Sun, 1 Oct 2023 18:28:56 -0400 Subject: [PATCH 4/4] df --- moleculer.config.js | 2 +- package-lock.json | 19 ++ package.json | 1 + services/github.service.js | 469 ++++++++++++++++++------------------- 4 files changed, 247 insertions(+), 244 deletions(-) diff --git a/moleculer.config.js b/moleculer.config.js index c622d88..6a44614 100644 --- a/moleculer.config.js +++ b/moleculer.config.js @@ -174,7 +174,7 @@ module.exports = { // Enable built-in tracing function. More info: https://moleculer.services/docs/0.14/tracing.html tracing: { - enabled: true, + enabled: false, // Available built-in exporters: "Console", "Datadog", "Event", "EventLegacy", "Jaeger", "Zipkin" exporter: { type: "Console", // Console exporter is only for development! diff --git a/package-lock.json b/package-lock.json index e7a2714..709da80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "dependencies": { "config-mixin": "git+https://github.com/PaaS-Shack/config-mixin.git", + "flat": "^6.0.1", "middlewares": "git+https://github.com/PaaS-Shack/middlewares.git", "moleculer": "^0.14.26", "nats": "^2.7.1" @@ -3016,6 +3017,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/flat": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/flat/-/flat-6.0.1.tgz", + "integrity": "sha512-/3FfIa8mbrg3xE7+wAhWeV+bd7L2Mof+xtZb5dRDKZ+wDvYJK4WDYeIOuOhre5Yv5aQObZrlbRmk3RTSiuQBtw==", + "bin": { + "flat": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/flat-cache": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.0.tgz", @@ -9238,6 +9250,7 @@ }, "config-mixin": { "version": "git+ssh://git@github.com/PaaS-Shack/config-mixin.git#a24dc2dbc0b77fe48dd7d23c1c0e60c7c29a168f", + "integrity": "sha512-nVGLqSFVaiP3tgzInkLFo6wp3nKfoewBY71UhYJApceBbwORCnjPUkXlW49rXTtBNWqy5pm4uff4eMOfp+kOhg==", "from": "config-mixin@git+https://github.com/PaaS-Shack/config-mixin.git", "requires": { "lodash": "^4.17.21", @@ -9872,6 +9885,11 @@ "path-exists": "^4.0.0" } }, + "flat": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/flat/-/flat-6.0.1.tgz", + "integrity": "sha512-/3FfIa8mbrg3xE7+wAhWeV+bd7L2Mof+xtZb5dRDKZ+wDvYJK4WDYeIOuOhre5Yv5aQObZrlbRmk3RTSiuQBtw==" + }, "flat-cache": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.0.tgz", @@ -11851,6 +11869,7 @@ }, "middlewares": { "version": "git+ssh://git@github.com/PaaS-Shack/middlewares.git#e582cb711befd74f2f6c4adb67c4b4246d64cee5", + "integrity": "sha512-589pk5ioJ17DSXJWtPQ/1aBoRUSXgjTAOxOWX2atSiD29VEzpfPeeCbho+D143yW3FEKwXKaGjJX/pTKybkAVg==", "from": "middlewares@git+https://github.com/PaaS-Shack/middlewares.git", "requires": { "lodash": "^4.17.21", diff --git a/package.json b/package.json index 787f5c8..4c5ce5c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ }, "dependencies": { "config-mixin": "git+https://github.com/PaaS-Shack/config-mixin.git", + "flat": "^6.0.1", "middlewares": "git+https://github.com/PaaS-Shack/middlewares.git", "moleculer": "^0.14.26", "nats": "^2.7.1" diff --git a/services/github.service.js b/services/github.service.js index 9918794..5ec193a 100644 --- a/services/github.service.js +++ b/services/github.service.js @@ -2,6 +2,164 @@ const ConfigLoader = require("config-mixin"); const { MoleculerClientError } = require("moleculer").Errors; +function isBuffer(obj) { + return obj && + obj.constructor && + (typeof obj.constructor.isBuffer === 'function') && + obj.constructor.isBuffer(obj) +} + +function keyIdentity(key) { + return key +} + +function flatten(target, opts) { + opts = opts || {} + + const delimiter = opts.delimiter || '.' + const maxDepth = opts.maxDepth + const transformKey = opts.transformKey || keyIdentity + const output = {} + + function step(object, prev, currentDepth) { + currentDepth = currentDepth || 1 + Object.keys(object).forEach(function (key) { + const value = object[key] + const isarray = opts.safe && Array.isArray(value) + const type = Object.prototype.toString.call(value) + const isbuffer = isBuffer(value) + const isobject = ( + type === '[object Object]' || + type === '[object Array]' + ) + + const newKey = prev + ? prev + delimiter + transformKey(key) + : transformKey(key) + + if (!isarray && !isbuffer && isobject && Object.keys(value).length && + (!opts.maxDepth || currentDepth < maxDepth)) { + return step(value, newKey, currentDepth + 1) + } + + output[newKey] = value + }) + } + + step(target) + + return output +} + +function unflatten(target, opts) { + opts = opts || {} + + const delimiter = opts.delimiter || '.' + const overwrite = opts.overwrite || false + const transformKey = opts.transformKey || keyIdentity + const result = {} + + const isbuffer = isBuffer(target) + if (isbuffer || Object.prototype.toString.call(target) !== '[object Object]') { + return target + } + + // safely ensure that the key is + // an integer. + function getkey(key) { + const parsedKey = Number(key) + + return ( + isNaN(parsedKey) || + key.indexOf('.') !== -1 || + opts.object + ) + ? key + : parsedKey + } + + function addKeys(keyPrefix, recipient, target) { + return Object.keys(target).reduce(function (result, key) { + result[keyPrefix + delimiter + key] = target[key] + + return result + }, recipient) + } + + function isEmpty(val) { + const type = Object.prototype.toString.call(val) + const isArray = type === '[object Array]' + const isObject = type === '[object Object]' + + if (!val) { + return true + } else if (isArray) { + return !val.length + } else if (isObject) { + return !Object.keys(val).length + } + } + + target = Object.keys(target).reduce(function (result, key) { + const type = Object.prototype.toString.call(target[key]) + const isObject = (type === '[object Object]' || type === '[object Array]') + if (!isObject || isEmpty(target[key])) { + result[key] = target[key] + return result + } else { + return addKeys( + key, + result, + flatten(target[key], opts) + ) + } + }, {}) + + Object.keys(target).forEach(function (key) { + const split = key.split(delimiter).map(transformKey) + let key1 = getkey(split.shift()) + let key2 = getkey(split[0]) + let recipient = result + + while (key2 !== undefined) { + if (key1 === '__proto__') { + return + } + + const type = Object.prototype.toString.call(recipient[key1]) + const isobject = ( + type === '[object Object]' || + type === '[object Array]' + ) + + // do not write over falsey, non-undefined values if overwrite is false + if (!overwrite && !isobject && typeof recipient[key1] !== 'undefined') { + return + } + + if ((overwrite && !isobject) || (!overwrite && recipient[key1] == null)) { + recipient[key1] = ( + typeof key2 === 'number' && + !opts.object + ? [] + : {} + ) + } + + recipient = recipient[key1] + if (split.length > 0) { + key1 = getkey(split.shift()) + key2 = getkey(split[0]) + } + } + + // unflatten again for 'messy objects' + recipient[key1] = unflatten(target[key], opts) + }) + + return result +} + /** * this service manages the github api and webhooks * @@ -57,18 +215,15 @@ module.exports = { const payload = ctx.params; // payload events - const events = this.compactPayload(payload); + const event = this.compactPayload(payload); - if (events.length == 0) { - console.log(payload) + if (event.payload == false) { return; } - // loop over events - for (const event of events) { - // emit event - ctx.emit(`github.${event.key}.${event.action}`, event.payload); - this.logger.info(`github.${event.key}.${event.action}`, event.payload); - } + + ctx.emit(`github.${event.key}.${event.action}`, event.payload); + this.logger.info(`github.${event.key}.${event.action}`, event.payload); + } }, }, @@ -84,272 +239,97 @@ module.exports = { * Methods */ methods: { - /** - * compact sender payload - * - * @param {Object} payload - sender payload - * - * @returns {Object} compacted sender payload - */ - compactSenderPayload(payload) { - return { - id: payload.id, - login: payload.login, - node_id: payload.node_id, - avatar_url: payload.avatar_url, - gravatar_id: payload.gravatar_id, - url: payload.url, - type: payload.type, - site_admin: payload.site_admin - }; - }, - /** - * compact repository payload - * - * @param {Object} payload - repository payload - * - * @returns {Object} compacted repository payload - */ - compactRepositoryPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - name: payload.name, - full_name: payload.full_name, - private: payload.private, - owner: this.compactSenderPayload(payload.owner), - description: payload.description, - fork: payload.fork, - url: payload.url, - created_at: payload.created_at, - updated_at: payload.updated_at, - pushed_at: payload.pushed_at, - homepage: payload.homepage, - size: payload.size, - stargazers_count: payload.stargazers_count, - watchers_count: payload.watch - }; - }, /** - * compact deployment payload + * payload to events * - * @param {Object} payload - deployment payload + * @param {Object} payload - github webhook payload * - * @returns {Object} compacted deployment payload + * @returns {Array} github action events */ - compactDeploymentPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - sha: payload.sha, - ref: payload.ref, - task: payload.task, - payload: payload.payload, - environment: payload.environment, - description: payload.description, - creator: this.compactSenderPayload(payload.creator), - created_at: payload.created_at, - updated_at: payload.updated_at - }; - }, + compactPayload(payload) { + const events = []; - /** - * compact deployment status payload - * - * @param {Object} payload - deployment status payload - * - * @returns {Object} compacted deployment status payload - */ - compactDeploymentStatusPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - state: payload.state, - creator: this.compactSenderPayload(payload.creator), - description: payload.description, - created_at: payload.created_at, - updated_at: payload.updated_at, - }; - }, + const keys = Object.keys(payload); - /** - * compact check run payload - * - * @param {Object} payload - check run payload - * - * @returns {Object} compacted check run payload - */ - compactCheckRunPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - head_sha: payload.head_sha, - external_id: payload.external_id, - url: payload.url, - status: payload.status, - conclusion: payload.conclusion, - started_at: payload.started_at, - completed_at: payload.completed_at, - output: payload.output, - name: payload.name, - check_suite: payload.check_suite, - app: payload.app, - pull_requests: payload.pull_requests - }; - }, + const action = this.getActionName(payload); - /** - * compact workflow payload - * - * @param {Object} payload - workflow payload - * - * @returns {Object} compacted workflow payload - */ - compactWorkflowPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - name: payload.name, - path: payload.path, - state: payload.state, - created_at: payload.created_at, - updated_at: payload.updated_at, - url: payload.url, - html_url: payload.html_url, - badge_url: payload.badge_url - }; - }, + const key = this.getEventKey(payload, action); + + let strippedPayload = false; + + if (key == 'package') { + strippedPayload = { + name: payload.package.name, + namespace: payload.package.namespace, + branch: payload.package.package_version.target_commitish, + version: payload.package.package_version.version, + }; + } else if (key == 'commit') { + strippedPayload = { + name: payload.repository.name, + namespace: payload.repository.owner.name, + branch: payload.ref.split('/').pop(), + ref: payload.ref, + commits: payload.commits, + head: payload.head_commit + } + } - /** - * compact workflow run payload - * - * @param {Object} payload - workflow run payload - * - * @returns {Object} compacted workflow run payload - */ - compactWorkflowRunPayload(payload) { return { - id: payload.id, - node_id: payload.node_id, - head_branch: payload.head_branch, - head_sha: payload.head_sha, - run_number: payload.run_number, - event: payload.event, - status: payload.status, - conclusion: payload.conclusion, - workflow_id: payload.workflow_id, - url: payload.url, - pull_requests: payload.pull_requests, - created_at: payload.created_at, - updated_at: payload.updated_at, - head_commit: payload.head_commit, - repository: this.compactRepositoryPayload(payload.repository), - head_repository: this.compactRepositoryPayload(payload.head_repository), - sender: this.compactSenderPayload(payload.sender) + action: action, + key: key, + payload: strippedPayload }; }, /** - * compact action payload + * deep walk payload adn strip any keys that include "_url" * - * @param {Object} payload - action payload + * @param {Object} payload - github webhook payload * - * @returns {Object} compacted action payload + * @returns {Object} stripped payload */ - compactActionPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - name: payload.name, - description: payload.description, - identifier: payload.identifier, - icon_url: payload.icon_url, - created_at: payload.created_at, - updated_at: payload.updated_at, - permissions: payload.permissions - }; - }, + stripUrls(payload = {}) { + const strippedPayload = {}; - /** - * compact organization payload - * - * @param {Object} payload - organization payload - * - * @returns {Object} compacted organization payload - */ - compactOrganizationPayload(payload) { - return { - id: payload.id, - node_id: payload.node_id, - login: payload.login, - url: payload.url, - description: payload.description, - gravatar_id: payload.gravatar_id, - name: payload.name, - company: payload.company, - blog: payload.blog, - location: payload.location, - email: payload.email, - twitter_username: payload.twitter_username, - is_verified: payload.is_verified, - has_organization_projects: payload.has_organization_projects, - has_repository_projects: payload.has_repository_projects, - public_repos: payload.public_repos, - public_gists: payload.public_gists, - followers: payload.followers, - following: payload.following, - created_at: payload.created_at, - updated_at: payload.updated_at, - type: payload.type - }; + const flattened = flatten(payload); + + const keys = Object.keys(flattened); + + keys.forEach(key => { + if (!key.includes('_url')) { + strippedPayload[key] = flattened[key]; + } + }); + + return unflatten(strippedPayload); }, + /** - * payload to events + * get event key * * @param {Object} payload - github webhook payload + * @param {String} action - github webhook action * - * @returns {Array} github action events + * @return {String} event key */ - compactPayload(payload) { - const events = []; + getEventKey(payload, action) { const keys = Object.keys(payload); - console.log(keys) - const action = this.getActionName(payload); - - if (action == 'synchronize') { - events.push({ - key: 'pull_request', - action: action, - payload: payload.pull_request - }); - return events; + if (keys[0] == 'action') { + keys.shift(); } - // if first key is action pick second key as object - if (keys[0] == 'action') { - const object = keys[1]; - const objectPayload = payload[object]; - events.push({ - key: object, - action: action, - payload: objectPayload - }); - } else { - // if first key is not action pick first key as object - const object = keys[0]; - const objectPayload = payload[object]; - events.push({ - key: object, - action: action, - payload: objectPayload - }); + let key = keys[0]; + + if (key == 'ref') { + key = 'commit'; } - return events; + return key; }, /** @@ -373,6 +353,9 @@ module.exports = { const payloadKeys = Object.keys(payload); if (payloadKeys.length > 0) { actionName = payloadKeys[0]; + if (actionName == 'ref') { + actionName = 'push'; + } } }