Merge pull request #384 from grafana/ci-tools

Improve CI workflow

.circleci/config.yml

@@ -2,31 +2,94 @@
 #
 # Check https://circleci.com/docs/2.0/language-javascript/ for more details
 #
+aliases:
+  # Workflow filters
+  - &filter-not-release-or-master
+    tags:
+      ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
+    branches:
+      ignore:
+        - master
+  - &filter-only-master
+    branches:
+      only: master
+
 version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
-      - image: circleci/node:7.10
-
+      - image: circleci/node:8
     working_directory: ~/repo
-
     steps:
       - checkout
-
-      # Download and cache dependencies
       - restore_cache:
           keys:
-          - v1-dependencies-{{ checksum "package.json" }}
+          - dependency-cache-{{ checksum "yarn.lock" }}
           # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+          - dependency-cache-
+      - run:
+          name: yarn install
+          command: 'yarn install --pure-lockfile --no-progress'
+      - save_cache:
+          paths:
+            - node_modules
+          key: dependency-cache-{{ checksum "yarn.lock" }}
+      - run: yarn lint-repo
 
-      - run: yarn install
+  check-repo-update:
+    docker:
+      - image: circleci/node:8
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - dependency-cache-{{ checksum "yarn.lock" }}
+          - dependency-cache-
+      - run:
+          name: yarn install
+          command: 'yarn install --pure-lockfile --no-progress'
+      - save_cache:
+          paths:
+            - node_modules
+          key: dependency-cache-{{ checksum "yarn.lock" }}
+      - run:
+          name: check-repo-update
+          command: node ./scripts/check-repo-update.js
 
+  lint-updated-plugins:
+    docker:
+      - image: circleci/node:8
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - dependency-cache-{{ checksum "yarn.lock" }}
+          - dependency-cache-
+      - run:
+          name: yarn install
+          command: 'yarn install --pure-lockfile --no-progress'
       - save_cache:
           paths:
             - node_modules
-          key: v1-dependencies-{{ checksum "package.json" }}
-
-      # run tests!
-      - run: yarn lint
+          key: dependency-cache-{{ checksum "yarn.lock" }}
+      - run:
+          name: lint-updated-plugins
+          command: node ./scripts/lint-repo-update.js
+
+workflows:
+  version: 2
+  build-master:
+    jobs:
+      - build:
+          filters: *filter-only-master
+
+  build-branches-and-prs:
+    jobs:
+      - check-repo-update:
+          filters: *filter-not-release-or-master
+      - lint-updated-plugins:
+          filters: *filter-not-release-or-master
+      # - build:
+      #     filters: *filter-not-release-or-master

.circleci/run-build-locally.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+BASEDIR=$(dirname "$0")
+
+CURRENT_COMMIT_HASH=$(git log -n 1 | grep -Po "(?<=commit )[0-9a-z]{40}")
+CURRENT_BRANCH=$(git status | grep -Po "(?<=On branch ).+")
+
+COMMIT_HASH=${2:-${CURRENT_COMMIT_HASH}}
+BRANCH=${1:-${CURRENT_BRANCH}}
+
+echo "Branch: ${BRANCH} commit: ${COMMIT_HASH}"
+
+curl --user ${CIRCLE_TOKEN}: \
+    --request POST \
+    --form revision=${CURRENT_COMMIT_HASH}\
+    --form config=${BASEDIR}/config.yml \
+    --form notify=false \
+        https://circleci.com/api/v1.1/project/github/grafana/grafana-plugin-repository/tree/${CURRENT_BRANCH}

.gitignore

@@ -1 +1,2 @@
-node_modules
+node_modules
+.vscode

package.json

@@ -6,11 +6,17 @@
   "author": "Daniel Lee <[email protected]>",
   "license": "MIT",
   "scripts": {
-    "lint": "grunt"
+    "lint": "grunt",
+    "lint-repo": "./scripts/lint-repo.js",
+    "lint-repo-update": "./scripts/lint-repo-update.js",
+    "test-ci-build": "./.circleci/run-build-locally.sh"
   },
   "devDependencies": {
+    "chalk": "^2.4.2",
     "grunt": "^1.0.1",
-    "request": "^2.83.0",
+    "lodash": "^4.17.11",
+    "request": "^2.88.0",
+    "request-promise-native": "^1.0.7",
     "semver": "^5.4.1"
   }
 }

scripts/check-plugins.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+
+// const prettyjson = require('prettyjson');
+
+// const repo = require('../repo.json');
+const repoPath = path.join(__dirname, '../repo.json');
+let repo = fs.readFileSync(repoPath);
+try {
+  repo = JSON.parse(repo);
+} catch (parseError) {
+  console.error(chalk.red('Error parsing repo.json'))
+  console.error(parseError);
+  process.exit(1);
+}
+
+// console.log(prettyjson.render(repo));
+// console.log(repo);
+const plugins = repo.plugins.map(plugin => plugin.id);
+// console.log(plugins);
+console.log(repo.plugins[1]);

scripts/check-repo-update.js

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+const request = require('request-promise-native');
+const { diffPluginsRepo } = require('./diffPluginsRepo');
+const { checkSemver } = require('./checkSemver');
+
+const repoMasterUrl = 'https://raw.githubusercontent.com/grafana/grafana-plugin-repository/master/repo.json';
+const repoCurrent = require('../repo.json');
+
+function main() {
+  return request(repoMasterUrl).then(body => {
+    const repoMaster = JSON.parse(body);
+    const diff = diffPluginsRepo(repoCurrent, repoMaster);
+    return { diff, repoMaster };
+  }).then(result => {
+    const { diff, repoMaster } = result;
+    console.log(JSON.stringify(diff, null, 2));
+    return checkSemver(diff, repoMaster);
+  }).catch(error => {
+    console.error(error);
+    process.exit(1);
+  });
+}
+
+main();

scripts/checkSemver.js

@@ -0,0 +1,34 @@
+const _ = require('lodash');
+const semver = require('semver');
+
+/**
+ * Checks whether plugin updates follow semver or not.
+ * @param {*} diff
+ * @param {*} base
+ */
+function checkSemver(diff, base) {
+  _.forEach(diff.diff, (pluginUpdate, pluginId) => {
+    _.forEach(pluginUpdate.versions, versionObj => {
+      const updateVersion = versionObj.version;
+      if (!semver.valid(updateVersion)) {
+        throw new Error(`Invalid version: ${updateVersion} (plugin: ${pluginId})`);
+      }
+      const basePlugin = _.find(base.plugins, { 'id': pluginId });
+      if (basePlugin) {
+
+        const baseVersions = basePlugin.versions;
+        _.forEach(baseVersions, baseVersionObj => {
+          const baseVersion = baseVersionObj.version;
+          if (semver.lt(updateVersion, baseVersion)) {
+            throw new Error(`Updated version should be greater than previous: ${updateVersion} < ${baseVersion} (plugin: ${pluginId})`);
+          }
+        });
+      }
+    });
+  });
+  return true;
+}
+
+module.exports = {
+  checkSemver
+};

scripts/diffPluginsRepo.js

@@ -0,0 +1,78 @@
+const _ = require('lodash');
+
+/**
+ * Returns plugins repo diff with additional metadata.
+ * @param {Object} update updated repo
+ * @param {Object} base   base repo
+ * @return {Object}       Diff object
+ */
+function diffPluginsRepo(update, base) {
+  let diff = {};
+  let diffMeta = {};
+
+  const pluginsBase = _.keyBy(base.plugins, 'id');
+  const pluginsUpdate = _.keyBy(update.plugins, 'id');
+
+  _.forEach(pluginsUpdate, (pluginUpdate, id) => {
+    const pluginBase = pluginsBase[id];
+
+    // New plugin added
+    if (!pluginBase) {
+      diff[id] = pluginUpdate;
+      diffMeta[id] = { newPlugin: true };
+      return;
+    }
+
+    // Plugin version added
+    if (!_.isEqual(pluginUpdate, pluginBase)) {
+      diff[id] = {};
+      diffMeta[id] = { updatedPlugin: true };
+
+      // Check if metadata was changed
+      _.forOwn(pluginUpdate, (value, key) => {
+        if (value !== pluginBase[key] && key !== 'versions') {
+          diffMeta[id].metadataUpdated = true;
+          diff[key] = value;
+        }
+      });
+
+      let versionsDiff = _.xorWith(pluginUpdate.versions, pluginBase.versions, _.isEqual);
+      if (versionsDiff && versionsDiff.length) {
+        diff[id].versions = [];
+        // Check if version was changed
+        versionsDiff.forEach(versionObj => {
+          const changedVersion = _.find(pluginBase.versions, { 'version': versionObj.version });
+          if (changedVersion) {
+            diffMeta[id].versionChanged = true;
+            const versionChange = _.find(pluginUpdate.versions, { 'version': versionObj.version });
+            if (versionChange && !_.find(diff[id].versions, { 'version': versionChange.version })) {
+              diff[id].versions.push(versionChange);
+              _.forOwn(changedVersion, (value, key) => {
+                if (value !== versionObj[key]) {
+                  if (diffMeta[id].versionChanges && diffMeta[id].versionChanges[versionObj.version]) {
+                    diffMeta[id].versionChanges[versionObj.version].push(key);
+                  } else {
+                    diffMeta[id].versionChanges = {
+                      [versionObj.version]: [key]
+                    };
+                  }
+                }
+              });
+            }
+          } else {
+            diff[id].versions.push(versionObj);
+            diffMeta[id].versionAdded = true;
+          }
+        });
+      }
+    }
+  });
+  return {
+    diff: diff,
+    meta: diffMeta
+  };
+}
+
+module.exports = {
+  diffPluginsRepo
+};