From 9c14c89de45d0140fd5ca181fbe88d028e1bc0ba Mon Sep 17 00:00:00 2001 From: SatyamMattoo Date: Wed, 7 Aug 2024 22:46:14 +0530 Subject: [PATCH] Adding new command 'configuration' to return the sample and full schema of a adaptor --- packages/cli/src/cli.ts | 2 + packages/cli/src/commands.ts | 5 +- packages/cli/src/configuration/command.ts | 37 +++++ packages/cli/src/configuration/handler.ts | 125 ++++++++++++++++ packages/cli/src/options.ts | 29 +++- .../test/configuration/configuration.test.ts | 141 ++++++++++++++++++ .../cli/test/configuration/options.test.ts | 49 ++++++ 7 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/configuration/command.ts create mode 100644 packages/cli/src/configuration/handler.ts create mode 100644 packages/cli/test/configuration/configuration.test.ts create mode 100644 packages/cli/test/configuration/options.test.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 9a500a4f0..2f0bc47e3 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -3,6 +3,7 @@ import { hideBin } from 'yargs/helpers'; import apolloCommand from './apollo/command'; import compileCommand from './compile/command'; +import configurationCommand from './configuration/command'; import deployCommand from './deploy/command'; import docgenCommand from './docgen/command'; import docsCommand from './docs/command'; @@ -25,6 +26,7 @@ export const cmd = y .command(testCommand) .command(docsCommand) .command(apolloCommand) + .command(configurationCommand) .command(metadataCommand as any) .command(docgenCommand as any) .command(pullCommand as any) diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 9c84e9ca4..5644fecff 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -2,6 +2,7 @@ import { Opts } from './options'; import apollo from './apollo/handler'; import execute from './execute/handler'; import compile from './compile/handler'; +import configuration from './configuration/handler'; import test from './test/handler'; import deploy from './deploy/handler'; import docgen from './docgen/handler'; @@ -19,6 +20,7 @@ import printVersions from './util/print-versions'; export type CommandList = | 'apollo' | 'compile' + | 'configuration' | 'deploy' | 'docgen' | 'docs' @@ -36,6 +38,7 @@ const handlers = { apollo, execute, compile, + configuration, test, deploy, docgen, @@ -84,7 +87,7 @@ const parse = async (options: Opts, log?: Logger) => { // TODO it would be nice to do this in the repoDir option, but // the logger isn't available yet if ( - !/^(pull|deploy|test|version|apollo)$/.test(options.command!) && + !/^(pull|deploy|test|version|apollo|configuration)$/.test(options.command!) && !options.repoDir ) { logger.warn( diff --git a/packages/cli/src/configuration/command.ts b/packages/cli/src/configuration/command.ts new file mode 100644 index 000000000..be8dab47a --- /dev/null +++ b/packages/cli/src/configuration/command.ts @@ -0,0 +1,37 @@ +import yargs from 'yargs'; +import * as o from '../options'; +import type { Opts } from '../options'; +import { build, ensure, override } from '../util/command-builders'; + +export type ConfigOptions = Pick< + Opts, + 'log' | 'logJson' | 'outputPath' | 'outputStdout' | 'configType' +> & { + adaptor: string; +}; + +const options = [ + o.log, + o.logJson, + o.outputPath, + o.configType, + override(o.outputStdout, { + default: true, + }), +]; + +export default { + command: 'configuration ', + handler: ensure('configuration', options), + describe: 'Returns the sample and full configuration of the adaptor. You can use flags (schema, sample or both(default)) to return only the sample or full configuration.', + builder: (yargs) => + build(options, yargs) + .example( + 'configuration adaptor_name@version', + 'Returns the sample and full configuration of the adaptor for a given version.' + ) + .example( + 'configuration adaptor_name --sample', + 'Returns only the sample configuration of the adaptor for the latest version.' + ), +} as yargs.CommandModule<{}>; diff --git a/packages/cli/src/configuration/handler.ts b/packages/cli/src/configuration/handler.ts new file mode 100644 index 000000000..521d0b42c --- /dev/null +++ b/packages/cli/src/configuration/handler.ts @@ -0,0 +1,125 @@ +import { Logger } from '../util/logger'; +import { ConfigOptions } from './command'; + +import { writeFile, mkdir } from 'node:fs/promises'; +import path from 'node:path'; + +const configurationHandler = async (options: ConfigOptions, logger: Logger) => { + logger.always(`Retrieving configuration for: ${options.adaptor}`); + + const { adaptor } = options; + + try { + const configData = await getConfig(adaptor, logger); + const filteredConfigData = filterConfigData(configData, options.configType); + + await serializeOutput(options, filteredConfigData, logger); + + logger.success('Done!'); + return filteredConfigData; + } catch (error: any) { + logger.error(`Failed to retrieve configuration`); + logger.error(error); + } +}; + +const getConfig = async (adaptor: string, logger: Logger) => { + // Fetch the configuration-schema.json file from the CDN + const configPath = `${adaptor}/configuration-schema.json`; + + logger.always("Fetching configuration file..."); + const configContent = await fetchFile(configPath, logger); + const fullSchema = JSON.parse(configContent); + + // Extract required fields and their examples + const requiredFields = fullSchema.required || []; + const properties = fullSchema.properties || {}; + const sampleConfig: Record = {}; + + requiredFields.forEach((field: string) => { + const fieldInfo = properties[field]; + if (fieldInfo && fieldInfo.examples && fieldInfo.examples.length > 0) { + sampleConfig[field] = fieldInfo.examples[0]; + } + }); + + const configData = { + sample_config: sampleConfig, + full_schema: fullSchema, + }; + + return configData; +}; + +const filterConfigData = (configData: any, schemaOption: string | undefined) => { + switch (schemaOption) { + case 'sample': + return { sample_config: configData.sample_config }; + case 'schema': + return { full_schema: configData.full_schema }; + case 'both': + default: + return configData; + } +}; + +const write = async ( + basePath: string, + filePath: string, + content: string, + logger: Logger +) => { + const ext = path.extname(basePath); + let dir; + if (ext) { + dir = path.dirname(path.resolve(basePath)); + } else { + dir = basePath; + } + + // Ensure the root dir exists + await mkdir(dir, { recursive: true }); + + const dest = path.resolve(basePath, filePath); + await writeFile(dest, content); + + logger.success(`Wrote content to ${dest}`); +}; + +// Serialize output to file and stdout +const serializeOutput = async ( + options: Pick, + result: any, + logger: Logger +) => { + /** Print to disk **/ + if (options.outputPath) { + await write( + options.outputPath, + '', + JSON.stringify(result, null, 2), + logger + ); + } + + /** print to stdout **/ + logger.success('Configuration Data:'); + logger.always(JSON.stringify(result, undefined, 2)); +}; + +async function fetchFile(path: string, logger: Logger) { + const resolvedPath = `https://cdn.jsdelivr.net/npm/${path}`; + + logger.debug("Fetching configuration from: ", resolvedPath); + const response = await fetch(resolvedPath); + + if (response.status === 200) { + return response.text(); + } + + throw new Error( + `Failed getting file at: ${path} got: ${response.status} ${response.statusText}` + ); +} + +export default configurationHandler; diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index 3082082b8..afa4f3a16 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -24,6 +24,7 @@ export type Opts = { cacheSteps?: boolean; compile?: boolean; configPath?: string; + configType?: string; confirm?: boolean; describe?: string; end?: string; // workflow end node @@ -335,7 +336,7 @@ export const outputPath: CLIOption = { }, ensure: (opts) => { // TODO these command specific rules don't sit well here - if (/^(compile|apollo)$/.test(opts.command!)) { + if (/^(compile|apollo|configuration)$/.test(opts.command!)) { if (opts.outputPath) { // If a path is set, remove the stdout flag delete opts.outputStdout; @@ -477,3 +478,29 @@ export const sanitize: CLIOption = { throw new Error(err); }, }; + +export const configType: CLIOption = { + name: 'configType', + yargs: { + alias: ['config-type'], + description: 'Specify the schema to return - schema, sample, or both', + default: 'both', + }, + ensure: (opts) => { + let didLoadShortcut = false; + ['schema', 'sample', 'both'].forEach((shortcut) => { + if (shortcut in opts) { + if (didLoadShortcut) { + throw new Error( + 'Invalid shortcut - please only enter one of sample, schema or both' + ); + } + + opts.configType = shortcut; + // @ts-ignore + delete opts[shortcut]; + didLoadShortcut = true; + } + }); + }, +}; diff --git a/packages/cli/test/configuration/configuration.test.ts b/packages/cli/test/configuration/configuration.test.ts new file mode 100644 index 000000000..7ed89e0c3 --- /dev/null +++ b/packages/cli/test/configuration/configuration.test.ts @@ -0,0 +1,141 @@ +import test from 'ava'; +import fs from 'node:fs/promises'; +import mockfs from 'mock-fs'; +import { createMockLogger } from '@openfn/logger'; + +import configurationHandler from "../../src/configuration/handler"; +import { ConfigOptions } from '../../src/configuration/command'; + +const logger = createMockLogger(); + +const MOCK_ADAPTOR = '@openfn/language-dhis2'; + +const sample_config = { + "hostUrl": "https://play.dhis2.org/2.36.6", + "password": "@some(!)Password", + "username": "admin" +} + +const full_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "hostUrl": { + "title": "Host URL", + "type": "string", + "description": "The base DHIS2 instance URL", + "format": "uri", + "minLength": 1, + "examples": [ + "https://play.dhis2.org/2.36.6" + ] + }, + "username": { + "title": "Username", + "type": "string", + "description": "Username", + "minLength": 1, + "examples": [ + "admin" + ] + }, + "password": { + "title": "Password", + "type": "string", + "description": "Password", + "writeOnly": true, + "minLength": 1, + "examples": [ + "@some(!)Password" + ] + }, + "apiVersion": { + "title": "API Version", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "placeholder": "38", + "description": "DHIS2 api version", + "minLength": 1, + "examples": [ + "v2" + ] + } + }, + "type": "object", + "additionalProperties": true, + "required": [ + "hostUrl", + "password", + "username" + ] +} + +const MOCK_OPTIONS: ConfigOptions = { + logJson: false, + outputPath: '', + outputStdout: true, + adaptor: MOCK_ADAPTOR, +}; + + +// Helper function to load JSON from a file +const loadJSON = async (path: string) => { + try { + const result = await fs.readFile(path, 'utf8'); + if (result) { + return JSON.parse(result); + } + } catch (e) { + return null; + } +}; + +test.beforeEach(() => { + mockfs.restore(); + logger._reset(); + mockfs({ + '/mock/output/path': {}, + }); +}); + +test.serial('fetch and process configuration', async (t) => { + const configData = await configurationHandler(MOCK_OPTIONS, logger); + + const expectedResult = { + sample_config, + full_schema + }; + t.deepEqual(configData, expectedResult); +}); + +test.serial('write configuration to file', async (t) => { + const options: ConfigOptions = { ...MOCK_OPTIONS, outputPath: '/mock/output/path/configuration.json' }; + await configurationHandler(options, logger); + + const writtenData = await loadJSON('/mock/output/path/configuration.json'); + const expectedResult = { + sample_config, + full_schema + }; + t.deepEqual(writtenData, expectedResult); +}); + +test.serial('filter configuration by schema option - schema', async (t) => { + const optionsFull: ConfigOptions = { ...MOCK_OPTIONS, configType: 'schema' }; + + const configDataFull = await configurationHandler(optionsFull, logger); + t.deepEqual(configDataFull, { full_schema }); +}); + +test.serial('filter configuration by schema option - sample', async (t) => { + const optionsSample: ConfigOptions = { ...MOCK_OPTIONS, configType: 'sample' }; + + const configDataSample = await configurationHandler(optionsSample, logger); + t.deepEqual(configDataSample, { sample_config }); +}); + diff --git a/packages/cli/test/configuration/options.test.ts b/packages/cli/test/configuration/options.test.ts new file mode 100644 index 000000000..25c713018 --- /dev/null +++ b/packages/cli/test/configuration/options.test.ts @@ -0,0 +1,49 @@ +import test from 'ava'; +import yargs from 'yargs'; +import configCommand, { ConfigOptions } from '../../src/configuration/command' + + +const cmd = yargs().command(configCommand as any); + +const parse = (command: string) => + cmd.parse(command) as yargs.Arguments; + +test('correct adaptor options', (t) => { + const options = parse('configuration @openfn/language-dhis2@4.0.3'); + + t.is(options.command, 'configuration') + t.is(options.adaptor, '@openfn/language-dhis2@4.0.3') + t.is(options.configType, 'both') + t.is(options.logJson, undefined) + t.is(options.outputPath, undefined) +}) + +test('correct output path options', (t) => { + const options = parse('configuration @openfn/language-dhis2@4.0.3 -o output.json'); + + t.is(options.command, 'configuration') + t.is(options.adaptor, '@openfn/language-dhis2@4.0.3') + t.is(options.configType, 'both') + t.is(options.logJson, undefined) + t.is(options.outputPath, 'output.json') +}) + +test('correct config type', (t) => { + const options = parse('configuration @openfn/language-dhis2@4.0.3 -o output.json --config-type sample'); + + t.is(options.command, 'configuration') + t.is(options.adaptor, '@openfn/language-dhis2@4.0.3') + t.is(options.configType, 'sample') + t.is(options.logJson, undefined) + t.is(options.outputPath, 'output.json') +}) + +test('correct config type for a shortcut command', (t) => { + const options = parse('configuration @openfn/language-dhis2@4.0.3 -o output.json --schema'); + + t.is(options.command, 'configuration') + t.is(options.adaptor, '@openfn/language-dhis2@4.0.3') + t.is(options.configType, 'schema') + t.is(options.logJson, undefined) + t.is(options.outputPath, 'output.json') +}) \ No newline at end of file