From c6a6933e877ceddc32c3b6781156b4ddd34cf304 Mon Sep 17 00:00:00 2001 From: Adriel Perkins Date: Mon, 20 Jun 2022 16:32:00 -0400 Subject: [PATCH] feat: initial commit of schemacheck code and tests --- .gitignore | 4 ++ .goreleaser.yaml | 31 +++++++++ Makefile | 24 +++++++ README.md | 26 ++++++++ go.mod | 18 +++++ go.sum | 22 +++++++ schemacheck.go | 145 +++++++++++++++++++++++++++++++++++++++++ schemacheck_test.go | 117 +++++++++++++++++++++++++++++++++ test_data/invalid.json | 9 +++ test_data/invalid.yaml | 6 ++ test_data/noseparator | 0 test_data/schema.json | 35 ++++++++++ test_data/values.json | 9 +++ test_data/values.txt | 9 +++ test_data/values.yaml | 6 ++ test_data/values.yml | 6 ++ 16 files changed, 467 insertions(+) create mode 100644 .goreleaser.yaml create mode 100644 Makefile create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 schemacheck.go create mode 100644 schemacheck_test.go create mode 100644 test_data/invalid.json create mode 100644 test_data/invalid.yaml create mode 100644 test_data/noseparator create mode 100644 test_data/schema.json create mode 100644 test_data/values.json create mode 100644 test_data/values.txt create mode 100644 test_data/values.yaml create mode 100644 test_data/values.yml diff --git a/.gitignore b/.gitignore index 66fd13c9..84f3098e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# General folders +.vscode/ +dist/ + # Binaries for programs and plugins *.exe *.exe~ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 00000000..397eed86 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,31 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + - make tidy + - go generate ./... + - make checks +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin +archives: + - replacements: + darwin: Darwin + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..dec930c7 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +# Makefile +INSTALL_PATH ?= /usr/local/bin +BIN_NAME ?= schemacheck +BINDIR := $(CURDIR)/bin + +.PHONY: tidy build test checks clean + +default: build + +tidy: + @go mod tidy + +build: + @goreleaser build --rm-dist --skip-validate + +test: + @go test -v + +checks: + @go fmt ./... + @go vet ./... + @staticcheck ./... + @gosec ./... + @goimports -w ./ diff --git a/README.md b/README.md new file mode 100644 index 00000000..1d88dd54 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# schemacheck +A CLI utility written in [go](go.dev) that validates `json` and `yaml` files +against a `schema`. + +## Install +There are a few different methods to install `schemacheck`. + +### Via `go` (Recommended) +* Run `go install github.com/adrielp/schemacheck` + +### Mac/Linux during local development +* Clone down this repository and run `make build` +* Install a binary for your platform from `dist/bin` locally to a path + + +### Windows +There's a binary for that, but it's not directly supported or tested because #windows + +## Getting Started +### Prereqs +* Have [make](https://www.gnu.org/software/make/) installed +* Have [GoReleaser](https://goreleaser.com/) installed + +### Instructions +* Clone down this repository +* Run commands in the [Makefile](./Makefile) like `make build` diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..916c874f --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module github.com/adrielp/schemacheck + +go 1.18 + +require ( + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.3.0 + github.com/xeipuuv/gojsonschema v1.2.0 + sigs.k8s.io/yaml v1.3.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..e015bfba --- /dev/null +++ b/go.sum @@ -0,0 +1,22 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/schemacheck.go b/schemacheck.go new file mode 100644 index 00000000..1c53c7dc --- /dev/null +++ b/schemacheck.go @@ -0,0 +1,145 @@ +package main + +import ( + "errors" + "log" + "os" + "path/filepath" + "strings" + + flag "github.com/spf13/pflag" + "github.com/xeipuuv/gojsonschema" + "sigs.k8s.io/yaml" +) + +// set default constants for usage messages and default file names +const ( + defaultSchema = "test_data/schema.json" + schemaUsage = "A valid JSON schema file to use for validation. Default: schema.json" + + defaultFileName = "test_data/values.json" + fileUsage = "A Yaml or JSON file to check against a given schema. Default: values.json (can acceptable multiples)" +) + +// Gloval variables for flags and logger +var ( + // Core flag variables + File []string + Schema string + + // Info and Error loggers + logger = log.New(os.Stderr, "INFO: ", log.Lshortfile) + errLogger = log.New(os.Stderr, "ERROR: ", log.Lshortfile) +) + +// initialize the flags from the command line and their shorthand counterparts +func init() { + defaultFile := []string{defaultFileName} + flag.StringVarP(&Schema, "schema", "s", defaultSchema, schemaUsage) + flag.StringSliceVarP(&File, "file", "f", defaultFile, fileUsage) +} + +// Checks whether a given file is of the supported extension type and if not +// returns false with an error. +// Valid file extensions are currently .yaml, .yml, and .json +func CheckFileIsSupported(file string, fileExt string) (bool, error) { + // default to false + fileValid := false + + // supported file extensions to check + supportedTypes := []string{"yaml", "yml", "json"} + + for _, ext := range supportedTypes { + if strings.HasSuffix(file, ext) { + logger.Printf("File: \"%s\" has valid file extension: \"%s\"", file, ext) + fileValid = true + } + } + + if !fileValid { + return fileValid, errors.New("file type not supported") + } + + return fileValid, nil + +} + +func GetFileExt(file string) (string, error) { + _, fileExt, found := strings.Cut(file, ".") + if !found { + return "", errors.New("file separator not found") + } + + return fileExt, nil +} + +func Validate(file string, fileExt string, loadedSchema gojsonschema.JSONLoader) error { + data, err := os.ReadFile(filepath.Clean(file)) + if err != nil { + errLogger.Panicf("Could not read file: '%s' cleanly.", file) + } + + if fileExt == "yaml" || fileExt == "yml" { + data, err = yaml.YAMLToJSON(data) + if err != nil { + logger.Panicf("Failed to convert yaml to json in yaml file %s", file) + } + } + + documentLoader := gojsonschema.NewBytesLoader(data) + + // Validate the JSON data against the loaded JSON Schema + result, err := gojsonschema.Validate(loadedSchema, documentLoader) + if err != nil { + errLogger.Printf("There was a problem validating %s", file) + logger.Panicf(err.Error()) + } + + // Check the validity of the result and throw a message is the document is valid or if it's not with errors. + if result.Valid() { + logger.Printf("%s is a valid document.\n", file) + } else { + logger.Printf("%s is not a valid document...\n", file) + for _, desc := range result.Errors() { + errLogger.Printf("--- %s\n", desc) + } + return errors.New("document not valid") + } + + return nil +} + +func main() { + + // parse the flags set in the init() function + flag.Parse() + + // Load schema file before running through and validating the other files to + // reduce how many times it's loaded. + schema, err := os.ReadFile(filepath.Clean(Schema)) + if err != nil { + errLogger.Panicf("Could not read schema file: '%s' cleanly.", Schema) + } + loadedSchema := gojsonschema.NewBytesLoader(schema) + + // Iterate through the files declared in the arguments and run validations + for _, file := range File { + // Create a specific logger with an ERROR message for easy readability. + + // Print out the values passed on the command line + logger.Printf("Validating %s file against %s schema...", file, Schema) + + // Get the file extension and error if it failed + fileExt, err := GetFileExt(file) + if err != nil { + errLogger.Panicf(err.Error()) + } + + // Pass the file name and extension to ensure it's a supported file type + if _, err := CheckFileIsSupported(file, fileExt); err != nil { + errLogger.Panicf(err.Error()) + } + + Validate(file, fileExt, loadedSchema) + } +} diff --git a/schemacheck_test.go b/schemacheck_test.go new file mode 100644 index 00000000..782a579c --- /dev/null +++ b/schemacheck_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/xeipuuv/gojsonschema" +) + +// =============================================== +// Tests against the CheckFileIsSupported function for various file types +func TestCheckFileIsSupportedYaml(t *testing.T) { + file := "test_data/values.yaml" + valid, err := CheckFileIsSupported(file, "yaml") + if err != nil { + t.Fatalf("An error occured during validation of file that should not have occurred:\n %s", err) + } + assert.True(t, valid) +} + +func TestCheckFileIsSupportedYml(t *testing.T) { + file := "test_data/values.yml" + valid, err := CheckFileIsSupported(file, "yml") + if err != nil { + t.Fatalf("An error occured during validation of file that should not have occurred:\n %s", err) + } + assert.True(t, valid) +} + +func TestCheckFileIsSupportedJSON(t *testing.T) { + file := "test_data/values.json" + valid, err := CheckFileIsSupported(file, "json") + if err != nil { + t.Fatalf("An error occured during validation of file that should not have occurred:\n %s", err) + } + assert.True(t, valid) +} + +func TestCheckFileIsSupportedTxt(t *testing.T) { + file := "test_data/values.txt" + _, err := CheckFileIsSupported(file, "txt") + assert.Error(t, err) +} + +// =============================================== +// Tests GetFileExt function for various filetypes +func TestGetFileExtYaml(t *testing.T) { + file := "test_data/values.yaml" + fileExt, _ := GetFileExt(file) + assert.Equal(t, "yaml", fileExt) +} + +func TestGetFileExtYml(t *testing.T) { + file := "test_data/values.yml" + fileExt, _ := GetFileExt(file) + assert.Equal(t, "yml", fileExt) +} + +func TestGetFileExtJSON(t *testing.T) { + file := "test_data/values.json" + fileExt, _ := GetFileExt(file) + assert.Equal(t, "json", fileExt) +} + +func TestGetFileExtNoSeparator(t *testing.T) { + file := "test_data/noseparator" + _, err := GetFileExt(file) + assert.Error(t, err) +} + +// =============================================== +// Tests Validate against test data files +func TestValidateValidYaml(t *testing.T) { + file := "test_data/values.yaml" + fileExt := "yaml" + schema, err := os.ReadFile(filepath.Clean("test_data/schema.json")) + if err != nil { + errLogger.Panicf("Could not read schema file: '%s' cleanly.", Schema) + } + loadedSchema := gojsonschema.NewBytesLoader(schema) + assert.NoError(t, Validate(file, fileExt, loadedSchema)) +} + +func TestValidateValidJSON(t *testing.T) { + file := "test_data/values.json" + fileExt := "yaml" + schema, err := os.ReadFile(filepath.Clean("test_data/schema.json")) + if err != nil { + errLogger.Panicf("Could not read schema file: '%s' cleanly.", Schema) + } + loadedSchema := gojsonschema.NewBytesLoader(schema) + assert.NoError(t, Validate(file, fileExt, loadedSchema)) +} + +func TestValidateInvalidYaml(t *testing.T) { + file := "test_data/invalid.yaml" + fileExt := "yaml" + schema, err := os.ReadFile(filepath.Clean("test_data/schema.json")) + if err != nil { + errLogger.Panicf("Could not read schema file: '%s' cleanly.", Schema) + } + loadedSchema := gojsonschema.NewBytesLoader(schema) + assert.Error(t, Validate(file, fileExt, loadedSchema)) +} + +func TestValidateInvalidJSON(t *testing.T) { + file := "test_data/invalid.json" + fileExt := "yaml" + schema, err := os.ReadFile(filepath.Clean("test_data/schema.json")) + if err != nil { + errLogger.Panicf("Could not read schema file: '%s' cleanly.", Schema) + } + loadedSchema := gojsonschema.NewBytesLoader(schema) + assert.Error(t, Validate(file, fileExt, loadedSchema)) +} diff --git a/test_data/invalid.json b/test_data/invalid.json new file mode 100644 index 00000000..3ddca9f7 --- /dev/null +++ b/test_data/invalid.json @@ -0,0 +1,9 @@ +{ + "key1": 1091, + "key2": { + "name": "Vader4Ever", + "id": "InvalidVader", + "coolOrNot": true + }, + "key3": 100 +} diff --git a/test_data/invalid.yaml b/test_data/invalid.yaml new file mode 100644 index 00000000..bd653d48 --- /dev/null +++ b/test_data/invalid.yaml @@ -0,0 +1,6 @@ +key1: true +key2: + name: Vader4Ever + id: Vaderrrrrrrr + coolOrNot: true +key3: 100 diff --git a/test_data/noseparator b/test_data/noseparator new file mode 100644 index 00000000..e69de29b diff --git a/test_data/schema.json b/test_data/schema.json new file mode 100644 index 00000000..bbe93aa0 --- /dev/null +++ b/test_data/schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Environment", + "description": "A JSON File.", + "type": "object", + "properties": { + "key1": { + "description": "Super cool key name", + "type": "string" + }, + "key2": { + "type": "object", + "properties": { + "name": { + "description": "Super cool name", + "type": "string" + }, + "id": { + "description": "Super cool ID number", + "type": "integer" + }, + "coolOrNot": { + "description": "Whether or not we're cool", + "type": "boolean" + } + } + }, + "key3": { + "description": "Coolness factor", + "type": "integer" + } + }, + "required": ["key1", "key2"] +} diff --git a/test_data/values.json b/test_data/values.json new file mode 100644 index 00000000..1bf89583 --- /dev/null +++ b/test_data/values.json @@ -0,0 +1,9 @@ +{ + "key1": "SithLord1", + "key2": { + "name": "Vader4Ever", + "id": 1, + "coolOrNot": true + }, + "key3": 100 +} diff --git a/test_data/values.txt b/test_data/values.txt new file mode 100644 index 00000000..1bf89583 --- /dev/null +++ b/test_data/values.txt @@ -0,0 +1,9 @@ +{ + "key1": "SithLord1", + "key2": { + "name": "Vader4Ever", + "id": 1, + "coolOrNot": true + }, + "key3": 100 +} diff --git a/test_data/values.yaml b/test_data/values.yaml new file mode 100644 index 00000000..38b76300 --- /dev/null +++ b/test_data/values.yaml @@ -0,0 +1,6 @@ +key1: SithLord1 +key2: + name: Vader4Ever + id: 1 + coolOrNot: true +key3: 100 diff --git a/test_data/values.yml b/test_data/values.yml new file mode 100644 index 00000000..38b76300 --- /dev/null +++ b/test_data/values.yml @@ -0,0 +1,6 @@ +key1: SithLord1 +key2: + name: Vader4Ever + id: 1 + coolOrNot: true +key3: 100