diff --git a/.bs-config.js b/.bs-config.js new file mode 100644 index 00000000..67ab68b8 --- /dev/null +++ b/.bs-config.js @@ -0,0 +1,55 @@ +"use strict"; + +/* + |-------------------------------------------------------------------------- + | Browser-sync config file + |-------------------------------------------------------------------------- + | + | For up-to-date information about the options: + | http://www.browsersync.io/docs/options/ + | + | There are more options than you see here, these are just the ones that are + | set internally. See the website for more info. + | + | + */ +module.exports = { + "ui": false, + "files": false, + "watchEvents": [ + "change", + ], + "watch": true, + "watchOptions": { + "ignoreInitial": true, + }, + "server": { + baseDir: "./docs", + routes: { + "/coverage": "./coverage/lcov-report", + }, + }, + "port": 5000, + "ghostMode": false, + "logLevel": "info", + "logPrefix": "peggyjs.org", + "logConnections": false, + "logFileChanges": true, + "logSnippet": true, + "open": "local", + "browser": "default", + "cors": false, + "hostnameSuffix": false, + "reloadOnRestart": true, + "notify": true, + "reloadDebounce": 500, + "injectChanges": true, + "startPath": null, + "minify": false, + "host": null, + "listen": "localhost", + "localOnly": false, + "codeSync": true, + "timestamps": true, + "injectNotification": false, +}; diff --git a/.eslintrc.js b/.eslintrc.js index cd08012f..e23b486b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -5,8 +5,8 @@ module.exports = { extends: "@peggyjs", ignorePatterns: [ "docs/", - "bin/peggy.js", // Generated "lib/parser.js", // Generated + "examples/*.js", // Testing examples "test/vendor/", "test/cli/fixtures/bad.js", // Intentionally-invalid "benchmark/vendor/", @@ -18,7 +18,38 @@ module.exports = { overrides: [ { files: ["rollup.config.js", "*.mjs"], - parserOptions: { sourceType: "module" }, + parserOptions: { + sourceType: "module", + ecmaVersion: 2018, + }, + rules: { + "comma-dangle": ["error", { + arrays: "always-multiline", + objects: "always-multiline", + imports: "always-multiline", + exports: "always-multiline", + functions: "never", + }], + }, + }, + { + files: ["bin/*.js"], + parserOptions: { + // Doesn't have to run in a browser, and Node 10 not supported. + ecmaVersion: 2020, + }, + env: { + node: true, + }, + rules: { + "comma-dangle": ["error", { + arrays: "always-multiline", + objects: "always-multiline", + imports: "always-multiline", + exports: "always-multiline", + functions: "never", + }], + }, }, ], }; diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 49870e26..3b7d6268 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -5,14 +5,14 @@ on: branches: - '*' pull_request: - branches: [main] + branches: ['*'] jobs: build: strategy: matrix: - node-version: [10.x, 12.x, 14.x, 15.x, 16.x] + node-version: [12.x, 14.x, 15.x, 16.x, 18.x] os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} @@ -26,6 +26,7 @@ jobs: - name: Install dependencies run: npm install - name: Check coding standards + if: matrix.node-version == '16.x' && matrix.os == 'ubuntu-latest' run: npm run lint - name: Static analysis - check types run: npm run ts diff --git a/.npmignore b/.npmignore index 57f7a826..a465f514 100644 --- a/.npmignore +++ b/.npmignore @@ -1,14 +1,19 @@ +.bs-config.js .editorconfig .eslintrc-modules.js .eslintrc.js +.gitattributes .github +.nyc_output .vscode/ +*.map benchmark -bin/.eslintrc.json +bin/*.mjs build/ coverage/ docs examples +jest.config.js lib/.eslintrc.json pnpm-lock.yaml rollup.config.js @@ -16,9 +21,5 @@ src test tools tsconfig.json +web-test/ yarn.lock -jest.config.js -.nyc_output -*.map -bin/*.mjs -.gitattributes diff --git a/AUTHORS b/AUTHORS index 57722afe..06715be7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Arlo Breault (https://github.com/arlolra/) Balázs Kutil (https://github.com/bkutil/) Caleb Hearon (https://github.com/chearon/) Charles Pick (https://github.com/phpnode/) +Christian Flach (https://github.com/cmfcmf/) David Berneda Futago-za Ryuu (https://github.com/futagoza/) Jakub Vrana (https://github.com/vrana/) diff --git a/CHANGELOG.md b/CHANGELOG.md index c746a11d..0cd79da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,22 +3,73 @@ Change Log This file documents all notable changes to Peggy. -1.3.0 +2.0.0 ----- Released: TBD +### Major Changes + +- [#163](https://github.com/peggyjs/peggy/pull/163): Add support for + generating source maps, from @Mingun +- [#160](https://github.com/peggyjs/peggy/pull/160): Introduce an API for + reporting errors, warnings and information messages from passes. New API + allows reporting several diagnostics at once with intermediate results + checking after each compilation stage, from @Mingun +- [#218](https://github.com/peggyjs/peggy/pull/218): Add a `sourceMappingURL` + to the generated code, from @hildjj +- [#248](https://github.com/peggyjs/peggy/pull/248): Remove support for + Node.js version 10. When updating dependencies, too many of the tools we + use no longer work on the Node 10, which went out of security maintenance + more than a year ago. Added support for Node.js version 18, from @hildjj +- [#251](https://github.com/peggyjs/peggy/pull/251): Make `commander` and + `source-map-generator` full dependencies. These are not needed for the + pre-packaged web build, but will be used by Node or people that are doing + their own packaging for the web, from @hildjj + ### Minor Changes -- New CLI [@hildjj](https://github.com/peggyjs/peggy/pull/167) +- [#167](https://github.com/peggyjs/peggy/pull/167): New CLI, from @hildjj - Backward compatible with the previous - New -t/--test and -T/--testfile flags to directly test the generated grammar -- Check allowedStartRules for validity [@hildjj](https://github.com/peggyjs/peggy/pull/175) +- [#169](https://github.com/peggyjs/peggy/issues/169): Expose string escape + functions, `stringEscape()` and `regexpClassEscape()`, from @hildjj +- [#175](https://github.com/peggyjs/peggy/pull/175): Check allowedStartRules + for validity, from @hildjj +- [#185](https://github.com/peggyjs/peggy/pull/185): Updated eslint rules, + from @hildjj +- [#196](https://github.com/peggyjs/peggy/pull/196): Add example grammars for + XML and source-mapping, from @hildjj +- [#204](https://github.com/peggyjs/peggy/pull/204): Increase coverage for the + tests, from @Mingun +- [#210](https://github.com/peggyjs/peggy/pull/210): Refactor CLI testing, + from @hildjj ### Bug fixes -- [#164](https://github.com/peggyjs/peggy/pull/164): Fix some errors in the typescript definitions -- [#169](https://github.com/peggyjs/peggy/issues/169): Expose string escape functions +- [#164](https://github.com/peggyjs/peggy/pull/164): Fix some errors in the + typescript definitions, from @Mingun +- [#170](https://github.com/peggyjs/peggy/issues/170): Add + missing argument in function call, from @darlanalves +- [#182](https://github.com/peggyjs/peggy/issues/182): Fix typo in + documentation, from @zargold +- [#197](https://github.com/peggyjs/peggy/pull/197): Fix a regression of + redundant commas in the character classes in the error messages, introduced + in fad4ab74d1de67ef1902cd22d479c81ccab73224, from @Mingun +- [#198](https://github.com/peggyjs/peggy/pull/198): Make all build scripts + run on Windows, from @hildjj +- [#199](https://github.com/peggyjs/peggy/pull/199): Test web version locally, + using puppeteer, from @hildjj +- [#211](https://github.com/peggyjs/peggy/pull/211):Command-line -t requires + from wrong directory, from @hildjj +- [#212](https://github.com/peggyjs/peggy/pull/212): Parse errors with zero + length give badly-formatted errors, from @hildjj +- [#214](https://github.com/peggyjs/peggy/pull/214): Failing tests don't + format errors +- [#216](https://github.com/peggyjs/peggy/issues/216): Fix typescript + definition of SyntaxError, from @cmfcmf +- [#220](https://github.com/peggyjs/peggy/issues/220): Fix rollup warnings, + from @hildjj 1.2.0 ----- diff --git a/LICENSE b/LICENSE index 7d0c286f..4db11507 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2010-2021 The Peggy AUTHORS +Copyright (c) 2010-2022 The Peggy AUTHORS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 32d25ca7..204a7d84 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Follow these steps to upgrade: — more powerful than traditional LL(_k_) and LR(_k_) parsers - Usable [from your browser](https://peggyjs.org/online), from the command line, or via JavaScript API +- [Source map](https://developer.mozilla.org/en-US/docs/Tools/Debugger/How_to/Use_a_source_map) support ## Getting Started @@ -120,13 +121,16 @@ You can tweak the generated parser with several options: name with extension changed to `.js`, or stdout if no input file is given. - `--plugin ` — makes Peggy use a specified plugin (can be specified multiple times) -- `-t`, `--test ` - Test the parser with the given text, outputting the - result of running the parser against this input -- `-T`, `--test-file ` - Test the parser with the contents of the - given file, outputting the result of running the parser against this input +- `-t`, `--test ` — Test the parser with the given text, outputting the + result of running the parser against this input. + If the input to be tested is not parsed, the CLI will exit with code 2 +- `-T`, `--test-file ` — Test the parser with the contents of the + given file, outputting the result of running the parser against this input. + If the input to be tested is not parsed, the CLI will exit with code 2 +- `--source-map` — generate a source map file with an optionally specified name - `--trace` — makes the parser trace its progress -- `-v`, `--version` - output the version number -- `-h`, `--help` - display help for command +- `-v`, `--version` — output the version number +- `-h`, `--help` — display help for command If you specify options using `-c ` or `--extra-options-file `, you will need to ensure you are using the correct types. In particular, you may @@ -150,6 +154,50 @@ module.exports = { }; ``` +You can test generated parser immediately if you specify the `-t/--test` or `-T/--test-file` +option. This option conflicts with the option `-m/--source-map` unless `-o/--output` is +also specified. + +The CLI will exit with the code: +- `0` if all was success +- `1` if you supply incorrect or conflicting parameters +- `2` if all parameters is correct, you specify the `-t/--test` or `-T/--test-file` option + and specified input does not parsed with the specified grammar + +Examples: + +```console +# - write test results to stdout (42) +# - exit with the code 0 +echo "foo = '1' { return 42 }" | peggy --test 1 + +# - write a parser error to stdout (Expected "1" but "2" found) +# - exit with the code 2 +echo "foo = '1' { return 42 }" | peggy --test 2 + +# - write an error to stdout (Generation of the source map is useless if you don't +# store a generated parser code, perhaps you forgot to add an `-o/--output` option?) +# - exit with the code 1 +echo "foo = '1' { return 42 }" | peggy --source-map --test 1 + +# - write an error to stdout (Generation of the source map is useless if you don't +# store a generated parser code, perhaps you forgot to add an `-o/--output` option?) +# - exit with the code 1 +echo "foo = '1' { return 42 }" | peggy --source-map --test 2 + +# - write an output to `parser.js`, +# - write a source map to `parser.js.map` +# - write test results to stdout (42) +# - exit with the code 0 +echo "foo = '1' { return 42 }" | peggy --output parser.js --source-map --test 1 + +# - write an output to `parser.js`, +# - write a source map to `parser.js.map` +# - write a parser error to stdout (Expected "1" but "2" found) +# - exit with the code 2 +echo "foo = '1' { return 42 }" | peggy --output parser.js --source-map --test 2 +``` + ### JavaScript API In Node.js, require the Peggy parser generator module: @@ -196,6 +244,7 @@ object to `peg.generate`. The following options are supported: _global initializer_ and the _per-parse initializer_. Unless the parser is to be generated in different formats, it is recommended to rather import dependencies from within the _global initializer_. (default: `{}`) +- `error` — a callback for errors. See [Error Reporting](#error-reporting) - `exportVar` — name of a global variable into which the parser object is assigned to when no module loader is detected; valid only when `format` is set to `"globals"` or `"umd"` (default: `null`) @@ -206,11 +255,42 @@ object to `peg.generate`. The following options are supported: `source` property (default: `undefined`). This object will be used even if `options.grammarSource` is redefined in the grammar. It is useful to attach the file information to the errors, for example +- `info` — a callback for informational messages. See [Error Reporting](#error-reporting) - `output` — if set to `"parser"`, the method will return generated parser - object; if set to `"source"`, it will return parser source code as a string + object; if set to `"source"`, it will return parser source code as a string. + If set to `"source-and-map"`, it will return a [`SourceNode`] object; you can + get source code by calling `toString()` method or source code and mapping by + calling `toStringWithSourceMap()` method, see the [`SourceNode`] documentation (default: `"parser"`) + + > **Note**: because of bug [source-map/444] you should also set `grammarSource` to + > a not-empty string if you set this value to `"source-and-map"` - `plugins` — plugins to use. See the [Plugins API](#plugins-api) section - `trace` — makes the parser trace its progress (default: `false`) +- `warning` — a callback for warnings. See [Error Reporting](#error-reporting) + +#### Error Reporting + +While generating the parser, the compiler may throw a `GrammarError` which collects +all of the issues that were seen. + +There is also another way to collect problems as fast as they are reported — +register one or more of these callbacks: + +- `error(stage: Stage, message: string, location?: LocationRange, notes?: DiagnosticNote[]): void` +- `warning(stage: Stage, message: string, location?: LocationRange, notes?: DiagnosticNote[]): void` +- `info(stage: Stage, message: string, location?: LocationRange, notes?: DiagnosticNote[]): void` + +All parameters are the same as the parameters of the [reporting API](#session-api) +except the first. The `stage` represent one of possible stages during which execution +a diagnostic was generated. This is a string enumeration, that currently has one of +three values: +- `check` +- `transform` +- `generate` + +[`SourceNode`]: https://github.com/mozilla/source-map#sourcenode +[source-map/444]: https://github.com/mozilla/source-map/issues/444 ## Using the Parser @@ -707,6 +787,10 @@ note: Step 3: call itself without input consumption - left recursion | ^^^^^ ``` +A plugin may register additional passes that can generate `GrammarError`s to report +about problems, but they shouldn't do that by throwing an instance of `GrammarError`. +They should use a [session API](#session-api) instead. + ## Locations During the parsing you can access to the information of the current parse location, @@ -801,12 +885,14 @@ method. the AST, add or remove nodes or their properties - `generate` — passes used for actual code generating - A plugin that implement a pass usually should push it to the end of one of that - arrays. Pass is a simple function with signature `pass(ast, options)`: + A plugin that implement a pass usually should push it to the end of the correct + array. Pass is a simple function with signature `pass(ast, options, session)`: - `ast` — the AST created by the `config.parser.parse()` method - `options` — compilation options passed to the `peggy.compiler.compile()` method. If parser generation is started because `generate()` function was called that is also an options, passed to the `generate()` method + - `session` — a [`Session`](#session-api) object that allows raising errors, + warnings and informational messages - `reservedWords` — string array with a list of words that shouldn't be used as label names. This list can be modified by plugins. That property is not required to be sorted or not contain duplicates, but it is recommend to remove duplicates. @@ -816,12 +902,63 @@ method. - `options` — build options passed to the `generate()` method. A best practice for a plugin would look for its own options under a `` key. +### Session API + +Each compilation request is represented by a `Session` instance. An object of this class +is created by the compiler and passed to an each pass as a 3rd parameter. The session +object gives access to the various compiler services. At the present time there is only +one such service: reporting of diagnostics. + +All diagnostics are divided into three groups: errors, warnings and informational +messages. For each of them the `Session` object has a method, described below. + +All reporting methods have an identical signature: + +```typescript +(message: string, location?: LocationRange, notes?: DiagnosticNote[]) => void; +``` + +- `message`: a main diagnostic message +- `location`: an optional location information if diagnostic is related to the grammar + source code +- `notes`: an array with additional details about diagnostic, pointing to the + different places in the grammar. For example, each note could be a location of + a duplicated rule definition + +#### `error(...)` + +Reports an error. Compilation process is subdivided into pieces called _stages_ and +each stage consist of one or more _passes_. Within the one stage all errors, reported +by different passes, are collected without interrupting the parsing process. + +When all passes in the stage are completed, the stage is checked for errors. If one +was registered, a `GrammarError` with all found problems in the `problems` property +is thrown. If there are no errors, then the next stage is processed. + +After processing all three stages (`check`, `transform` and `generate`) the compilation +process is finished. + +The process, described above, means that passes should be careful about what they do. +For example, if you place your pass into the `check` stage there is no guarantee that +all rules exists, because checking for existing rules is also performed during the +`check` stage. On the contrary, passes in the `transform` and `generate` stages can be +sure that all rules exists, because that precondition was checked on the `check` stage. + +#### `warning(...)` + +Reports a warning. Warnings are similar to errors, but they do not interrupt a compilation. + +#### `info(...)` + +Report an informational message. This method can be used to inform user about significant +changes in the grammar, for example, replacing proxy rules. + ## Compatibility Both the parser generator and generated parsers should run well in the following environments: -- Node.js 10+ +- Node.js 12+ - Internet Explorer 9+ - Edge - Firefox diff --git a/bin/.eslintrc.json b/bin/.eslintrc.json deleted file mode 100644 index 7ff0b31e..00000000 --- a/bin/.eslintrc.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "env": { - "node": true - }, - "rules": { - "no-console": 0, - "prefer-object-spread": 0 - } -} diff --git a/bin/peggy-cli.js b/bin/peggy-cli.js new file mode 100644 index 00000000..9d43d980 --- /dev/null +++ b/bin/peggy-cli.js @@ -0,0 +1,684 @@ +"use strict"; + +const { + Command, CommanderError, InvalidArgumentError, Option, +} = require("commander"); +const { Module } = require("module"); +const fs = require("fs"); +const path = require("path"); +const peggy = require("../lib/peg.js"); +const util = require("util"); +const vm = require("vm"); + +exports.CommanderError = CommanderError; +exports.InvalidArgumentError = InvalidArgumentError; + +// Options that aren't for the API directly: +const PROG_OPTIONS = ["input", "output", "sourceMap", "test", "testFile", "verbose"]; +const MODULE_FORMATS = ["amd", "bare", "commonjs", "es", "globals", "umd"]; +const MODULE_FORMATS_WITH_DEPS = ["amd", "commonjs", "es", "umd"]; +const MODULE_FORMATS_WITH_GLOBAL = ["globals", "umd"]; + +// Helpers + +function select(obj, sel) { + const ret = {}; + for (const s of sel) { + if (Object.prototype.hasOwnProperty.call(obj, s)) { + ret[s] = obj[s]; + delete obj[s]; + } + } + return ret; +} + +function commaArg(val, prev) { + return (prev || []).concat(val.split(",").map(x => x.trim())); +} + +// Files + +function readStream(inputStream) { + return new Promise((resolve, reject) => { + const input = []; + inputStream.on("data", data => { input.push(data); }); + inputStream.on("end", () => resolve(Buffer.concat(input).toString())); + inputStream.on("error", reject); + }); +} + +function readFile(name) { + let f = null; + try { + f = fs.readFileSync(name, "utf8"); + } catch (e) { + throw new InvalidArgumentError(`Can't read from file "${name}".`, e); + } + return f; +} + +/** + * @typedef {object} Stdio + * @property {stream.Readable} [in] StdIn. + * @property {stream.Writable} [out] StdOut. + * @property {stream.Writable} [err] StdErr. + */ + +// Command line processing +class PeggyCLI extends Command { + /** + * Create a CLI environment. + * + * @param {Stdio} [stdio] Replacement streams for stdio, for testing. + */ + constructor(stdio) { + super("peggy"); + + /** @type {Stdio} */ + this.std = { + in: process.stdin, + out: process.stdout, + err: process.stderr, + ...stdio, + }; + + /** @type {peggy.BuildOptionsBase} */ + this.argv = {}; + this.colors = this.std.err.isTTY; + /** @type {string?} */ + this.inputFile = null; + /** @type {string?} */ + this.outputFile = null; + /** @type {object} */ + this.progOptions = {}; + /** @type {string?} */ + this.testFile = null; + /** @type {string?} */ + this.testGrammarSource = null; + /** @type {string?} */ + this.testText = null; + /** @type {string?} */ + this.outputJS = null; + + this + .version(peggy.VERSION, "-v, --version") + .argument("[input_file]", 'Grammar file to read. Use "-" to read stdin.', "-") + .allowExcessArguments(false) + .addOption( + new Option( + "--allowed-start-rules ", + "Comma-separated list of rules the generated parser will be allowed to start parsing from. (Can be specified multiple times)" + ) + .default([], "the first rule in the grammar") + .argParser(commaArg) + ) + .option( + "--cache", + "Make generated parser cache results", + false + ) + .option( + "-d, --dependency ", + "Comma-separated list of dependencies, either as a module name, or as `variable:module`. (Can be specified multiple times)", + commaArg + ) + .option( + "-D, --dependencies ", + "Dependencies, in JSON object format with variable:module pairs. (Can be specified multiple times).", + (val, prev = {}) => { + let v = null; + try { + v = JSON.parse(val); + } catch (e) { + throw new InvalidArgumentError(`Error parsing JSON: ${e.message}`); + } + return Object.assign(prev, v); + } + ) + .option( + "-e, --export-var ", + "Name of a global variable into which the parser object is assigned to when no module loader is detected." + ) + .option( + "--extra-options ", + "Additional options (in JSON format as an object) to pass to peggy.generate", + val => this.addExtraOptionsJSON(val, "extra-options") + ) + .option( + "-c, --extra-options-file ", + "File with additional options (in JSON as an object or commonjs module format) to pass to peggy.generate", + val => { + if (/\.c?js$/.test(val)) { + return this.addExtraOptions(require(path.resolve(val)), "extra-options-file"); + } else { + return this.addExtraOptionsJSON(readFile(val), "extra-options-file"); + } + } + ) + .addOption( + new Option( + "--format ", + "Format of the generated parser" + ) + .choices(MODULE_FORMATS) + .default("commonjs") + ) + .option("-o, --output ", "Output file for generated parser. Use '-' for stdout (the default, unless a test is specified, in which case no parser is output without this option)") + .option( + "--plugin ", + "Comma-separated list of plugins. (can be specified multiple times)", + commaArg + ) + .option( + "-m, --source-map [mapfile]", + "Generate a source map. If name is not specified, the source map will be named \".map\" if input is a file and \"source.map\" if input is a standard input. If the special filename `inline` is given, the sourcemap will be embedded in the output file as a data URI. If the filename is prefixed with `hidden:`, no mapping URL will be included so that the mapping can be specified with an HTTP SourceMap: header. This option conflicts with the `-t/--test` and `-T/--test-file` options unless `-o/--output` is also specified" + ) + .option( + "-t, --test ", + "Test the parser with the given text, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2" + ) + .option( + "-T, --test-file ", + "Test the parser with the contents of the given file, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2" + ) + .option("--trace", "Enable tracing in generated parser", false) + .addOption( + // Not interesting yet. If it becomes so, unhide the help. + new Option("--verbose", "Enable verbose logging") + .hideHelp() + .default(false) + ) + .addOption( + new Option("-O, --optimize