Skip to content

Commit

Permalink
Mark : as token
Browse files Browse the repository at this point in the history
  • Loading branch information
@oovm
oovm committed Nov 28, 2023
1 parent f651cde commit ea5dcd6
Show file tree
Hide file tree
Showing 9 changed files with 647 additions and 455 deletions.
154 changes: 93 additions & 61 deletions projects/json5-parser/src/codegen/debug.ron
View file

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion projects/json5-parser/src/codegen/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#![allow(clippy::unnecessary_cast)]
#![doc = include_str!("readme.md")]

mod parse_cst;
mod parse_ast;
mod parse_cst;

use core::str::FromStr;
use std::{borrow::Cow, ops::Range, sync::OnceLock};
Expand Down Expand Up @@ -33,6 +33,7 @@ pub enum Json5Rule {
Value,
Object,
ObjectPair,
COLON,
ObjectKey,
Array,
String,
Expand All @@ -56,6 +57,7 @@ impl YggdrasilRule for Json5Rule {
Self::Value => "",
Self::Object => "",
Self::ObjectPair => "",
Self::COLON => "",
Self::ObjectKey => "",
Self::Array => "",
Self::String => "",
Expand Down Expand Up @@ -88,12 +90,18 @@ pub struct ObjectNode {
#[derive(Clone, Debug, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ObjectPairNode {
pub colon: ColonNode,
pub object_key: ObjectKeyNode,
pub value: ValueNode,
pub span: Range<usize>,
}
#[derive(Clone, Debug, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ColonNode {
pub span: Range<usize>,
}
#[derive(Clone, Debug, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ObjectKeyNode {
Identifier(IdentifierNode),
String(StringNode),
Expand Down
49 changes: 31 additions & 18 deletions projects/json5-parser/src/codegen/parse_ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ impl YggdrasilNode for ObjectNode {
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
object_pair: pair.take_tagged_items::<ObjectPairNode>(Cow::Borrowed("object_pair")).collect::<Result<Vec<_>, _>>()?,
object_pair: pair
.take_tagged_items::<ObjectPairNode>(Cow::Borrowed("object_pair"))
.collect::<Result<Vec<_>, _>>()?,
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
}
Expand All @@ -77,6 +79,7 @@ impl YggdrasilNode for ObjectPairNode {
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
colon: pair.take_tagged_one::<ColonNode>(Cow::Borrowed("colon"))?,
object_key: pair.take_tagged_one::<ObjectKeyNode>(Cow::Borrowed("object_key"))?,
value: pair.take_tagged_one::<ValueNode>(Cow::Borrowed("value"))?,
span: Range { start: _span.start() as usize, end: _span.end() as usize },
Expand All @@ -92,6 +95,26 @@ impl FromStr for ObjectPairNode {
}
}
#[automatically_derived]
impl YggdrasilNode for ColonNode {
type Rule = Json5Rule;

fn get_range(&self) -> Range<usize> {
Range { start: self.span.start as usize, end: self.span.end as usize }
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
impl FromStr for ColonNode {
type Err = YggdrasilError<Json5Rule>;

fn from_str(input: &str) -> Result<Self, YggdrasilError<Json5Rule>> {
Self::from_cst(Json5Parser::parse_cst(input, Json5Rule::COLON)?)
}
}
#[automatically_derived]
impl YggdrasilNode for ObjectKeyNode {
type Rule = Json5Rule;

Expand Down Expand Up @@ -175,9 +198,7 @@ impl YggdrasilNode for StringEscapedNode {
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
Expand All @@ -197,9 +218,7 @@ impl YggdrasilNode for NumberNode {
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
Expand All @@ -223,10 +242,10 @@ impl YggdrasilNode for BooleanNode {
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
if let Some(_) = pair.find_first_tag("false") {
return Ok(Self::False)
return Ok(Self::False);
}
if let Some(_) = pair.find_first_tag("true") {
return Ok(Self::True)
return Ok(Self::True);
}
Err(YggdrasilError::invalid_node(Json5Rule::Boolean, _span))
}
Expand All @@ -248,9 +267,7 @@ impl YggdrasilNode for NullNode {
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
Expand All @@ -270,9 +287,7 @@ impl YggdrasilNode for IdentifierNode {
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
Expand All @@ -292,9 +307,7 @@ impl YggdrasilNode for WhiteSpaceNode {
}
fn from_pair(pair: TokenPair<Self::Rule>) -> Result<Self, YggdrasilError<Self::Rule>> {
let _span = pair.get_span();
Ok(Self {
span: Range { start: _span.start() as usize, end: _span.end() as usize },
})
Ok(Self { span: Range { start: _span.start() as usize, end: _span.end() as usize } })
}
}
#[automatically_derived]
Expand Down
152 changes: 133 additions & 19 deletions projects/json5-parser/src/codegen/parse_cst.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ pub(super) fn parse_cst(input: &str, rule: Json5Rule) -> OutputResult<Json5Rule>
Json5Rule::Value => parse_value(state),
Json5Rule::Object => parse_object(state),
Json5Rule::ObjectPair => parse_object_pair(state),
Json5Rule::COLON => parse_colon(state),
Json5Rule::ObjectKey => parse_object_key(state),
Json5Rule::Array => parse_array(state),
Json5Rule::String => parse_string(state),
Expand All @@ -20,82 +21,195 @@ pub(super) fn parse_cst(input: &str, rule: Json5Rule) -> OutputResult<Json5Rule>
#[inline]
fn parse_value(state: Input) -> Output {
state.rule(Json5Rule::Value, |s| {
Err(s).or_else(|s|parse_object(s).and_then(|s| s.tag_node("object"))).or_else(|s|parse_array(s).and_then(|s| s.tag_node("array"))).or_else(|s|parse_string(s).and_then(|s| s.tag_node("string"))).or_else(|s|parse_number(s).and_then(|s| s.tag_node("number"))).or_else(|s|parse_boolean(s).and_then(|s| s.tag_node("boolean"))).or_else(|s|parse_null(s).and_then(|s| s.tag_node("null")))
Err(s)
.or_else(|s| parse_object(s).and_then(|s| s.tag_node("object")))
.or_else(|s| parse_array(s).and_then(|s| s.tag_node("array")))
.or_else(|s| parse_string(s).and_then(|s| s.tag_node("string")))
.or_else(|s| parse_number(s).and_then(|s| s.tag_node("number")))
.or_else(|s| parse_boolean(s).and_then(|s| s.tag_node("boolean")))
.or_else(|s| parse_null(s).and_then(|s| s.tag_node("null")))
})
}
#[inline]
fn parse_object(state: Input) -> Output {
state.rule(Json5Rule::Object, |s| {
s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,"{",false)).and_then(|s|builtin_ignore(s)).and_then(|s|s.optional(|s|s.sequence(|s|Ok(s).and_then(|s|parse_object_pair(s).and_then(|s| s.tag_node("object_pair"))).and_then(|s|builtin_ignore(s)).and_then(|s|s.repeat(0..4294967295,|s|s.sequence(|s|Ok(s).and_then(|s|builtin_ignore(s)).and_then(|s|s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,",",false)).and_then(|s|builtin_ignore(s)).and_then(|s|parse_object_pair(s).and_then(|s| s.tag_node("object_pair")))))))).and_then(|s|builtin_ignore(s)).and_then(|s|builtin_text(s,",",false))))).and_then(|s|builtin_ignore(s)).and_then(|s|builtin_text(s,"}",false)))
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, "{", false))
.and_then(|s| builtin_ignore(s))
.and_then(|s| {
s.optional(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| parse_object_pair(s).and_then(|s| s.tag_node("object_pair")))
.and_then(|s| builtin_ignore(s))
.and_then(|s| {
s.repeat(0..4294967295, |s| {
s.sequence(|s| {
Ok(s).and_then(|s| builtin_ignore(s)).and_then(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, ",", false))
.and_then(|s| builtin_ignore(s))
.and_then(|s| {
parse_object_pair(s).and_then(|s| s.tag_node("object_pair"))
})
})
})
})
})
})
.and_then(|s| builtin_ignore(s))
.and_then(|s| builtin_text(s, ",", false))
})
})
})
.and_then(|s| builtin_ignore(s))
.and_then(|s| builtin_text(s, "}", false))
})
})
}
#[inline]
fn parse_object_pair(state: Input) -> Output {
state.rule(Json5Rule::ObjectPair, |s| {
s.sequence(|s|Ok(s).and_then(|s|parse_object_key(s).and_then(|s| s.tag_node("object_key"))).and_then(|s|builtin_ignore(s)).and_then(|s|builtin_text(s,":",false)).and_then(|s|builtin_ignore(s)).and_then(|s|parse_value(s).and_then(|s| s.tag_node("value"))))
s.sequence(|s| {
Ok(s)
.and_then(|s| parse_object_key(s).and_then(|s| s.tag_node("object_key")))
.and_then(|s| builtin_ignore(s))
.and_then(|s| parse_colon(s).and_then(|s| s.tag_node("colon")))
.and_then(|s| builtin_ignore(s))
.and_then(|s| parse_value(s).and_then(|s| s.tag_node("value")))
})
})
}
#[inline]
fn parse_colon(state: Input) -> Output {
state.rule(Json5Rule::COLON, |s| s.match_string(":", false))
}
#[inline]
fn parse_object_key(state: Input) -> Output {
state.rule(Json5Rule::ObjectKey, |s| {
Err(s).or_else(|s|parse_identifier(s).and_then(|s| s.tag_node("identifier"))).or_else(|s|parse_string(s).and_then(|s| s.tag_node("string")))
Err(s)
.or_else(|s| parse_identifier(s).and_then(|s| s.tag_node("identifier")))
.or_else(|s| parse_string(s).and_then(|s| s.tag_node("string")))
})
}
#[inline]
fn parse_array(state: Input) -> Output {
state.rule(Json5Rule::Array, |s| {
s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,"[",false)).and_then(|s|builtin_ignore(s)).and_then(|s|s.optional(|s|s.sequence(|s|Ok(s).and_then(|s|parse_value(s).and_then(|s| s.tag_node("value"))).and_then(|s|builtin_ignore(s)).and_then(|s|s.repeat(0..4294967295,|s|s.sequence(|s|Ok(s).and_then(|s|builtin_ignore(s)).and_then(|s|s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,",",false)).and_then(|s|builtin_ignore(s)).and_then(|s|parse_value(s).and_then(|s| s.tag_node("value")))))))).and_then(|s|builtin_ignore(s)).and_then(|s|builtin_text(s,",",false))))).and_then(|s|builtin_ignore(s)).and_then(|s|builtin_text(s,"]",false)))
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, "[", false))
.and_then(|s| builtin_ignore(s))
.and_then(|s| {
s.optional(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| parse_value(s).and_then(|s| s.tag_node("value")))
.and_then(|s| builtin_ignore(s))
.and_then(|s| {
s.repeat(0..4294967295, |s| {
s.sequence(|s| {
Ok(s).and_then(|s| builtin_ignore(s)).and_then(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, ",", false))
.and_then(|s| builtin_ignore(s))
.and_then(|s| parse_value(s).and_then(|s| s.tag_node("value")))
})
})
})
})
})
.and_then(|s| builtin_ignore(s))
.and_then(|s| builtin_text(s, ",", false))
})
})
})
.and_then(|s| builtin_ignore(s))
.and_then(|s| builtin_text(s, "]", false))
})
})
}
#[inline]
fn parse_string(state: Input) -> Output {
state.rule(Json5Rule::String, |s| {
Err(s).or_else(|s|s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,"\"",false)).and_then(|s|Err(s).or_else(|s|parse_string_escaped(s).and_then(|s| s.tag_node("string_escaped"))).or_else(|s|builtin_any(s))).and_then(|s|builtin_text(s,"\"",false)))).or_else(|s|s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,"'",false)).and_then(|s|Err(s).or_else(|s|parse_string_escaped(s).and_then(|s| s.tag_node("string_escaped"))).or_else(|s|builtin_any(s))).and_then(|s|builtin_text(s,"'",false))))
Err(s)
.or_else(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, "\"", false))
.and_then(|s| {
Err(s)
.or_else(|s| parse_string_escaped(s).and_then(|s| s.tag_node("string_escaped")))
.or_else(|s| builtin_any(s))
})
.and_then(|s| builtin_text(s, "\"", false))
})
})
.or_else(|s| {
s.sequence(|s| {
Ok(s)
.and_then(|s| builtin_text(s, "'", false))
.and_then(|s| {
Err(s)
.or_else(|s| parse_string_escaped(s).and_then(|s| s.tag_node("string_escaped")))
.or_else(|s| builtin_any(s))
})
.and_then(|s| builtin_text(s, "'", false))
})
})
})
}
#[inline]
fn parse_string_escaped(state: Input) -> Output {
state.rule(Json5Rule::StringEscaped, |s| {
s.sequence(|s|Ok(s).and_then(|s|builtin_text(s,"\\",false)).and_then(|s|builtin_any(s)))
s.sequence(|s| Ok(s).and_then(|s| builtin_text(s, "\\", false)).and_then(|s| builtin_any(s)))
})
}
#[inline]
fn parse_number(state: Input) -> Output {
state.rule(Json5Rule::Number, |s| {
s.match_regex({static REGEX:OnceLock<Regex>=OnceLock::new();REGEX.get_or_init(||Regex::new("^(?x)([+-]?(0|[1-9][0-9]*))").unwrap())})
s.match_regex({
static REGEX: OnceLock<Regex> = OnceLock::new();
REGEX.get_or_init(|| Regex::new("^(?x)([+-]?(0|[1-9][0-9]*))").unwrap())
})
})
}
#[inline]
fn parse_boolean(state: Input) -> Output {
state.rule(Json5Rule::Boolean, |s| {
Err(s).or_else(|s|builtin_text(s,"true",false).and_then(|s| s.tag_node("true"))).or_else(|s|builtin_text(s,"false",false).and_then(|s| s.tag_node("false")))
Err(s)
.or_else(|s| builtin_text(s, "true", false).and_then(|s| s.tag_node("true")))
.or_else(|s| builtin_text(s, "false", false).and_then(|s| s.tag_node("false")))
})
}
#[inline]
fn parse_null(state: Input) -> Output {
state.rule(Json5Rule::Null, |s| {
s.match_string("null", false)
})
state.rule(Json5Rule::Null, |s| s.match_string("null", false))
}
#[inline]
fn parse_identifier(state: Input) -> Output {
state.rule(Json5Rule::Identifier, |s| {
s.match_regex({static REGEX:OnceLock<Regex>=OnceLock::new();REGEX.get_or_init(||Regex::new("^(?x)([_\\p{XID_start}][\\p{XID_continue}]*)").unwrap())})
s.match_regex({
static REGEX: OnceLock<Regex> = OnceLock::new();
REGEX.get_or_init(|| Regex::new("^(?x)([_\\p{XID_start}][\\p{XID_continue}]*)").unwrap())
})
})
}
#[inline]
fn parse_white_space(state: Input) -> Output {
state.rule(Json5Rule::WhiteSpace, |s| {
Err(s).or_else(|s|builtin_text(s," ",false)).or_else(|s|builtin_text(s,"\\n",false)).or_else(|s|builtin_text(s,"\\r",false))
Err(s)
.or_else(|s| builtin_text(s, " ", false))
.or_else(|s| builtin_text(s, "\\n", false))
.or_else(|s| builtin_text(s, "\\r", false))
})
}

/// All rules ignored in ast mode, inline is not recommended
fn builtin_ignore(state: Input) -> Output {
state.repeat(0..u32::MAX, |s| {
parse_white_space(s)
})

state.repeat(0..u32::MAX, |s| parse_white_space(s))
}

fn builtin_any(state: Input) -> Output {
Expand All @@ -108,4 +222,4 @@ fn builtin_text<'i>(state: Input<'i>, text: &'static str, case: bool) -> Output<

fn builtin_regex<'i, 'r>(state: Input<'i>, regex: &'r Regex) -> Output<'i> {
state.rule(Json5Rule::HiddenText, |s| s.match_regex(regex))
}
}
Loading

0 comments on commit ea5dcd6

Please sign in to comment.