From f1bd815173114b28175a39617f419d99e8de0a88 Mon Sep 17 00:00:00 2001 From: Vladimir Motylenko Date: Thu, 16 Nov 2023 11:46:13 +0200 Subject: [PATCH] Add unquoted text custom node integration --- build.rs | 17 ++++ examples/html-to-string-macro/src/lib.rs | 2 +- examples/html-to-string-macro/tests/tests.rs | 2 + src/node/mod.rs | 2 +- src/node/raw_text.rs | 40 ++++++-- src/parser/recoverable.rs | 6 +- tests/test.rs | 99 +++++++++++++++++--- 7 files changed, 140 insertions(+), 28 deletions(-) create mode 100644 build.rs diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..386474d --- /dev/null +++ b/build.rs @@ -0,0 +1,17 @@ +use std::{env, process::Command, str::from_utf8}; + +fn main() { + if is_rustc_nightly() { + println!("cargo:rustc-cfg=rstml_signal_nightly"); + } +} + +fn is_rustc_nightly() -> bool { + || -> Option { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = from_utf8(&output.stdout).ok()?; + Some(version.contains("nightly")) + }() + .unwrap_or_default() +} diff --git a/examples/html-to-string-macro/src/lib.rs b/examples/html-to-string-macro/src/lib.rs index c86d96b..4a6d97c 100644 --- a/examples/html-to-string-macro/src/lib.rs +++ b/examples/html-to-string-macro/src/lib.rs @@ -8,7 +8,7 @@ use rstml::{ Parser, ParserConfig, }; use syn::spanned::Spanned; -mod escape; +// mod escape; #[derive(Default)] struct WalkNodesOutput<'a> { static_format: String, diff --git a/examples/html-to-string-macro/tests/tests.rs b/examples/html-to-string-macro/tests/tests.rs index 0ecd413..afc2e6a 100644 --- a/examples/html-to-string-macro/tests/tests.rs +++ b/examples/html-to-string-macro/tests/tests.rs @@ -1,3 +1,4 @@ +use rstml::node::RawText; use rstml_to_string_macro::html_ide; // Using this parser, one can write docs and link html tags to them. @@ -11,6 +12,7 @@ pub mod docs { fn test() { let nightly_unqoted = " Hello world with spaces "; let stable_unqoted = "Hello world with spaces"; + assert_eq!(cfg!(rstml_signal_nightly), RawText::is_source_text_available()); let unquoted_text = if cfg!(rstml_signal_nightly) { nightly_unqoted } else { diff --git a/src/node/mod.rs b/src/node/mod.rs index ef57214..a80316a 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -64,7 +64,7 @@ pub enum Node { Element(NodeElement), Block(NodeBlock), Text(NodeText), - RawText(RawText), + RawText(RawText), Custom(C), } diff --git a/src/node/raw_text.rs b/src/node/raw_text.rs index 88e4cb8..94c8d30 100644 --- a/src/node/raw_text.rs +++ b/src/node/raw_text.rs @@ -1,3 +1,5 @@ +use std::{convert::Infallible, marker::PhantomData}; + use proc_macro2::{Span, TokenStream, TokenTree}; use quote::ToTokens; use syn::{ @@ -25,13 +27,25 @@ use super::{CustomNode, Node}; /// source_text method is not available in `quote!` context, or in context where /// input is generated by another macro. In still can return default formatting /// for TokenStream. -#[derive(Clone, Debug, Default)] -pub struct RawText { +#[derive(Clone, Debug)] +pub struct RawText { token_stream: TokenStream, // Span that started before previous token, and after next. context_span: Option<(Span, Span)>, + _c: PhantomData } -impl RawText { + +impl Default for RawText { + fn default() -> Self { + Self { + token_stream: Default::default(), + context_span: Default::default(), + _c: PhantomData + } + } +} + +impl RawText { pub(crate) fn set_tag_spans(&mut self, before: impl Spanned, after: impl Spanned) { // todo: use span.after/before when it will be available in proc_macro2 // for now just join full span an remove tokens from it. @@ -84,7 +98,7 @@ impl RawText { self.token_stream.is_empty() } - pub(crate) fn vec_set_context( + pub(crate) fn vec_set_context( open_tag_end: Span, close_tag_start: Option, mut children: Vec>, @@ -114,11 +128,19 @@ impl RawText { } } -impl Parse for RawText { +impl RawText { + /// Returns true if we on nightly rust and join_spans available + pub fn is_source_text_available() -> bool { + // TODO: Add feature join_spans check. + cfg!(rstml_signal_nightly) + } +} + +impl Parse for RawText { fn parse(input: ParseStream) -> syn::Result { let mut token_stream = TokenStream::new(); let any_node = - |input: ParseStream| input.peek(Token![<]) || input.peek(Brace) || input.peek(LitStr); + |input: ParseStream| input.peek(Token![<]) || input.peek(Brace) || input.peek(LitStr) || C::peek_element(&input.fork()); // Parse any input until catching any node. // Fail only on eof. while !any_node(input) && !input.is_empty() { @@ -127,21 +149,23 @@ impl Parse for RawText { Ok(Self { token_stream, context_span: None, + _c: PhantomData, }) } } -impl ToTokens for RawText { +impl ToTokens for RawText { fn to_tokens(&self, tokens: &mut TokenStream) { self.token_stream.to_tokens(tokens) } } -impl From for RawText { +impl From for RawText { fn from(token_stream: TokenStream) -> Self { Self { token_stream, context_span: None, + _c: PhantomData, } } } diff --git a/src/parser/recoverable.rs b/src/parser/recoverable.rs index df1118c..f04aeba 100644 --- a/src/parser/recoverable.rs +++ b/src/parser/recoverable.rs @@ -38,7 +38,7 @@ //! [`Parser::parse_recoverable`]: struct.Parser.html#method.parse_recoverable //! [`Node`]: struct.Node.html -use std::{backtrace, collections::HashSet, fmt::Debug, rc::Rc}; +use std::{collections::HashSet, fmt::Debug, rc::Rc}; use proc_macro2_diagnostics::{Diagnostic, Level}; use syn::parse::{Parse, ParseStream}; @@ -155,9 +155,7 @@ impl RecoverableContext { /// [`proc_macro2_diagnostics::Diagnostic`] pub fn push_diagnostic(&mut self, diagnostic: impl Into) { let diag = diagnostic.into(); - - println!("{}", std::backtrace::Backtrace::capture().to_string()); - self.diagnostics.push(dbg!(diag)); + self.diagnostics.push(diag); } } diff --git a/tests/test.rs b/tests/test.rs index ebf4aeb..1914dc9 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -4,10 +4,10 @@ use eyre::Result; use proc_macro2::TokenStream; use quote::{quote, ToTokens}; use rstml::{ - node::{KeyedAttribute, KeyedAttributeValue, Node, NodeAttribute, NodeElement, NodeType}, - parse2, Parser, ParserConfig, + node::{KeyedAttribute, KeyedAttributeValue, Node, NodeAttribute, NodeElement, NodeType, CustomNode}, + parse2, Parser, ParserConfig, recoverable::RecoverableContext, }; -use syn::{parse_quote, token::Colon, Block, LifetimeParam, Pat, PatType, Token, TypeParam}; +use syn::{parse_quote, token::{Colon, Bracket}, Block, LifetimeParam, Pat, PatType, Token, TypeParam, bracketed, parse::ParseStream}; #[test] fn test_single_empty_element() -> Result<()> { @@ -71,6 +71,51 @@ fn test_single_element_with_unquoted_text_simple() -> Result<()> { Ok(()) } + + +#[test] +fn test_css_selector_unquoted_text() -> Result<()> { + let tokens = quote! { + // Note two spaces between bar and baz + --css-selector & with @strange + .puncts + }; + + let nodes = parse2(tokens)?; + let Node::RawText(child) = &nodes[0] else { + panic!("expected child") + }; + + // We can't use source text if token stream was created with quote!. + assert_eq!(child.to_token_stream_string(), "- - css - selector & with @ strange + . puncts"); + assert_eq!(child.to_token_stream_string(), child.to_string_best()); + Ok(()) +} + +#[test] +fn test_css_selector_unquoted_text_string() -> Result<()> { + let tokens = TokenStream::from_str( + r#" + + "#, + ) + .unwrap(); + + let nodes = parse2(tokens)?; + let Node::RawText(child) = get_element_child(&nodes, 0, 0) else { + panic!("expected child") + }; + + // source text should be available + assert_eq!(child.to_source_text(true).unwrap(), " --css-selector & with @strange + .puncts "); + assert_eq!(child.to_source_text(false).unwrap(), "--css-selector & with @strange + .puncts"); + + // without source text - it will return invalid css + assert_eq!(child.to_token_stream_string(), "-- css - selector & with @ strange + . puncts"); + // When source is available, best should + assert_eq!(child.to_string_best(), child.to_source_text(true).unwrap()); + Ok(()) +} + #[test] fn test_single_element_with_unquoted_text_advance() -> Result<()> { let tokens = TokenStream::from_str( @@ -94,6 +139,31 @@ fn test_single_element_with_unquoted_text_advance() -> Result<()> { Ok(()) } +#[derive(Clone, Debug)] +struct TestCustomNode { + bracket: Bracket, + data: TokenStream +} + +impl CustomNode for TestCustomNode { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.bracket.surround(tokens, |c|self.data.to_tokens(c)) + } + + fn peek_element(input: ParseStream) -> bool { + input.peek(Bracket) + } + + fn parse_element(parser: &mut RecoverableContext, input: ParseStream) -> Option { + let inner_parser = |_parser: &mut RecoverableContext, input: ParseStream| { + let content; + let bracket = bracketed!(content in input); + Ok(Some(TestCustomNode { bracket, data: content.parse()? })) + }; + parser.parse_mixed_fn(input, inner_parser)? + } +} + macro_rules! test_unquoted { ($($name:ident => $constructor: expr => Node::$getter:ident($bind:ident) => $check:expr ;)* ) => { $( @@ -104,14 +174,13 @@ macro_rules! test_unquoted { let tokens = TokenStream::from_str( concat!(" bar bar ", $constructor, " baz baz ") ).unwrap(); + let nodes = Parser::new(ParserConfig::default().custom_node::()).parse_simple(tokens)?; + let Node::RawText(child1) = get_element_child(&*nodes, 0, 0) else { panic!("expected unquoted child") }; - let nodes = parse2(tokens)?; - let Node::RawText(child1) = get_element_child(&nodes, 0, 0) else { panic!("expected unquoted child") }; - - let Node::$getter($bind) = get_element_child(&nodes, 0, 1) else { panic!("expected matcher child") }; - - let Node::RawText(child3) = get_element_child(&nodes, 0, 2) else { panic!("expected unquoted child") }; + let Node::$getter($bind) = get_element_child(&*nodes, 0, 1) else { panic!("expected matcher child") }; + let Node::RawText(child3) = get_element_child(&*nodes, 0, 2) else { panic!("expected unquoted child") }; + // source text should be available assert_eq!(child1.to_source_text(true).unwrap(), " bar bar "); assert_eq!(child1.to_source_text(false).unwrap(), "bar bar"); @@ -164,7 +233,9 @@ test_unquoted!( assert!(child.close_tag.is_none()); }; - + custom_node => "[bracketed text]" => Node::Custom(v) => { + assert_eq!(v.data.to_string(), "bracketed text"); + }; ); #[test] @@ -885,15 +956,15 @@ fn test_empty_input() -> Result<()> { Ok(()) } -fn get_element(nodes: &[Node], element_index: usize) -> &NodeElement { +fn get_element(nodes: &[Node], element_index: usize) -> &NodeElement { let Some(Node::Element(element)) = nodes.get(element_index) else { panic!("expected element") }; element } -fn get_element_attribute( - nodes: &[Node], +fn get_element_attribute( + nodes: &[Node], element_index: usize, attribute_index: usize, ) -> &KeyedAttribute { @@ -908,7 +979,7 @@ fn get_element_attribute( attribute } -fn get_element_child(nodes: &[Node], element_index: usize, child_index: usize) -> &Node { +fn get_element_child(nodes: &[Node], element_index: usize, child_index: usize) -> &Node { let Some(Node::Element(element)) = nodes.get(element_index) else { panic!("expected element") };