Custom Error Default trait with extra information from the Lexer as parameters

src/internal.rs

@@ -1,5 +1,5 @@
 use crate::source::Chunk;
-use crate::{Filter, FilterResult, Lexer, Logos, Skip};
+use crate::{utils, Filter, FilterResult, Lexer, Logos, Skip};
 
 /// Trait used by the functions contained in the `Lexicon`.
 ///
@@ -72,7 +72,7 @@ impl<'s, T: Logos<'s>> CallbackResult<'s, (), T> for bool {
     {
         match self {
             true => lex.set(Ok(c(()))),
-            false => lex.set(Err(T::Error::default())),
+            false => lex.set(Err(utils::error_from_lexer(lex))),
         }
     }
 }
@@ -85,7 +85,7 @@ impl<'s, P, T: Logos<'s>> CallbackResult<'s, P, T> for Option<P> {
     {
         match self {
             Some(product) => lex.set(Ok(c(product))),
-            None => lex.set(Err(T::Error::default())),
+            None => lex.set(Err(utils::error_from_lexer(lex))),
         }
     }
 }

src/lexer.rs

@@ -1,6 +1,7 @@
 use super::internal::LexerInternal;
 use super::Logos;
 use crate::source::{self, Source};
+use crate::utils;
 
 use core::fmt::{self, Debug};
 use core::ops::{Deref, DerefMut};
@@ -378,11 +379,11 @@ where
         self.token_end = self.source.find_boundary(self.token_end);
         #[cfg(not(feature = "forbid_unsafe"))]
         {
-            self.token = core::mem::ManuallyDrop::new(Some(Err(Token::Error::default())));
+            self.token = core::mem::ManuallyDrop::new(Some(Err(utils::error_from_lexer(self))));
         }
         #[cfg(feature = "forbid_unsafe")]
         {
-            self.token = Some(Err(Token::Error::default()));
+            self.token = Some(Err(utils::error_from_lexer(self)));
         }
     }
 

src/lib.rs

@@ -35,6 +35,7 @@ pub mod source;
 
 #[doc(hidden)]
 pub mod internal;
+pub(crate) mod utils;
 
 pub use crate::lexer::{Lexer, Span, SpannedIter};
 pub use crate::source::Source;
@@ -53,7 +54,7 @@ pub trait Logos<'source>: Sized {
 
     /// Error type returned by the lexer. This can be set using
     /// `#[logos(error = MyError)]`. Defaults to `()` if not set.
-    type Error: Default + Clone + PartialEq + Debug + 'source;
+    type Error: DefaultLexerError<'source, Self::Source, Self::Extras>;
 
     /// The heart of Logos. Called by the `Lexer`. The implementation for this function
     /// is generated by the `logos-derive` crate.
@@ -78,6 +79,19 @@ pub trait Logos<'source>: Sized {
     }
 }
 
+/// Trait used to construct the default error type for a `Logos` implementation.
+/// More information is communicated than simply using [`Default`].
+pub trait DefaultLexerError<'source, Source: ?Sized, Extras = ()>:
+    Clone + PartialEq + Debug
+{
+    /// Constructs a default error, given contextual information from the Lexer.
+    fn from_lexer<'e>(source: &'source Source, span: Span, extras: &'e Extras) -> Self;
+}
+
+impl<'source, Source: ?Sized, Extras> DefaultLexerError<'source, Source, Extras> for () {
+    fn from_lexer<'e>(_: &'source Source, _: Span, _: &'e Extras) -> Self {}
+}
+
 /// Type that can be returned from a callback, informing the `Lexer`, to skip
 /// current token match. See also [`logos::skip`](./fn.skip.html).
 ///

src/utils.rs

@@ -0,0 +1,13 @@
+pub(crate) fn error_from_lexer<
+    'source,
+    Token: super::Logos<'source>,
+    Error: super::DefaultLexerError<'source, Token::Source, Token::Extras>,
+>(
+    lex: &crate::Lexer<'source, Token>,
+) -> Error {
+    let source = lex.source();
+    let span = lex.span();
+    let extras = &lex.extras;
+
+    Error::from_lexer(source, span, extras)
+}

tests/Cargo.toml

@@ -5,14 +5,11 @@ publish = false
 version = "0.1.0"
 
 [dependencies]
-logos-derive = {path = "../logos-derive"}
-logos = {path = "../", default-features = false, features = ["std"]}
+logos-derive = { path = "../logos-derive" }
+logos = { path = "../", default-features = false, features = ["std"] }
 
 [features]
-forbid_unsafe = [
-    "logos-derive/forbid_unsafe",
-    "logos/forbid_unsafe"
-]
+forbid_unsafe = ["logos-derive/forbid_unsafe", "logos/forbid_unsafe"]
 
 [dev-dependencies]
 criterion = "0.4"

tests/tests/callbacks.rs

@@ -1,14 +1,19 @@
-use logos::{Lexer, Logos as _, Skip};
+use logos::{DefaultLexerError, Lexer, Logos as _, Skip};
 use logos_derive::Logos;
 use tests::assert_lex;
 
-#[derive(Default, Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
 enum LexingError {
     ParseNumberError,
-    #[default]
     Other,
 }
 
+impl<'source> DefaultLexerError<'source, str, ()> for LexingError {
+    fn from_lexer<'e>(_: &'source str, _: logos::Span, _: &'e ()) -> Self {
+        LexingError::Other
+    }
+}
+
 impl From<std::num::ParseIntError> for LexingError {
     fn from(_: std::num::ParseIntError) -> Self {
         LexingError::ParseNumberError
@@ -176,11 +181,19 @@ mod any_token_callback {
 mod return_result_skip {
     use super::*;
 
-    #[derive(Debug, Default, PartialEq, Clone)]
+    #[derive(Debug, PartialEq, Clone)]
     enum LexerError {
         UnterminatedComment,
-        #[default]
-        Other,
+        Other { source: String, span: logos::Span },
+    }
+
+    impl<'source, Extras> DefaultLexerError<'source, str, Extras> for LexerError {
+        fn from_lexer<'e>(source: &'source str, span: logos::Span, _: &'e Extras) -> Self {
+            LexerError::Other {
+                source: source.to_owned(),
+                span,
+            }
+        }
     }
 
     #[derive(Logos, Debug, PartialEq)]

tests/tests/custom_error.rs

@@ -1,35 +1,45 @@
+use logos::DefaultLexerError;
 use logos_derive::Logos;
 use std::num::{IntErrorKind, ParseIntError};
 use tests::assert_lex;
 
-#[derive(Debug, Clone, PartialEq, Default)]
-enum LexingError {
+#[derive(Debug, Clone, PartialEq)]
+enum CustomError {
     NumberTooLong,
     NumberNotEven(u32),
-    #[default]
-    Other,
+    Unknown,
+    Generic { source: String, span: logos::Span },
 }
 
-impl From<ParseIntError> for LexingError {
+impl<'source, Extras> DefaultLexerError<'source, str, Extras> for CustomError {
+    fn from_lexer<'e>(source: &'source str, span: logos::Span, _: &'e Extras) -> Self {
+        CustomError::Generic {
+            source: source.to_owned(),
+            span,
+        }
+    }
+}
+
+impl From<ParseIntError> for CustomError {
     fn from(value: ParseIntError) -> Self {
         match value.kind() {
-            IntErrorKind::PosOverflow => LexingError::NumberTooLong,
-            _ => LexingError::Other,
+            IntErrorKind::PosOverflow => CustomError::NumberTooLong,
+            _ => CustomError::Unknown,
         }
     }
 }
 
-fn parse_number(input: &str) -> Result<u32, LexingError> {
+fn parse_number(input: &str) -> Result<u32, CustomError> {
     let num = input.parse::<u32>()?;
-    if num % 2 == 0 {
+    if input.parse::<u32>()? % 2 == 0 {
         Ok(num)
     } else {
-        Err(LexingError::NumberNotEven(num))
+        Err(CustomError::NumberNotEven(num))
     }
 }
 
 #[derive(Logos, Debug, Clone, Copy, PartialEq)]
-#[logos(error = LexingError)]
+#[logos(error = CustomError)]
 enum Token<'a> {
     #[regex(r"[0-9]+", |lex| parse_number(lex.slice()))]
     Number(u32),
@@ -39,19 +49,27 @@ enum Token<'a> {
 
 #[test]
 fn test() {
+    let source = "123abc1234xyz1111111111111111111111111111111111111111111111111111111,";
     assert_lex(
-        "123abc1234xyz1111111111111111111111111111111111111111111111111111111,",
+        source,
         &[
-            (Err(LexingError::NumberNotEven(123)), "123", 0..3),
+            (Err(CustomError::NumberNotEven(123)), "123", 0..3),
             (Ok(Token::Identifier("abc")), "abc", 3..6),
             (Ok(Token::Number(1234)), "1234", 6..10),
             (Ok(Token::Identifier("xyz")), "xyz", 10..13),
             (
-                Err(LexingError::NumberTooLong),
+                Err(CustomError::NumberTooLong),
                 "1111111111111111111111111111111111111111111111111111111",
                 13..68,
             ),
-            (Err(LexingError::Other), ",", 68..69),
+            (
+                Err(CustomError::Generic {
+                    source: source.into(),
+                    span: 68..69,
+                }),
+                ",",
+                68..69,
+            ),
         ],
     );
 }

tests/tests/edgecase.rs

@@ -324,12 +324,24 @@ mod type_params {
     use super::*;
     use std::num::ParseIntError;
 
-    #[derive(Debug, Clone, PartialEq, Default)]
-    struct LexingError;
+    #[derive(Debug, Clone, PartialEq)]
+    enum LexingError {
+        ParseIntError(ParseIntError),
+        Other { source: String, span: logos::Span },
+    }
+
+    impl<'source, Extras> logos::DefaultLexerError<'source, str, Extras> for LexingError {
+        fn from_lexer<'e>(source: &'source str, span: logos::Span, _: &'e Extras) -> Self {
+            LexingError::Other {
+                source: source.to_owned(),
+                span,
+            }
+        }
+    }
 
     impl From<ParseIntError> for LexingError {
-        fn from(_: ParseIntError) -> Self {
-            LexingError
+        fn from(e: ParseIntError) -> Self {
+            LexingError::ParseIntError(e)
         }
     }