Implement nested block comments

glox/internal/lscanner/scanner.go

@@ -67,8 +67,6 @@ func (s *Scanner) scanToken() error {
 		s.addToken(ltoken.Plus, nil)
 	case ';':
 		s.addToken(ltoken.Semicolon, nil)
-	case '*':
-		s.addToken(ltoken.Star, nil)
 
 	// Double-character tokens
 	case '!':
@@ -96,14 +94,25 @@ func (s *Scanner) scanToken() error {
 		}
 		s.addToken(ttype, nil)
 	case '/':
-		if s.match('/') {
-			// Consume comment
+		switch n := s.peek(); n {
+		case '/':
+			s.next() // Consume the '/'
 			for s.peek() != '\n' && !s.done() {
 				s.next()
 			}
-		} else {
+		case '*':
+			s.next() // Consume the '*'
+			if err := s.blockComment(); err != nil {
+				return err
+			}
+		default:
 			s.addToken(ltoken.Slash, nil)
 		}
+	case '*':
+		if s.peek() == '/' {
+			return s.error("found block comment without matching /*")
+		}
+		s.addToken(ltoken.Star, nil)
 
 	// Ignore whitespace
 	case ' ', '\r', '\t':
@@ -122,7 +131,9 @@ func (s *Scanner) scanToken() error {
 		case unicode.IsDigit(r):
 			s.number()
 		case isLetter(r):
-			s.identifier()
+			if err := s.identifier(); err != nil {
+				return err
+			}
 		default:
 			return s.error("unexpected character")
 		}
@@ -141,7 +152,7 @@ func (s *Scanner) string() error {
 	}
 
 	if s.done() {
-		return s.error("unterminated string")
+		return s.error("unterminated string: " + string(s.source[s.start:s.curr]))
 	}
 
 	// The closing "
@@ -176,15 +187,43 @@ func (s *Scanner) number() {
 	s.addToken(ltoken.Number, value)
 }
 
-func (s *Scanner) identifier() {
+func (s *Scanner) identifier() error {
 	// Find end of identifier
 	for r := s.peek(); isLetter(r) || unicode.IsDigit(r); r = s.peek() {
 		s.next()
 	}
 
+	if s.match('"') {
+		return s.error(`string missing opening quote: ` + string(s.source[s.start:s.curr]))
+	}
+
 	ident := string(s.source[s.start:s.curr])
 	ttype := ltoken.LookupKeyword(ident)
 	s.addToken(ttype, ident)
+
+	return nil
+}
+
+func (s *Scanner) blockComment() error {
+	var depth int
+
+	for !s.done() {
+		switch {
+		case s.match('/') && s.match('*'):
+			depth++
+		case s.match('*') && s.match('/'):
+			depth--
+			if depth < 0 {
+				return nil
+			}
+		case s.match('\n'):
+			s.line++
+		default:
+			s.next()
+		}
+	}
+
+	return s.error("unterminated block comment")
 }
 
 func (s *Scanner) done() bool {

glox/internal/lscanner/scanner_test.go

@@ -2,10 +2,12 @@ package lscanner_test
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/hexops/autogold/v2"
 	"github.com/tzcl/lox/glox/internal/lscanner"
+	"github.com/tzcl/lox/glox/internal/ltoken"
 )
 
 func TestScan(t *testing.T) {
@@ -15,23 +17,45 @@ func TestScan(t *testing.T) {
 	}{
 		"RightBrace": {
 			source: "}",
-			expect: autogold.Expect("[RightBrace EOF]"),
+			expect: autogold.Expect("1 RightBrace → 1 EOF"),
 		},
 		"BangEqual": {
 			source: "!=",
-			expect: autogold.Expect("[BangEqual EOF]"),
+			expect: autogold.Expect("1 BangEqual → 1 EOF"),
 		},
 		"String": {
 			source: `"string"`,
-			expect: autogold.Expect(`[String("string") EOF]`),
+			expect: autogold.Expect(`1 String("string") → 1 EOF`),
 		},
 		"Number": {
 			source: "4.123",
-			expect: autogold.Expect("[Number(4.123) EOF]"),
+			expect: autogold.Expect("1 Number(4.123) → 1 EOF"),
 		},
 		"Identifier": {
 			source: "test",
-			expect: autogold.Expect("[Identifier(test) EOF]"),
+			expect: autogold.Expect("1 Identifier(test) → 1 EOF"),
+		},
+		"Comment": {
+			source: `1234
+				// This is a comment
+				test`,
+			expect: autogold.Expect("1 Number(1234) → 3 Identifier(test) → 3 EOF"),
+		},
+		"BlockComment": {
+			source: `1234
+				/* This is a multiline comment
+			       that is very useful */
+				test`,
+			expect: autogold.Expect("1 Number(1234) → 4 Identifier(test) → 4 EOF"),
+		},
+		"NestedBlockComment": {
+			source: `1234
+				/* This is a 
+				   /* nested */ 
+				   block comment 
+				*/
+				test`,
+			expect: autogold.Expect("1 Number(1234) → 6 Identifier(test) → 6 EOF"),
 		},
 	}
 
@@ -42,7 +66,65 @@ func TestScan(t *testing.T) {
 			if err != nil {
 				t.Fatal("failed to scan text")
 			}
-			test.expect.Equal(t, fmt.Sprint(tokens))
+			test.expect.Equal(t, formatTokens(tokens))
 		})
 	}
 }
+
+func TestScanError(t *testing.T) {
+	tests := map[string]struct {
+		source string
+		expect autogold.Value
+	}{
+		"UnterminatedString": {
+			source: `"asdf`,
+			expect: autogold.Expect(`[line 1]: Error: unterminated string: "asdf`),
+		},
+		"LongUnterminatedString": {
+			source: `"asdf
+jkl;
+zxcv
+bnm,`,
+			expect: autogold.Expect(`[line 4]: Error: unterminated string: "asdf
+jkl;
+zxcv
+bnm,`),
+		},
+		"UninitiatedString": {
+			source: `asdf" 1234`,
+			expect: autogold.Expect(`[line 1]: Error: string missing opening quote: asdf"`),
+		},
+		"UnterminatedBlockComment": {
+			source: `/* asdf`,
+			expect: autogold.Expect("[line 1]: Error: unterminated block comment"),
+		},
+		"UninitiatedBlockComment": {
+			source: `asdf */`,
+			expect: autogold.Expect("[line 1]: Error: found block comment without matching /*"),
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			scanner := lscanner.New(test.source)
+			tokens, err := scanner.Scan()
+			t.Log(tokens)
+			if err == nil {
+				t.Fatal("expected error")
+			}
+			test.expect.Equal(t, err.Error())
+		})
+	}
+}
+
+func formatTokens(tokens []ltoken.Token) string {
+	var builder strings.Builder
+	for i, token := range tokens {
+		if i > 0 {
+			builder.WriteString(" → ")
+		}
+		builder.WriteString(fmt.Sprintf("%d %s", token.Line, token.String()))
+	}
+
+	return builder.String()
+}