Make parser properly handle multiline inputs

This commits makes the parser properly handle multiline inputs as well as empty lines. Though, the parser still assumes that instructions are not multiline — which seems to be reasonable. This commit is prerequisite to parse stream of tokens.
rhartert · May 27, 2024 · 2c4ea3e · 2c4ea3e
1 parent bfe0288
commit 2c4ea3e
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 53 deletions.
diff --git a/fzn/fzn.go b/fzn/fzn.go
@@ -59,10 +59,6 @@ func Parse(reader io.Reader, handler Handler) error {
 		i++
 
 		line := scanner.Text()
-		if line == "" { // TODO: this should ideally be done in the parser itself
-			continue
-		}
-
 		tokens, err := tokenizer.Tokenize(line)
 		if err != nil {
 			return fmt.Errorf("tokenizer error at line %d: %w", i, err)

diff --git a/fzn/fzn_model_test.go b/fzn/fzn_model_test.go
@@ -9,19 +9,11 @@ import (
 	"github.com/rhartert/ptr"
 )
 
-var testCakesFZN = `
-array [1..2] of int: X_INTRODUCED_2_ = [250,200];
-array [1..2] of int: X_INTRODUCED_6_ = [75,150];
-array [1..2] of int: X_INTRODUCED_8_ = [100,150];
-var 0..3: b:: output_var;
-var 0..6: c:: output_var;
-var 0..85000: X_INTRODUCED_0_:: is_defined_var;
-constraint int_lin_le(X_INTRODUCED_2_,[b,c],4000);
-constraint int_lin_le(X_INTRODUCED_6_,[b,c],2000);
-constraint int_lin_le(X_INTRODUCED_8_,[b,c],500);
-constraint int_lin_eq([400,450,-1],[b,c,X_INTRODUCED_0_],0):: ctx_pos:: defines_var(X_INTRODUCED_0_);
-solve  maximize X_INTRODUCED_0_;
-`
+//go:embed testdata/cakes.fzn
+var testCakesFZN string
+
+//go:embed testdata/cakes_inline.fzn
+var testCakesFZNInline string
 
 var testCakesModel = Model{
 	ParamDeclarations: []ParamDeclaration{
@@ -145,6 +137,11 @@ func TestParseModel(t *testing.T) {
 			input: testCakesFZN,
 			want:  &testCakesModel,
 		},
+		{
+			desc:  "cakes_inline.fzn",
+			input: testCakesFZNInline,
+			want:  &testCakesModel,
+		},
 	}
 
 	for _, tc := range testCases {

diff --git a/fzn/parser.go b/fzn/parser.go
@@ -46,7 +46,8 @@ func (p *parser) nextIf(tt tok.Type) bool {
 }
 
 // lookAhead returns the token at n positions from the current position without
-// impacting the result of p.next.
+// impacting the result of p.next. In particular, lookAhead(0) peeks at the
+// next token without consuming it.
 func (p *parser) lookAhead(n int) tok.Token {
 	if i := p.pos + n; i < len(p.tokens) {
 		return p.tokens[i]
@@ -58,41 +59,57 @@ func (p *parser) lookAhead(n int) tok.Token {
 // appropriate Handler method. It returns an error if parsing fails or if the
 // Handler reports an error.
 func (p *parser) parse() error {
-	switch {
-	case isComment(p):
-		_, err := parseComment(p) // drop comments
-		return err
-	case isPredicate(p):
-		pred, err := parsePredicate(p)
-		if err != nil {
-			return err
+	for p.lookAhead(0).Type != tok.EOF {
+		switch {
+		case isComment(p):
+			_, err := parseComment(p) // drop comments
+			if err != nil {
+				return err
+			}
+		case isPredicate(p):
+			pred, err := parsePredicate(p)
+			if err != nil {
+				return err
+			}
+			if err := p.handler.AddPredicate(pred); err != nil {
+				return err
+			}
+		case isParamDeclaration(p):
+			param, err := parseParamDeclaration(p)
+			if err != nil {
+				return err
+			}
+			if err := p.handler.AddParamDeclaration(param); err != nil {
+				return err
+			}
+		case isVarDeclaration(p):
+			v, err := parseVarDeclaration(p)
+			if err != nil {
+				return err
+			}
+			if err := p.handler.AddVarDeclaration(v); err != nil {
+				return err
+			}
+		case isConstraint(p):
+			c, err := parseConstraint(p)
+			if err != nil {
+				return err
+			}
+			if err := p.handler.AddConstraint(c); err != nil {
+				return err
+			}
+		case isSolveGoal(p):
+			s, err := parseSolveGoal(p)
+			if err != nil {
+				return err
+			}
+			if err := p.handler.AddSolveGoal(s); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unrecognized instruction")
 		}
-		return p.handler.AddPredicate(pred)
-	case isParamDeclaration(p):
-		param, err := parseParamDeclaration(p)
-		if err != nil {
-			return err
-		}
-		return p.handler.AddParamDeclaration(param)
-	case isVarDeclaration(p):
-		v, err := parseVarDeclaration(p)
-		if err != nil {
-			return err
-		}
-		return p.handler.AddVarDeclaration(v)
-	case isConstraint(p):
-		c, err := parseConstraint(p)
-		if err != nil {
-			return err
-		}
-		return p.handler.AddConstraint(c)
-	case isSolveGoal(p):
-		s, err := parseSolveGoal(p)
-		if err != nil {
-			return err
-		}
-		return p.handler.AddSolveGoal(s)
-	default:
-		return fmt.Errorf("unrecognized instruction")
 	}
+
+	return nil
 }
diff --git a/fzn/testdata/cakes.fzn b/fzn/testdata/cakes.fzn
@@ -0,0 +1,11 @@
+array [1..2] of int: X_INTRODUCED_2_ = [250,200];
+array [1..2] of int: X_INTRODUCED_6_ = [75,150];
+array [1..2] of int: X_INTRODUCED_8_ = [100,150];
+var 0..3: b:: output_var;
+var 0..6: c:: output_var;
+var 0..85000: X_INTRODUCED_0_:: is_defined_var;
+constraint int_lin_le(X_INTRODUCED_2_,[b,c],4000);
+constraint int_lin_le(X_INTRODUCED_6_,[b,c],2000);
+constraint int_lin_le(X_INTRODUCED_8_,[b,c],500);
+constraint int_lin_eq([400,450,-1],[b,c,X_INTRODUCED_0_],0):: ctx_pos:: defines_var(X_INTRODUCED_0_);
+solve  maximize X_INTRODUCED_0_;
diff --git a/fzn/testdata/cakes_inline.fzn b/fzn/testdata/cakes_inline.fzn
@@ -0,0 +1 @@
+array [1..2] of int: X_INTRODUCED_2_ = [250,200];array [1..2] of int: X_INTRODUCED_6_ = [75,150];array [1..2] of int: X_INTRODUCED_8_ = [100,150];var 0..3: b:: output_var;var 0..6: c:: output_var;var 0..85000: X_INTRODUCED_0_:: is_defined_var;constraint int_lin_le(X_INTRODUCED_2_,[b,c],4000);constraint int_lin_le(X_INTRODUCED_6_,[b,c],2000);constraint int_lin_le(X_INTRODUCED_8_,[b,c],500);constraint int_lin_eq([400,450,-1],[b,c,X_INTRODUCED_0_],0):: ctx_pos:: defines_var(X_INTRODUCED_0_);solve  maximize X_INTRODUCED_0_;