diff --git a/document/core/text/lexical.rst b/document/core/text/lexical.rst index 1dd34c8635..4584b0c424 100644 --- a/document/core/text/lexical.rst +++ b/document/core/text/lexical.rst @@ -85,7 +85,9 @@ The allowed formatting characters correspond to a subset of the |ASCII|_ *format \production{white space} & \Tspace &::=& (\text{~~} ~|~ \Tformat ~|~ \Tcomment)^\ast \\ \production{format} & \Tformat &::=& - \unicode{09} ~|~ \unicode{0A} ~|~ \unicode{0D} \\ + \Tnewline ~|~ \unicode{09} \\ + \production{newline} & \Tnewline &::=& + \unicode{0A} ~|~ \unicode{0D} ~|~ \unicode{0D}~\unicode{0A} \\ \end{array} The only relevance of white space is to separate :ref:`tokens `. It is otherwise ignored. @@ -107,13 +109,13 @@ Block comments can be nested. \production{comment} & \Tcomment &::=& \Tlinecomment ~|~ \Tblockcomment \\ \production{line comment} & \Tlinecomment &::=& - \Tcommentd~~\Tlinechar^\ast~~(\unicode{0A} ~|~ \T{eof}) \\ + \Tcommentd~~\Tlinechar^\ast~~(\Tnewline ~|~ \T{eof}) \\ \production{line character} & \Tlinechar &::=& - c{:}\Tchar & (\iff c \neq \unicode{0A}) \\ + c{:}\Tchar & (\iff c \neq \unicode{0A} \land c \neq \unicode{0D}) \\ \production{block comment} & \Tblockcomment &::=& \Tcommentl~~\Tblockchar^\ast~~\Tcommentr \\ \production{block character} & \Tblockchar &::=& - c{:}\Tchar & (\iff c \neq \text{;} \wedge c \neq \text{(}) \\ &&|& + c{:}\Tchar & (\iff c \neq \text{;} \land c \neq \text{(}) \\ &&|& \text{;} & (\iff~\mbox{the next character is not}~\text{)}) \\ &&|& \text{(} & (\iff~\mbox{the next character is not}~\text{;}) \\ &&|& \Tblockcomment \\ diff --git a/document/core/util/macros.def b/document/core/util/macros.def index df4af62f83..5e7a4889b2 100644 --- a/document/core/util/macros.def +++ b/document/core/util/macros.def @@ -700,6 +700,7 @@ .. |Tchar| mathdef:: \xref{text/lexical}{text-char}{\T{char}} .. |Tspace| mathdef:: \xref{text/lexical}{text-space}{\T{space}} .. |Tformat| mathdef:: \xref{text/lexical}{text-format}{\T{format}} +.. |Tnewline| mathdef:: \xref{text/lexical}{text-newline}{\T{newline}} .. |Ttoken| mathdef:: \xref{text/lexical}{text-token}{\T{token}} .. |Tkeyword| mathdef:: \xref{text/lexical}{text-keyword}{\T{keyword}} diff --git a/interpreter/Makefile b/interpreter/Makefile index 5939f838ec..dfd9064bbf 100644 --- a/interpreter/Makefile +++ b/interpreter/Makefile @@ -33,7 +33,7 @@ zip: $(ZIP) # Building -.PHONY: $(NAME) $(JSLIB) +.PHONY: $(NAME) $(JSLIB) $(NAME): rm -f $@ diff --git a/interpreter/text/lexer.mll b/interpreter/text/lexer.mll index d9a12b5d21..34099e849f 100644 --- a/interpreter/text/lexer.mll +++ b/interpreter/text/lexer.mll @@ -27,9 +27,9 @@ let string s = while !i < String.length s - 1 do let c = if s.[!i] <> '\\' then s.[!i] else match (incr i; s.[!i]) with - | 'n' -> '\n' - | 'r' -> '\r' - | 't' -> '\t' + | 'n' -> '\x0a' + | 'r' -> '\x0d' + | 't' -> '\x09' | '\\' -> '\\' | '\'' -> '\'' | '\"' -> '\"' @@ -61,10 +61,12 @@ let letter = ['a'-'z''A'-'Z'] let symbol = ['+''-''*''/''\\''^''~''=''<''>''!''?''@''#''$''%''&''|'':''`''.''\''] -let space = [' ''\t''\n''\r'] +let ascii_newline = ['\x0a''\x0d'] +let newline = ascii_newline | "\x0a\x0d" +let space = [' ''\x09''\x0a''\x0d'] let control = ['\x00'-'\x1f'] # space let ascii = ['\x00'-'\x7f'] -let ascii_no_nl = ascii # '\x0a' +let ascii_no_nl = ascii # ascii_newline let utf8cont = ['\x80'-'\xbf'] let utf8enc = ['\xc2'-'\xdf'] utf8cont @@ -127,8 +129,8 @@ rule token = parse | float as s { FLOAT s } | string as s { STRING (string s) } - | '"'character*('\n'|eof) { error lexbuf "unclosed string literal" } - | '"'character*['\x00'-'\x09''\x0b'-'\x1f''\x7f'] + | '"'character*(newline|eof) { error lexbuf "unclosed string literal" } + | '"'character*(control#ascii_newline) { error lexbuf "illegal control character in string literal" } | '"'character*'\\'_ { error_nest (Lexing.lexeme_end_p lexbuf) lexbuf "illegal escape" } @@ -698,11 +700,11 @@ rule token = parse | id as s { VAR s } | ";;"utf8_no_nl*eof { EOF } - | ";;"utf8_no_nl*'\n' { Lexing.new_line lexbuf; token lexbuf } + | ";;"utf8_no_nl*newline { Lexing.new_line lexbuf; token lexbuf } | ";;"utf8_no_nl* { token lexbuf (* causes error on following position *) } | "(;" { comment (Lexing.lexeme_start_p lexbuf) lexbuf; token lexbuf } - | space#'\n' { token lexbuf } - | '\n' { Lexing.new_line lexbuf; token lexbuf } + | space#ascii_newline { token lexbuf } + | newline { Lexing.new_line lexbuf; token lexbuf } | eof { EOF } | reserved { unknown lexbuf } @@ -713,7 +715,7 @@ rule token = parse and comment start = parse | ";)" { () } | "(;" { comment (Lexing.lexeme_start_p lexbuf) lexbuf; comment start lexbuf } - | '\n' { Lexing.new_line lexbuf; comment start lexbuf } + | newline { Lexing.new_line lexbuf; comment start lexbuf } | utf8_no_nl { comment start lexbuf } | eof { error_nest start lexbuf "unclosed comment" } | _ { error lexbuf "malformed UTF-8 encoding" } diff --git a/test/core/comments.wast b/test/core/comments.wast index c291370fa7..5ef64a6c1e 100644 Binary files a/test/core/comments.wast and b/test/core/comments.wast differ