Skip to content

Commit

Permalink
Restrict aliases to letters & numbers
Browse files Browse the repository at this point in the history 
This gives a huge usability improvement, because now you can write e.g.

 "@A!", @C shouted as she ran towards her friend.

and it would get parsed properly:

 "@A|!", @C| shouted as she ran towards her friend.

Before this change, you would have had to add the bar terminator to @A
to get this to work, otherwise it would have been parsed as:

 "@A!",| @C| shouted as she ran towards her friend.

Note that that would have changed slightly as soon as #22 is done, to
something like this:

 "@A!|", @C| shouted as she ran towards her friend.

But that also still needs a terminator to be correct. With how often
punctuation occurs after an alias (e.g. a character's name), that isn't
great. This way we only lose the ability to have aliases with
punctuation in them, which doesn't strike me as a problem. I did keep
numbers in because I can see those being useful (@A1, etc.).

Closes #39
  • Loading branch information
@Infernio
Infernio committed Feb 7, 2023
1 parent 1f0ce93 commit 94df930
Show file tree
Hide file tree
Showing 10 changed files with 545 additions and 308 deletions.
25 changes: 14 additions & 11 deletions grammar/zoia.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,19 @@ textFragment: Word | Spaces;
// Version of textFragment that can only resolve to a word.
textFragmentWord: Word;

// An at sign followed by a word. May optionally be ended by a
// vertical bar (this is necessary when an alias is followed by
// something other than whitespace, for example a comma).
alias: At Word Bar?;
// An alias is an at sign followed by (potentially Unicode) letters
// and/or numbers (but no punctuation). May optionally be ended by a
// vertical bar (this is necessary when an alias is directly followed
// by letters, e.g. @A|s).
alias: Alias Bar?;

// A backslash and a word or backslash, followed optionally by an argument
// list. May optionally be ended by a vertical bar (this is necessary when
// using an argumentless command that you don't want a space after.
// For example, '\atmyHandle' would be parsed as a single command
// named 'atMyHandle', which doesn't exist. '\at|myHandle' would be
// parsed as a command named 'at' and a text fragment 'myHandle'.
// A backslash and a word or backslash, followed optionally by an
// argument list. May optionally be ended by a vertical bar (this is
// necessary when using an argumentless command that you don't want a
// space after. For example, '\atmyHandle' would be parsed as a
// single command named 'atMyHandle', which doesn't exist.
// '\at|myHandle' would be parsed as a command named 'at' and a text
// fragment 'myHandle'.
command: Backslash (Word | Backslash) arguments? Bar?;

// One or more arguments. Multiple arguments must be separated by
Expand All @@ -84,7 +86,6 @@ COMMENT: '#' ~[\r\n]* -> skip;

// Regular tokens
Asterisk: '*';
At: '@';
Backslash: '\\';
Bar: '|';
BracketsClose: ']';
Expand All @@ -96,5 +97,7 @@ Semicolon: ';';

// See https://unicode.org/reports/tr44/#General_Category_Values
Spaces: [\p{Z}]+;
// Put aliases before words, so that ANTLR prefers a full alias
Alias: '@' [\p{L}\p{N}]+;
// Everything that isn't one of the tokens above.
Word: ~[\r\n@\\|[\];=*#\p{Z}]+;
182 changes: 150 additions & 32 deletions src/grammar/cpp/zoiaLexer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,8 @@ void zoialexerLexerInitialize() {
assert(zoialexerLexerStaticData == nullptr);
auto staticData = std::make_unique<ZoiaLexerStaticData>(
std::vector<std::string>{
"COMMENT", "Asterisk", "At", "Backslash", "Bar", "BracketsClose",
"BracketsOpen", "Equals", "Header", "Newline", "Semicolon", "Spaces",
"Word"
"COMMENT", "Asterisk", "Backslash", "Bar", "BracketsClose", "BracketsOpen",
"Equals", "Header", "Newline", "Semicolon", "Spaces", "Alias", "Word"
},
std::vector<std::string>{
"DEFAULT_TOKEN_CHANNEL", "HIDDEN"
Expand All @@ -59,42 +58,161 @@ void zoialexerLexerInitialize() {
"DEFAULT_MODE"
},
std::vector<std::string>{
"", "", "'*'", "'@'", "'\\'", "'|'", "']'", "'['", "'='", "'\\header'",
"", "';'"
"", "", "'*'", "'\\'", "'|'", "']'", "'['", "'='", "'\\header'", "",
"';'"
},
std::vector<std::string>{
"", "COMMENT", "Asterisk", "At", "Backslash", "Bar", "BracketsClose",
"BracketsOpen", "Equals", "Header", "Newline", "Semicolon", "Spaces",
"Word"
"", "COMMENT", "Asterisk", "Backslash", "Bar", "BracketsClose", "BracketsOpen",
"Equals", "Header", "Newline", "Semicolon", "Spaces", "Alias", "Word"
}
);
static const int32_t serializedATNSegment[] = {
4,0,13,75,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,
4,0,13,79,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,
6,2,7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,2,11,7,11,2,12,7,12,1,0,1,0,5,0,30,
8,0,10,0,12,0,33,9,0,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,4,1,4,1,5,1,5,
1,6,1,6,1,7,1,7,1,8,1,8,1,8,1,8,1,8,1,8,1,8,1,8,1,9,1,9,1,9,3,9,62,8,
9,1,10,1,10,1,11,4,11,67,8,11,11,11,12,11,68,1,12,4,12,72,8,12,11,12,
12,12,73,0,0,13,1,1,3,2,5,3,7,4,9,5,11,6,13,7,15,8,17,9,19,10,21,11,23,
12,25,13,1,0,3,2,0,10,10,13,13,8,0,32,32,160,160,5760,5760,8192,8202,
8232,8233,8239,8239,8287,8287,12288,12288,17,0,10,10,13,13,32,32,35,35,
42,42,59,59,61,61,64,64,91,93,124,124,160,160,5760,5760,8192,8202,8232,
8233,8239,8239,8287,8287,12288,12288,78,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,
0,0,0,0,7,1,0,0,0,0,9,1,0,0,0,0,11,1,0,0,0,0,13,1,0,0,0,0,15,1,0,0,0,
0,17,1,0,0,0,0,19,1,0,0,0,0,21,1,0,0,0,0,23,1,0,0,0,0,25,1,0,0,0,1,27,
1,0,0,0,3,36,1,0,0,0,5,38,1,0,0,0,7,40,1,0,0,0,9,42,1,0,0,0,11,44,1,0,
0,0,13,46,1,0,0,0,15,48,1,0,0,0,17,50,1,0,0,0,19,61,1,0,0,0,21,63,1,0,
0,0,23,66,1,0,0,0,25,71,1,0,0,0,27,31,5,35,0,0,28,30,8,0,0,0,29,28,1,
0,0,0,30,33,1,0,0,0,31,29,1,0,0,0,31,32,1,0,0,0,32,34,1,0,0,0,33,31,1,
0,0,0,34,35,6,0,0,0,35,2,1,0,0,0,36,37,5,42,0,0,37,4,1,0,0,0,38,39,5,
64,0,0,39,6,1,0,0,0,40,41,5,92,0,0,41,8,1,0,0,0,42,43,5,124,0,0,43,10,
1,0,0,0,44,45,5,93,0,0,45,12,1,0,0,0,46,47,5,91,0,0,47,14,1,0,0,0,48,
49,5,61,0,0,49,16,1,0,0,0,50,51,5,92,0,0,51,52,5,104,0,0,52,53,5,101,
0,0,53,54,5,97,0,0,54,55,5,100,0,0,55,56,5,101,0,0,56,57,5,114,0,0,57,
18,1,0,0,0,58,59,5,13,0,0,59,62,5,10,0,0,60,62,7,0,0,0,61,58,1,0,0,0,
61,60,1,0,0,0,62,20,1,0,0,0,63,64,5,59,0,0,64,22,1,0,0,0,65,67,7,1,0,
0,66,65,1,0,0,0,67,68,1,0,0,0,68,66,1,0,0,0,68,69,1,0,0,0,69,24,1,0,0,
0,70,72,8,2,0,0,71,70,1,0,0,0,72,73,1,0,0,0,73,71,1,0,0,0,73,74,1,0,0,
0,74,26,1,0,0,0,5,0,31,61,68,73,1,6,0,0
1,6,1,6,1,7,1,7,1,7,1,7,1,7,1,7,1,7,1,7,1,8,1,8,1,8,3,8,60,8,8,1,9,1,
9,1,10,4,10,65,8,10,11,10,12,10,66,1,11,1,11,4,11,71,8,11,11,11,12,11,
72,1,12,4,12,76,8,12,11,12,12,12,77,0,0,13,1,1,3,2,5,3,7,4,9,5,11,6,13,
7,15,8,17,9,19,10,21,11,23,12,25,13,1,0,4,2,0,10,10,13,13,8,0,32,32,160,
160,5760,5760,8192,8202,8232,8233,8239,8239,8287,8287,12288,12288,733,
0,48,57,65,90,97,122,170,170,178,179,181,181,185,186,188,190,192,214,
216,246,248,705,710,721,736,740,748,748,750,750,880,884,886,887,890,893,
895,895,902,902,904,906,908,908,910,929,931,1013,1015,1153,1162,1327,
1329,1366,1369,1369,1376,1416,1488,1514,1519,1522,1568,1610,1632,1641,
1646,1647,1649,1747,1749,1749,1765,1766,1774,1788,1791,1791,1808,1808,
1810,1839,1869,1957,1969,1969,1984,2026,2036,2037,2042,2042,2048,2069,
2074,2074,2084,2084,2088,2088,2112,2136,2144,2154,2160,2183,2185,2190,
2208,2249,2308,2361,2365,2365,2384,2384,2392,2401,2406,2415,2417,2432,
2437,2444,2447,2448,2451,2472,2474,2480,2482,2482,2486,2489,2493,2493,
2510,2510,2524,2525,2527,2529,2534,2545,2548,2553,2556,2556,2565,2570,
2575,2576,2579,2600,2602,2608,2610,2611,2613,2614,2616,2617,2649,2652,
2654,2654,2662,2671,2674,2676,2693,2701,2703,2705,2707,2728,2730,2736,
2738,2739,2741,2745,2749,2749,2768,2768,2784,2785,2790,2799,2809,2809,
2821,2828,2831,2832,2835,2856,2858,2864,2866,2867,2869,2873,2877,2877,
2908,2909,2911,2913,2918,2927,2929,2935,2947,2947,2949,2954,2958,2960,
2962,2965,2969,2970,2972,2972,2974,2975,2979,2980,2984,2986,2990,3001,
3024,3024,3046,3058,3077,3084,3086,3088,3090,3112,3114,3129,3133,3133,
3160,3162,3165,3165,3168,3169,3174,3183,3192,3198,3200,3200,3205,3212,
3214,3216,3218,3240,3242,3251,3253,3257,3261,3261,3293,3294,3296,3297,
3302,3311,3313,3314,3332,3340,3342,3344,3346,3386,3389,3389,3406,3406,
3412,3414,3416,3425,3430,3448,3450,3455,3461,3478,3482,3505,3507,3515,
3517,3517,3520,3526,3558,3567,3585,3632,3634,3635,3648,3654,3664,3673,
3713,3714,3716,3716,3718,3722,3724,3747,3749,3749,3751,3760,3762,3763,
3773,3773,3776,3780,3782,3782,3792,3801,3804,3807,3840,3840,3872,3891,
3904,3911,3913,3948,3976,3980,4096,4138,4159,4169,4176,4181,4186,4189,
4193,4193,4197,4198,4206,4208,4213,4225,4238,4238,4240,4249,4256,4293,
4295,4295,4301,4301,4304,4346,4348,4680,4682,4685,4688,4694,4696,4696,
4698,4701,4704,4744,4746,4749,4752,4784,4786,4789,4792,4798,4800,4800,
4802,4805,4808,4822,4824,4880,4882,4885,4888,4954,4969,4988,4992,5007,
5024,5109,5112,5117,5121,5740,5743,5759,5761,5786,5792,5866,5870,5880,
5888,5905,5919,5937,5952,5969,5984,5996,5998,6000,6016,6067,6103,6103,
6108,6108,6112,6121,6128,6137,6160,6169,6176,6264,6272,6276,6279,6312,
6314,6314,6320,6389,6400,6430,6470,6509,6512,6516,6528,6571,6576,6601,
6608,6618,6656,6678,6688,6740,6784,6793,6800,6809,6823,6823,6917,6963,
6981,6988,6992,7001,7043,7072,7086,7141,7168,7203,7232,7241,7245,7293,
7296,7304,7312,7354,7357,7359,7401,7404,7406,7411,7413,7414,7418,7418,
7424,7615,7680,7957,7960,7965,7968,8005,8008,8013,8016,8023,8025,8025,
8027,8027,8029,8029,8031,8061,8064,8116,8118,8124,8126,8126,8130,8132,
8134,8140,8144,8147,8150,8155,8160,8172,8178,8180,8182,8188,8304,8305,
8308,8313,8319,8329,8336,8348,8450,8450,8455,8455,8458,8467,8469,8469,
8473,8477,8484,8484,8486,8486,8488,8488,8490,8493,8495,8505,8508,8511,
8517,8521,8526,8526,8528,8585,9312,9371,9450,9471,10102,10131,11264,11492,
11499,11502,11506,11507,11517,11517,11520,11557,11559,11559,11565,11565,
11568,11623,11631,11631,11648,11670,11680,11686,11688,11694,11696,11702,
11704,11710,11712,11718,11720,11726,11728,11734,11736,11742,11823,11823,
12293,12295,12321,12329,12337,12341,12344,12348,12353,12438,12445,12447,
12449,12538,12540,12543,12549,12591,12593,12686,12690,12693,12704,12735,
12784,12799,12832,12841,12872,12879,12881,12895,12928,12937,12977,12991,
13312,19903,19968,42124,42192,42237,42240,42508,42512,42539,42560,42606,
42623,42653,42656,42735,42775,42783,42786,42888,42891,42954,42960,42961,
42963,42963,42965,42969,42994,43009,43011,43013,43015,43018,43020,43042,
43056,43061,43072,43123,43138,43187,43216,43225,43250,43255,43259,43259,
43261,43262,43264,43301,43312,43334,43360,43388,43396,43442,43471,43481,
43488,43492,43494,43518,43520,43560,43584,43586,43588,43595,43600,43609,
43616,43638,43642,43642,43646,43695,43697,43697,43701,43702,43705,43709,
43712,43712,43714,43714,43739,43741,43744,43754,43762,43764,43777,43782,
43785,43790,43793,43798,43808,43814,43816,43822,43824,43866,43868,43881,
43888,44002,44016,44025,44032,55203,55216,55238,55243,55291,63744,64109,
64112,64217,64256,64262,64275,64279,64285,64285,64287,64296,64298,64310,
64312,64316,64318,64318,64320,64321,64323,64324,64326,64433,64467,64829,
64848,64911,64914,64967,65008,65019,65136,65140,65142,65276,65296,65305,
65313,65338,65345,65370,65382,65470,65474,65479,65482,65487,65490,65495,
65498,65500,65536,65547,65549,65574,65576,65594,65596,65597,65599,65613,
65616,65629,65664,65786,65799,65843,65856,65912,65930,65931,66176,66204,
66208,66256,66273,66299,66304,66339,66349,66378,66384,66421,66432,66461,
66464,66499,66504,66511,66513,66517,66560,66717,66720,66729,66736,66771,
66776,66811,66816,66855,66864,66915,66928,66938,66940,66954,66956,66962,
66964,66965,66967,66977,66979,66993,66995,67001,67003,67004,67072,67382,
67392,67413,67424,67431,67456,67461,67463,67504,67506,67514,67584,67589,
67592,67592,67594,67637,67639,67640,67644,67644,67647,67669,67672,67702,
67705,67742,67751,67759,67808,67826,67828,67829,67835,67867,67872,67897,
67968,68023,68028,68047,68050,68096,68112,68115,68117,68119,68121,68149,
68160,68168,68192,68222,68224,68255,68288,68295,68297,68324,68331,68335,
68352,68405,68416,68437,68440,68466,68472,68497,68521,68527,68608,68680,
68736,68786,68800,68850,68858,68899,68912,68921,69216,69246,69248,69289,
69296,69297,69376,69415,69424,69445,69457,69460,69488,69505,69552,69579,
69600,69622,69635,69687,69714,69743,69745,69746,69749,69749,69763,69807,
69840,69864,69872,69881,69891,69926,69942,69951,69956,69956,69959,69959,
69968,70002,70006,70006,70019,70066,70081,70084,70096,70106,70108,70108,
70113,70132,70144,70161,70163,70187,70272,70278,70280,70280,70282,70285,
70287,70301,70303,70312,70320,70366,70384,70393,70405,70412,70415,70416,
70419,70440,70442,70448,70450,70451,70453,70457,70461,70461,70480,70480,
70493,70497,70656,70708,70727,70730,70736,70745,70751,70753,70784,70831,
70852,70853,70855,70855,70864,70873,71040,71086,71128,71131,71168,71215,
71236,71236,71248,71257,71296,71338,71352,71352,71360,71369,71424,71450,
71472,71483,71488,71494,71680,71723,71840,71922,71935,71942,71945,71945,
71948,71955,71957,71958,71960,71983,71999,71999,72001,72001,72016,72025,
72096,72103,72106,72144,72161,72161,72163,72163,72192,72192,72203,72242,
72250,72250,72272,72272,72284,72329,72349,72349,72368,72440,72704,72712,
72714,72750,72768,72768,72784,72812,72818,72847,72960,72966,72968,72969,
72971,73008,73030,73030,73040,73049,73056,73061,73063,73064,73066,73097,
73112,73112,73120,73129,73440,73458,73648,73648,73664,73684,73728,74649,
74752,74862,74880,75075,77712,77808,77824,78894,82944,83526,92160,92728,
92736,92766,92768,92777,92784,92862,92864,92873,92880,92909,92928,92975,
92992,92995,93008,93017,93019,93025,93027,93047,93053,93071,93760,93846,
93952,94026,94032,94032,94099,94111,94176,94177,94179,94179,94208,100343,
100352,101589,101632,101640,110576,110579,110581,110587,110589,110590,
110592,110882,110928,110930,110948,110951,110960,111355,113664,113770,
113776,113788,113792,113800,113808,113817,119520,119539,119648,119672,
119808,119892,119894,119964,119966,119967,119970,119970,119973,119974,
119977,119980,119982,119993,119995,119995,119997,120003,120005,120069,
120071,120074,120077,120084,120086,120092,120094,120121,120123,120126,
120128,120132,120134,120134,120138,120144,120146,120485,120488,120512,
120514,120538,120540,120570,120572,120596,120598,120628,120630,120654,
120656,120686,120688,120712,120714,120744,120746,120770,120772,120779,
120782,120831,122624,122654,123136,123180,123191,123197,123200,123209,
123214,123214,123536,123565,123584,123627,123632,123641,124896,124902,
124904,124907,124909,124910,124912,124926,124928,125124,125127,125135,
125184,125251,125259,125259,125264,125273,126065,126123,126125,126127,
126129,126132,126209,126253,126255,126269,126464,126467,126469,126495,
126497,126498,126500,126500,126503,126503,126505,126514,126516,126519,
126521,126521,126523,126523,126530,126530,126535,126535,126537,126537,
126539,126539,126541,126543,126545,126546,126548,126548,126551,126551,
126553,126553,126555,126555,126557,126557,126559,126559,126561,126562,
126564,126564,126567,126570,126572,126578,126580,126583,126585,126588,
126590,126590,126592,126601,126603,126619,126625,126627,126629,126633,
126635,126651,127232,127244,130032,130041,131072,173791,173824,177976,
177984,178205,178208,183969,183984,191456,194560,195101,196608,201546,
17,0,10,10,13,13,32,32,35,35,42,42,59,59,61,61,64,64,91,93,124,124,160,
160,5760,5760,8192,8202,8232,8233,8239,8239,8287,8287,12288,12288,83,
0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,9,1,0,0,0,0,11,1,0,
0,0,0,13,1,0,0,0,0,15,1,0,0,0,0,17,1,0,0,0,0,19,1,0,0,0,0,21,1,0,0,0,
0,23,1,0,0,0,0,25,1,0,0,0,1,27,1,0,0,0,3,36,1,0,0,0,5,38,1,0,0,0,7,40,
1,0,0,0,9,42,1,0,0,0,11,44,1,0,0,0,13,46,1,0,0,0,15,48,1,0,0,0,17,59,
1,0,0,0,19,61,1,0,0,0,21,64,1,0,0,0,23,68,1,0,0,0,25,75,1,0,0,0,27,31,
5,35,0,0,28,30,8,0,0,0,29,28,1,0,0,0,30,33,1,0,0,0,31,29,1,0,0,0,31,32,
1,0,0,0,32,34,1,0,0,0,33,31,1,0,0,0,34,35,6,0,0,0,35,2,1,0,0,0,36,37,
5,42,0,0,37,4,1,0,0,0,38,39,5,92,0,0,39,6,1,0,0,0,40,41,5,124,0,0,41,
8,1,0,0,0,42,43,5,93,0,0,43,10,1,0,0,0,44,45,5,91,0,0,45,12,1,0,0,0,46,
47,5,61,0,0,47,14,1,0,0,0,48,49,5,92,0,0,49,50,5,104,0,0,50,51,5,101,
0,0,51,52,5,97,0,0,52,53,5,100,0,0,53,54,5,101,0,0,54,55,5,114,0,0,55,
16,1,0,0,0,56,57,5,13,0,0,57,60,5,10,0,0,58,60,7,0,0,0,59,56,1,0,0,0,
59,58,1,0,0,0,60,18,1,0,0,0,61,62,5,59,0,0,62,20,1,0,0,0,63,65,7,1,0,
0,64,63,1,0,0,0,65,66,1,0,0,0,66,64,1,0,0,0,66,67,1,0,0,0,67,22,1,0,0,
0,68,70,5,64,0,0,69,71,7,2,0,0,70,69,1,0,0,0,71,72,1,0,0,0,72,70,1,0,
0,0,72,73,1,0,0,0,73,24,1,0,0,0,74,76,8,3,0,0,75,74,1,0,0,0,76,77,1,0,
0,0,77,75,1,0,0,0,77,78,1,0,0,0,78,26,1,0,0,0,6,0,31,59,66,72,77,1,6,
0,0
};
staticData->serializedATN = antlr4::atn::SerializedATNView(serializedATNSegment, sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0]));

Expand Down
6 changes: 3 additions & 3 deletions src/grammar/cpp/zoiaLexer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
class zoiaLexer : public antlr4::Lexer {
public:
enum {
COMMENT = 1, Asterisk = 2, At = 3, Backslash = 4, Bar = 5, BracketsClose = 6,
BracketsOpen = 7, Equals = 8, Header = 9, Newline = 10, Semicolon = 11,
Spaces = 12, Word = 13
COMMENT = 1, Asterisk = 2, Backslash = 3, Bar = 4, BracketsClose = 5,
BracketsOpen = 6, Equals = 7, Header = 8, Newline = 9, Semicolon = 10,
Spaces = 11, Alias = 12, Word = 13
};

explicit zoiaLexer(antlr4::CharStream *input);
Expand Down
Loading

0 comments on commit 94df930

Please sign in to comment.